From 4385ade39c55082ce54bb67aa42d6fc6740719db Mon Sep 17 00:00:00 2001 From: jneen Date: Thu, 26 Mar 2026 13:12:50 -0400 Subject: [PATCH 001/154] simplify the applescript lexer --- .rubocop_todo.yml | 2 - lib/rouge/lexers/apple_script.rb | 370 ----------------------- lib/rouge/lexers/applescript.rb | 84 +++++ lib/rouge/lexers/applescript/keywords.rb | 85 ++++++ lib/rouge/regex_lexer.rb | 6 + spec/visual/samples/applescript | 11 + 6 files changed, 186 insertions(+), 372 deletions(-) delete mode 100644 lib/rouge/lexers/apple_script.rb create mode 100644 lib/rouge/lexers/applescript.rb create mode 100644 lib/rouge/lexers/applescript/keywords.rb diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index f655d1ee10..9e3f46f062 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -50,7 +50,6 @@ Naming/VariableName: # Offense count: 113 Rouge/NoBuildingAlternationPatternInRegexp: Exclude: - - 'lib/rouge/lexers/apple_script.rb' - 'lib/rouge/lexers/armasm.rb' - 'lib/rouge/lexers/bbcbasic.rb' - 'lib/rouge/lexers/bicep.rb' @@ -91,7 +90,6 @@ Rouge/NoBuildingAlternationPatternInRegexp: # Offense count: 20 Rouge/NoHugeCollections: Exclude: - - 'lib/rouge/lexers/apple_script.rb' - 'lib/rouge/lexers/cobol.rb' - 'lib/rouge/lexers/common_lisp.rb' - 'lib/rouge/lexers/css.rb' diff --git a/lib/rouge/lexers/apple_script.rb b/lib/rouge/lexers/apple_script.rb deleted file mode 100644 index 7a47b2906f..0000000000 --- a/lib/rouge/lexers/apple_script.rb +++ /dev/null @@ -1,370 +0,0 @@ -# -*- coding: utf-8 -*- # -# frozen_string_literal: true - -module Rouge - module Lexers - class AppleScript < RegexLexer - title "AppleScript" - desc "The AppleScript scripting language by Apple Inc. (https://developer.apple.com/library/archive/documentation/AppleScript/Conceptual/AppleScriptLangGuide/introduction/ASLR_intro.html)" - - tag 'applescript' - aliases 'applescript' - - filenames '*.applescript', '*.scpt' - - mimetypes 'application/x-applescript' - - def self.literals - @literals ||= ['AppleScript', 'current application', 'false', 'linefeed', - 'missing value', 'pi','quote', 'result', 'return', 'space', - 'tab', 'text item delimiters', 'true', 'version'] - end - - def self.classes - @classes ||= ['alias ', 'application ', 'boolean ', 'class ', 'constant ', - 'date ', 'file ', 'integer ', 'list ', 'number ', 'POSIX file ', - 'real ', 'record ', 'reference ', 'RGB color ', 'script ', - 'text ', 'unit types', '(?:Unicode )?text', 'string'] - end - - def self.builtins - @builtins ||= ['attachment', 'attribute run', 'character', 'day', 'month', - 'paragraph', 'word', 'year'] - end - - def self.handler_params - @handler_params ||= ['about', 'above', 'against', 'apart from', 'around', - 'aside from', 'at', 'below', 'beneath', 'beside', - 'between', 'for', 'given', 'instead of', 'on', 'onto', - 'out of', 'over', 'since'] - end - - def self.commands - @commands ||= ['ASCII (character|number)', 'activate', 'beep', 'choose URL', - 'choose application', 'choose color', 'choose file( name)?', - 'choose folder', 'choose from list', - 'choose remote application', 'clipboard info', - 'close( access)?', 'copy', 'count', 'current date', 'delay', - 'delete', 'display (alert|dialog)', 'do shell script', - 'duplicate', 'exists', 'get eof', 'get volume settings', - 'info for', 'launch', 'list (disks|folder)', 'load script', - 'log', 'make', 'mount volume', 'new', 'offset', - 'open( (for access|location))?', 'path to', 'print', 'quit', - 'random number', 'read', 'round', 'run( script)?', - 'say', 'scripting components', - 'set (eof|the clipboard to|volume)', 'store script', - 'summarize', 'system attribute', 'system info', - 'the clipboard', 'time to GMT', 'write', 'quoted form'] - end - - def self.references - @references ||= ['(in )?back of', '(in )?front of', '[0-9]+(st|nd|rd|th)', - 'first', 'second', 'third', 'fourth', 'fifth', 'sixth', - 'seventh', 'eighth', 'ninth', 'tenth', 'after', 'back', - 'before', 'behind', 'every', 'front', 'index', 'last', - 'middle', 'some', 'that', 'through', 'thru', 'where', 'whose'] - end - - def self.operators - @operators ||= ["and", "or", "is equal", "equals", "(is )?equal to", "is not", - "isn't", "isn't equal( to)?", "is not equal( to)?", - "doesn't equal", "does not equal", "(is )?greater than", - "comes after", "is not less than or equal( to)?", - "isn't less than or equal( to)?", "(is )?less than", - "comes before", "is not greater than or equal( to)?", - "isn't greater than or equal( to)?", - "(is )?greater than or equal( to)?", "is not less than", - "isn't less than", "does not come before", - "doesn't come before", "(is )?less than or equal( to)?", - "is not greater than", "isn't greater than", - "does not come after", "doesn't come after", "starts? with", - "begins? with", "ends? with", "contains?", "does not contain", - "doesn't contain", "is in", "is contained by", "is not in", - "is not contained by", "isn't contained by", "div", "mod", - "not", "(a )?(ref( to)?|reference to)", "is", "does"] - end - - def self.controls - @controls ||= ['considering', 'else', 'error', 'exit', 'from', 'if', - 'ignoring', 'in', 'repeat', 'tell', 'then', 'times', 'to', - 'try', 'until', 'using terms from', 'while', 'whith', - 'with timeout( of)?', 'with transaction', 'by', 'continue', - 'end', 'its?', 'me', 'my', 'return', 'of' , 'as'] - end - - def self.declarations - @declarations ||= ['global', 'local', 'prop(erty)?', 'set', 'get'] - end - - def self.reserved - @reserved ||= ['but', 'put', 'returning', 'the'] - end - - def self.studio_classes - @studio_classes ||= ['action cell', 'alert reply', 'application', 'box', - 'browser( cell)?', 'bundle', 'button( cell)?', 'cell', - 'clip view', 'color well', 'color-panel', - 'combo box( item)?', 'control', - 'data( (cell|column|item|row|source))?', 'default entry', - 'dialog reply', 'document', 'drag info', 'drawer', - 'event', 'font(-panel)?', 'formatter', - 'image( (cell|view))?', 'matrix', 'menu( item)?', 'item', - 'movie( view)?', 'open-panel', 'outline view', 'panel', - 'pasteboard', 'plugin', 'popup button', - 'progress indicator', 'responder', 'save-panel', - 'scroll view', 'secure text field( cell)?', 'slider', - 'sound', 'split view', 'stepper', 'tab view( item)?', - 'table( (column|header cell|header view|view))', - 'text( (field( cell)?|view))?', 'toolbar( item)?', - 'user-defaults', 'view', 'window'] - end - - def self.studio_events - @studio_events ||= ['accept outline drop', 'accept table drop', 'action', - 'activated', 'alert ended', 'awake from nib', 'became key', - 'became main', 'begin editing', 'bounds changed', - 'cell value', 'cell value changed', 'change cell value', - 'change item value', 'changed', 'child of item', - 'choose menu item', 'clicked', 'clicked toolbar item', - 'closed', 'column clicked', 'column moved', - 'column resized', 'conclude drop', 'data representation', - 'deminiaturized', 'dialog ended', 'document nib name', - 'double clicked', 'drag( (entered|exited|updated))?', - 'drop', 'end editing', 'exposed', 'idle', 'item expandable', - 'item value', 'item value changed', 'items changed', - 'keyboard down', 'keyboard up', 'launched', - 'load data representation', 'miniaturized', 'mouse down', - 'mouse dragged', 'mouse entered', 'mouse exited', - 'mouse moved', 'mouse up', 'moved', - 'number of browser rows', 'number of items', - 'number of rows', 'open untitled', 'opened', 'panel ended', - 'parameters updated', 'plugin loaded', 'prepare drop', - 'prepare outline drag', 'prepare outline drop', - 'prepare table drag', 'prepare table drop', - 'read from file', 'resigned active', 'resigned key', - 'resigned main', 'resized( sub views)?', - 'right mouse down', 'right mouse dragged', - 'right mouse up', 'rows changed', 'scroll wheel', - 'selected tab view item', 'selection changed', - 'selection changing', 'should begin editing', - 'should close', 'should collapse item', - 'should end editing', 'should expand item', - 'should open( untitled)?', - 'should quit( after last window closed)?', - 'should select column', 'should select item', - 'should select row', 'should select tab view item', - 'should selection change', 'should zoom', 'shown', - 'update menu item', 'update parameters', - 'update toolbar item', 'was hidden', 'was miniaturized', - 'will become active', 'will close', 'will dismiss', - 'will display browser cell', 'will display cell', - 'will display item cell', 'will display outline cell', - 'will finish launching', 'will hide', 'will miniaturize', - 'will move', 'will open', 'will pop up', 'will quit', - 'will resign active', 'will resize( sub views)?', - 'will select tab view item', 'will show', 'will zoom', - 'write to file', 'zoomed'] - end - - def self.studio_commands - @studio_commands ||= ['animate', 'append', 'call method', 'center', - 'close drawer', 'close panel', 'display', - 'display alert', 'display dialog', 'display panel', 'go', - 'hide', 'highlight', 'increment', 'item for', - 'load image', 'load movie', 'load nib', 'load panel', - 'load sound', 'localized string', 'lock focus', 'log', - 'open drawer', 'path for', 'pause', 'perform action', - 'play', 'register', 'resume', 'scroll', 'select( all)?', - 'show', 'size to fit', 'start', 'step back', - 'step forward', 'stop', 'synchronize', 'unlock focus', - 'update'] - end - - def self.studio_properties - @studio_properties ||= ['accepts arrow key', 'action method', 'active', - 'alignment', 'allowed identifiers', - 'allows branch selection', 'allows column reordering', - 'allows column resizing', 'allows column selection', - 'allows customization', 'allows editing text attributes', - 'allows empty selection', 'allows mixed state', - 'allows multiple selection', 'allows reordering', - 'allows undo', 'alpha( value)?', 'alternate image', - 'alternate increment value', 'alternate title', - 'animation delay', 'associated file name', - 'associated object', 'auto completes', 'auto display', - 'auto enables items', 'auto repeat', 'auto resizes( outline column)?', - 'auto save expanded items', 'auto save name', - 'auto save table columns', 'auto saves configuration', - 'auto scroll', 'auto sizes all columns to fit', - 'auto sizes cells', 'background color', 'bezel state', - 'bezel style', 'bezeled', 'border rect', 'border type', - 'bordered', 'bounds( rotation)?', 'box type', - 'button returned', 'button type', - 'can choose directories', 'can choose files', 'can draw', 'can hide', - 'cell( (background color|size|type))?', 'characters', - 'class', 'click count', 'clicked( data)? column', - 'clicked data item', 'clicked( data)? row', - 'closeable', 'collating', 'color( (mode|panel))', - 'command key down', 'configuration', - 'content(s| (size|view( margins)?))?', 'context', - 'continuous', 'control key down', 'control size', - 'control tint', 'control view', - 'controller visible', 'coordinate system', - 'copies( on scroll)?', 'corner view', 'current cell', - 'current column', 'current( field)? editor', - 'current( menu)? item', 'current row', - 'current tab view item', 'data source', - 'default identifiers', 'delta (x|y|z)', - 'destination window', 'directory', 'display mode', - 'displayed cell', 'document( (edited|rect|view))?', - 'double value', 'dragged column', 'dragged distance', - 'dragged items', 'draws( cell)? background', - 'draws grid', 'dynamically scrolls', 'echos bullets', - 'edge', 'editable', 'edited( data)? column', - 'edited data item', 'edited( data)? row', 'enabled', - 'enclosing scroll view', 'ending page', - 'error handling', 'event number', 'event type', - 'excluded from windows menu', 'executable path', - 'expanded', 'fax number', 'field editor', 'file kind', - 'file name', 'file type', 'first responder', - 'first visible column', 'flipped', 'floating', - 'font( panel)?', 'formatter', 'frameworks path', - 'frontmost', 'gave up', 'grid color', 'has data items', - 'has horizontal ruler', 'has horizontal scroller', - 'has parent data item', 'has resize indicator', - 'has shadow', 'has sub menu', 'has vertical ruler', - 'has vertical scroller', 'header cell', 'header view', - 'hidden', 'hides when deactivated', 'highlights by', - 'horizontal line scroll', 'horizontal page scroll', - 'horizontal ruler view', 'horizontally resizable', - 'icon image', 'id', 'identifier', - 'ignores multiple clicks', - 'image( (alignment|dims when disabled|frame style|scaling))?', - 'imports graphics', 'increment value', - 'indentation per level', 'indeterminate', 'index', - 'integer value', 'intercell spacing', 'item height', - 'key( (code|equivalent( modifier)?|window))?', - 'knob thickness', 'label', 'last( visible)? column', - 'leading offset', 'leaf', 'level', 'line scroll', - 'loaded', 'localized sort', 'location', 'loop mode', - 'main( (bunde|menu|window))?', 'marker follows cell', - 'matrix mode', 'maximum( content)? size', - 'maximum visible columns', - 'menu( form representation)?', 'miniaturizable', - 'miniaturized', 'minimized image', 'minimized title', - 'minimum column width', 'minimum( content)? size', - 'modal', 'modified', 'mouse down state', - 'movie( (controller|file|rect))?', 'muted', 'name', - 'needs display', 'next state', 'next text', - 'number of tick marks', 'only tick mark values', - 'opaque', 'open panel', 'option key down', - 'outline table column', 'page scroll', 'pages across', - 'pages down', 'palette label', 'pane splitter', - 'parent data item', 'parent window', 'pasteboard', - 'path( (names|separator))?', 'playing', - 'plays every frame', 'plays selection only', 'position', - 'preferred edge', 'preferred type', 'pressure', - 'previous text', 'prompt', 'properties', - 'prototype cell', 'pulls down', 'rate', - 'released when closed', 'repeated', - 'requested print time', 'required file type', - 'resizable', 'resized column', 'resource path', - 'returns records', 'reuses columns', 'rich text', - 'roll over', 'row height', 'rulers visible', - 'save panel', 'scripts path', 'scrollable', - 'selectable( identifiers)?', 'selected cell', - 'selected( data)? columns?', 'selected data items?', - 'selected( data)? rows?', 'selected item identifier', - 'selection by rect', 'send action on arrow key', - 'sends action when done editing', 'separates columns', - 'separator item', 'sequence number', 'services menu', - 'shared frameworks path', 'shared support path', - 'sheet', 'shift key down', 'shows alpha', - 'shows state by', 'size( mode)?', - 'smart insert delete enabled', 'sort case sensitivity', - 'sort column', 'sort order', 'sort type', - 'sorted( data rows)?', 'sound', 'source( mask)?', - 'spell checking enabled', 'starting page', 'state', - 'string value', 'sub menu', 'super menu', 'super view', - 'tab key traverses cells', 'tab state', 'tab type', - 'tab view', 'table view', 'tag', 'target( printer)?', - 'text color', 'text container insert', - 'text container origin', 'text returned', - 'tick mark position', 'time stamp', - 'title(d| (cell|font|height|position|rect))?', - 'tool tip', 'toolbar', 'trailing offset', 'transparent', - 'treat packages as directories', 'truncated labels', - 'types', 'unmodified characters', 'update views', - 'use sort indicator', 'user defaults', - 'uses data source', 'uses ruler', 'uses threaded animation', - 'uses title from previous column', 'value wraps', 'version', - 'vertical( (line scroll|page scroll|ruler view))?', 'vertically resizable', 'view', - 'visible( document rect)?', 'volume', 'width', 'window', - 'windows menu', 'wraps', 'zoomable', 'zoomed'] - end - - operators = %r(\b(#{self.operators.to_a.join('|')})\b) - classes = %r(\b(as )(#{self.classes.to_a.join('|')})\b) - literals = %r(\b(#{self.literals.to_a.join('|')})\b) - commands = %r(\b(#{self.commands.to_a.join('|')})\b) - controls = %r(\b(#{self.controls.to_a.join('|')})\b) - declarations = %r(\b(#{self.declarations.to_a.join('|')})\b) - reserved = %r(\b(#{self.reserved.to_a.join('|')})\b) - builtins = %r(\b(#{self.builtins.to_a.join('|')})s?\b) - handler_params = %r(\b(#{self.handler_params.to_a.join('|')})\b) - references = %r(\b(#{self.references.to_a.join('|')})\b) - studio_properties = %r(\b(#{self.studio_properties.to_a.join('|')})\b) - studio_classes = %r(\b(#{self.studio_classes.to_a.join('|')})s?\b) - studio_commands = %r(\b(#{self.studio_commands.to_a.join('|')})\b) - identifiers = %r(\b([a-zA-Z]\w*)\b) - - state :root do - rule %r/\s+/, Text::Whitespace - rule %r/¬\n/, Literal::String::Escape - rule %r/'s\s+/, Text - rule %r/(--|#).*?$/, Comment::Single - rule %r/\(\*/, Comment::Multiline - rule %r/[\(\){}!,.:]/, Punctuation - rule %r/(«)([^»]+)(»)/ do |match| - token Text, match[1] - token Name::Builtin, match[2] - token Text, match[3] - end - rule %r/\b((?:considering|ignoring)\s*)(application responses|case|diacriticals|hyphens|numeric strings|punctuation|white space)/ do |match| - token Keyword, match[1] - token Name::Builtin, match[2] - end - rule %r/(-|\*|\+|&|≠|>=?|<=?|=|≥|≤|\/|÷|\^)/, Operator - rule operators, Operator::Word - rule %r/^(\s*(?:on|end)\s+)'r'(%s)/ do |match| - token Keyword, match[1] - token Name::Function, match[2] - end - rule %r/^(\s*)(in|on|script|to)(\s+)/ do |match| - token Text, match[1] - token Keyword, match[2] - token Text, match[3] - end - rule classes do |match| - token Keyword, match[1] - token Name::Class, match[2] - end - rule literals, Name::Builtin - rule commands, Name::Builtin - rule controls, Keyword - rule declarations, Keyword - rule reserved, Name::Builtin - rule builtins, Name::Builtin - rule handler_params, Name::Builtin - rule studio_properties, Name::Attribute - rule studio_classes, Name::Builtin - rule studio_commands, Name::Builtin - rule references, Name::Builtin - rule %r/"(\\\\|\\"|[^"])*"/, Literal::String::Double - rule identifiers, Name::Variable - rule %r/[-+]?(\d+\.\d*|\d*\.\d+)(E[-+][0-9]+)?/, Literal::Number::Float - rule %r/[-+]?\d+/, Literal::Number::Integer - end - end - end -end diff --git a/lib/rouge/lexers/applescript.rb b/lib/rouge/lexers/applescript.rb new file mode 100644 index 0000000000..ccc4e95a9b --- /dev/null +++ b/lib/rouge/lexers/applescript.rb @@ -0,0 +1,84 @@ +# -*- coding: utf-8 -*- # +# frozen_string_literal: true + +# documentation: https://developer.apple.com/library/archive/documentation/AppleScript/Conceptual/AppleScriptLangGuide/conceptual/ASLR_lexical_conventions.html#//apple_ref/doc/uid/TP40000983-CH214-SW1 + +module Rouge + module Lexers + class AppleScript < RegexLexer + title "AppleScript" + desc "The AppleScript scripting language by Apple Inc. (https://developer.apple.com/library/archive/documentation/AppleScript/Conceptual/AppleScriptLangGuide/introduction/ASLR_intro.html)" + + tag 'applescript' + aliases 'applescript' + + filenames '*.applescript', '*.scpt' + + mimetypes 'application/x-applescript' + + lazy { require_relative 'applescript/keywords' } + + state :root do + rule %r/\s+/, Text::Whitespace + rule %r/¬\n/, Literal::String::Escape + rule %r/'s\s+/, Text + rule %r/(--|#).*?$/, Comment::Single + rule %r/\(\*/, Comment::Multiline, :multicomment + rule %r/[\(\){}!,.:]/, Punctuation + rule %r/(«)([^»]+)(»)/ do |match| + token Text, match[1] + token Name::Builtin, match[2] + token Text, match[3] + end + rule %r/\b((?:considering|ignoring)\s*)(application responses|case|diacriticals|hyphens|numeric strings|punctuation|white space)/ do |match| + token Keyword, match[1] + token Name::Builtin, match[2] + end + rule %r/(-|\*|\+|&|≠|>=?|<=?|=|≥|≤|\/|÷|\^)/, Operator + + rule OPERATOR_RE, Operator::Word + + rule %r/^(\s*(?:on|end)\s+)'r'(%s)/ do |match| + token Keyword, match[1] + token Name::Function, match[2] + end + + rule %r/^(\s*)(in|on|script|to)(\s+)/ do |match| + token Text, match[1] + token Keyword, match[2] + token Text, match[3] + end + + ident = %r/([a-zA-Z]\w*)\b|[|].*?[|]/ + + rule /(as)([ \t]+)(#{ident})/ do |match| + groups Keyword, Text, Name::Class + end + + rule %r/"(\\\\|\\"|[^"])*"/, Str::Double + + rule MULTI_WORD_BUILTINS_RE, Name::Builtin + + rule ident do |m| + if RESERVED.include?(m[0]) + token Keyword + elsif BUILTINS.include?(m[0]) + token Name::Builtin + else + token Name + end + end + + rule %r/[-+]?(\d+\.\d*|\d*\.\d+)(E[-+][0-9]+)?/, Literal::Number::Float + rule %r/[-+]?\d+/, Literal::Number::Integer + end + + state :multicomment do + rule %r/[^()*]+/, Comment::Multiline + rule %r/[(][*]/, Comment::Multiline, :multicomment + rule %r/[*][)]/, Comment::Multiline, :pop! + rule %r/[()*]/, Comment::Multiline + end + end + end +end diff --git a/lib/rouge/lexers/applescript/keywords.rb b/lib/rouge/lexers/applescript/keywords.rb new file mode 100644 index 0000000000..146bd62add --- /dev/null +++ b/lib/rouge/lexers/applescript/keywords.rb @@ -0,0 +1,85 @@ +module Rouge + module Lexers + class AppleScript + RESERVED = Set.new %w( + about above after against and apart from around as aside from + at back before beginning behind below beneath beside between but + by considering contain contains contains continue copy div does + eighth else end equal equals error every exit false fifth first + for fourth from front get given global if ignoring in instead of + into is it its last local me middle mod my ninth not of on onto + or out of over prop property put ref reference repeat return + returning script second set seventh since sixth some tell tenth + that the then third through thru timeout times to transaction + true try until where while whose with without + ) + + MULTI_WORD_BUILTINS_RE = dangerous_alternation_regexp([ + /text +item +delimiters/, + /current +application/, + /missing +value/, + /do +shell +script/, + /display +dialog/, + ]) + + # ref: https://developer.apple.com/library/archive/documentation/AppleScript/Conceptual/AppleScriptLangGuide/conceptual/ASLR_fundamentals.html#//apple_ref/doc/uid/TP40000983-CH218-BAJBDEJI + BUILTINS = Set.new %w( + AppleScript + pi + result + version + application + character + ) + + OPERATOR_RE = dangerous_alternation_regexp([ + "and", + "or", + "is equal", + "equals", + /(is )?equal to/, + "is not", + "isn't", + /isn't equal( to)?/, + /is not equal( to)?/, + "doesn't equal", + "does not equal", + /(is )?greater than/, + "comes after", + /is not less than or equal( to)?/, + /isn't less than or equal( to)?/, + /(is )?less than/, + "comes before", + "is not greater than or equal( to)?", + "isn't greater than or equal( to)?", + /(is )?greater than or equal( to)?/, + "is not less than", + "isn't less than", + "does not come before", + "doesn't come before", + /(is )?less than or equal( to)?/, + "is not greater than", + "isn't greater than", + "does not come after", + "doesn't come after", + "starts? with", + "begins? with", + "ends? with", + "contains?", + "does not contain", + "doesn't contain", + "is in", + "is contained by", + "is not in", + "is not contained by", + "isn't contained by", + "div", + "mod", + "not", + /(a )?(ref( to)?|reference to)/, + "is", + "does", + ]) + end + end +end diff --git a/lib/rouge/regex_lexer.rb b/lib/rouge/regex_lexer.rb index f14a994d56..ed18c7a222 100644 --- a/lib/rouge/regex_lexer.rb +++ b/lib/rouge/regex_lexer.rb @@ -281,6 +281,12 @@ def self.get_state(name) end end + def self.dangerous_alternation_regexp(list) + # rubocop:disable Rouge/NoBuildingAlternationPatternInRegexp + /#{Regexp.union(list)}\b/ + # rubocop:enable Rouge/NoBuildingAlternationPatternInRegexp + end + def inspect_details yield ' stack:[' first = true diff --git a/spec/visual/samples/applescript b/spec/visual/samples/applescript index c6f50cf532..1a3f574b71 100644 --- a/spec/visual/samples/applescript +++ b/spec/visual/samples/applescript @@ -55,3 +55,14 @@ set jp to "日本語" set ru to "Русский" jp & " and " & ru -- returns "日本語 and Русский" + +-- valid identifiers according to the docs: + +set |back&forth| to |*| as |Right*Now!| + +(* nested +multiline +(* comments *) +ending *) + +set thing to 1 From cf0b0f83ac3692f334ba578d27a3f25c0cc5d9cc Mon Sep 17 00:00:00 2001 From: jneen Date: Sat, 4 Apr 2026 11:05:46 -0400 Subject: [PATCH 002/154] raise if both a callback and args are passed --- lib/rouge/lexers/common_lisp.rb | 2 +- lib/rouge/lexers/racket.rb | 2 +- lib/rouge/lexers/scheme.rb | 2 +- lib/rouge/regex_lexer.rb | 8 ++++++++ 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/lib/rouge/lexers/common_lisp.rb b/lib/rouge/lexers/common_lisp.rb index bd779ef6bc..1cc171664f 100644 --- a/lib/rouge/lexers/common_lisp.rb +++ b/lib/rouge/lexers/common_lisp.rb @@ -319,7 +319,7 @@ class complex concatenated-stream cons echo-stream file-stream end rule %r/\(/, Punctuation, :root - rule %r/\)/, Punctuation do + rule %r/\)/ do if stack.size == 1 token Error else diff --git a/lib/rouge/lexers/racket.rb b/lib/rouge/lexers/racket.rb index a005439ac7..fb67ab4bde 100644 --- a/lib/rouge/lexers/racket.rb +++ b/lib/rouge/lexers/racket.rb @@ -548,7 +548,7 @@ def self.builtins end state :command do - rule id, Name::Function do |m| + rule id do |m| if self.class.keywords.include? m[0] token Keyword elsif self.class.builtins.include? m[0] diff --git a/lib/rouge/lexers/scheme.rb b/lib/rouge/lexers/scheme.rb index 96965d9765..4386ffc7d1 100644 --- a/lib/rouge/lexers/scheme.rb +++ b/lib/rouge/lexers/scheme.rb @@ -93,7 +93,7 @@ def self.builtins end state :command do - rule id, Name::Function do |m| + rule id do |m| if self.class.keywords.include? m[0] token Keyword elsif self.class.builtins.include? m[0] diff --git a/lib/rouge/regex_lexer.rb b/lib/rouge/regex_lexer.rb index ed18c7a222..a5d949720b 100644 --- a/lib/rouge/regex_lexer.rb +++ b/lib/rouge/regex_lexer.rb @@ -16,6 +16,12 @@ def to_s end end + class InvalidRule < StandardError + def to_s + "#rule used both arguments and a block. Only one is supported." + end + end + class ClosedState < StandardError attr_reader :state def initialize(state) @@ -139,6 +145,8 @@ def rule(re, tok=nil, next_state=nil, &callback) matches_empty = re =~ '' + raise InvalidRule if callback && tok + callback ||= case next_state when :pop! proc do |stream| From 3c7eb049c59f188976fcb2f665c0a1c59524d595 Mon Sep 17 00:00:00 2001 From: jneen Date: Sat, 4 Apr 2026 10:29:47 -0400 Subject: [PATCH 003/154] disable Lint/NonLocalExitFromIterator --- .rubocop.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.rubocop.yml b/.rubocop.yml index 5290220358..e5eeb4995d 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -100,6 +100,9 @@ Lint/NestedPercentLiteral: Lint/DuplicateBranch: Enabled: false +Lint/NonLocalExitFromIterator: + Enabled: false + Naming/BlockForwarding: Enabled: false From ec70b316856c8d926403a00b9bd9dc25d6b78460 Mon Sep 17 00:00:00 2001 From: jneen Date: Sat, 4 Apr 2026 10:55:43 -0400 Subject: [PATCH 004/154] add minitest-focus --- Gemfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Gemfile b/Gemfile index 4eed883182..a869eb8f3a 100644 --- a/Gemfile +++ b/Gemfile @@ -8,6 +8,7 @@ gem 'rake' gem 'minitest', '>= 5.0' gem 'minitest-power_assert' +gem 'minitest-focus' gem 'power_assert', '~> 2.0' # don't try to install redcarpet under jruby From 7ff389290f27fac437e4d2e6ca027089e7c56dc7 Mon Sep 17 00:00:00 2001 From: jneen Date: Sat, 4 Apr 2026 11:11:18 -0400 Subject: [PATCH 005/154] new keywords API --- docs/LexerDevelopment.md | 84 +++++++++---- lib/rouge/lexers/llvm.rb | 16 +-- lib/rouge/lexers/powershell.rb | 13 +- lib/rouge/regex_lexer.rb | 223 +++++++++++++++++++++++++-------- spec/lexers/cpp_spec.rb | 8 ++ 5 files changed, 249 insertions(+), 95 deletions(-) diff --git a/docs/LexerDevelopment.md b/docs/LexerDevelopment.md index 7d7cde2ef2..34ec397657 100644 --- a/docs/LexerDevelopment.md +++ b/docs/LexerDevelopment.md @@ -303,7 +303,7 @@ Filename Globs][conflict-globs] below. [conflict-globs]: #conflicting-filename-globs -#### Special Words +#### Keywords, Builtins, and other word sets Most programming languages reserve certain words for use as identifiers that have a special meaning in the language. To make regular expressions that search @@ -311,47 +311,83 @@ for these words easier, many lexers will put the applicable keywords in a set and make them available in a particular way (be it as a local variable, an instance variable or what have you). -For performance and safety, we strongly recommend lexers use a class method: +For small sets, you can simply use a constant: ```rb module Rouge module Lexers class YetAnotherLanguage < RegexLexer - ... + # ... + KEYWORDS = Set.new %w(key words used in this language) + # ... + end + end +end +``` + +If the keyword sets are very large (>75 elements or so), please put them in a constant in a separate file, which is lazily loaded: + +``` +# lib/rouge/lexers/my_lang.rb +module Rouge + module Lexers + class MyLang < RegexLexer + tag :my_lang + # ... - def self.keywords - @keywords ||= Set.new %w(key words used in this language) + lazy do + requre_relative 'my_lang/keywords' end - ... + # ... end end end -``` - -These keywords can then be used like so: -```rb -rule /\w+/ do |m| - if self.class.keywords.include?(m[0]) - token Keyword - elsif - token Name +# lib/rouge/lexers/my_lang/keywords.rb +module Rouge + module Lexers + class MyLang + KEYWORDS = Set.new %w(massive set goes here) + end end end ``` -In some cases, you may want to interpolate your keywords into a regular -expression. **We strongly recommend you avoid doing this.** Having a large -number of rules that are searching for particular words is not as performant as -a rule with a generic pattern with a block that checks whether the pattern is a -member of a predefined set and assigns tokens, pushes new states, etc. - -If you do need to use interpolation, be careful to use the `\b` anchor to avoid -inadvertently matching part of a longer word (eg. `if` matching `iff`):: +This way, they will not affect users of Rouge who are not using your language. These keywords can then be used with the special `#keywords` api: ```rb -rule /\b(#{keywords.join('|')})\b/, Keyword +state :my_cool_state do + # ... + + # Use a "covering regex" here, like /\w+/, which matches the general form + # of keywords. This should be possible for almost all languages. If there is no + # appropriate regex that will cover all cases, consider special-casing some of + # the exceptions. If the language has *truly* free-form keyword sets (like Gherkin), + # please let us know and we'll consider allowing a large regex instead. + keywords %r/\w+/ do + # this method acts much like a normal #rule call - it takes a token type, and + # potentially an action block. Except instead of a regex, it takes a Set, which + # the match from above will be checked against, in the order they appear here. + rule KEYWORDS, Keyword + rule BUILTINS, Name::Builtin, :pop! + + # symbols will be mapped to class methods on the lexer + rule :small_set, Name::Function + + # or you can just inline small sets here + rule Set[%w(one two three four)], Num::Integer + # etc... + + # optional: transform the match before checking set membership + transform(&:downcase) + + # optional: a default rule if the match isn't contained in any of the sets. + # If this is not given, Rouge will simply fall through to the next rules after + # the keywords block. + default Name + end +end ``` #### Startup diff --git a/lib/rouge/lexers/llvm.rb b/lib/rouge/lexers/llvm.rb index 93e51f8556..e95d4106a7 100644 --- a/lib/rouge/lexers/llvm.rb +++ b/lib/rouge/lexers/llvm.rb @@ -42,16 +42,12 @@ class LLVM < RegexLexer rule %r/i[1-9]\d*/, Keyword::Type - rule %r/\w+/ do |m| - if TYPES.include? m[0] - token Keyword::Type - elsif INSTRUCTIONS.include? m[0] - token Keyword - elsif KEYWORDS.include? m[0] - token Keyword - else - token Error - end + keywords %r/\w+/ do + transform(&:downcase) + rule TYPES, Keyword::Type + rule INSTRUCTIONS, Keyword + rule KEYWORDS, Keyword + default Error end end end diff --git a/lib/rouge/lexers/powershell.rb b/lib/rouge/lexers/powershell.rb index 8546dbdf82..dff7ca17a5 100644 --- a/lib/rouge/lexers/powershell.rb +++ b/lib/rouge/lexers/powershell.rb @@ -13,7 +13,7 @@ class Powershell < RegexLexer mimetypes 'text/x-powershell' # https://docs.microsoft.com/en-us/powershell/module/microsoft.powershell.core/about/about_functions_cmdletbindingattribute?view=powershell-6 - ATTRIBUTES = %w( + ATTRIBUTES = Set.new %w( ConfirmImpact DefaultParameterSetName HelpURI PositionalBinding SupportsPaging SupportsShouldProcess ) @@ -170,13 +170,12 @@ class Powershell < RegexLexer state :bracket do rule %r/\]/, Punctuation, :pop! rule %r/[A-Za-z]\w+\./, Name - rule %r/([A-Za-z]\w+)/ do |m| - if ATTRIBUTES.include? m[0] - token Name::Builtin::Pseudo - else - token Name - end + + keywords %r/([A-Za-z]\w+)/ do + rule ATTRIBUTES, Name::Builtin::Pseudo + default Name end + mixin :root end diff --git a/lib/rouge/regex_lexer.rb b/lib/rouge/regex_lexer.rb index a5d949720b..8adc66de32 100644 --- a/lib/rouge/regex_lexer.rb +++ b/lib/rouge/regex_lexer.rb @@ -22,6 +22,12 @@ def to_s end end + class InvalidKeywordSet < StandardError + def to_s + "Invalid keyword set: Keys must be an instance of Set." + end + end + class ClosedState < StandardError attr_reader :state def initialize(state) @@ -64,6 +70,61 @@ def initialize(re, callback) def inspect "#" end + + def self.make_action(re, tok=nil, next_state=nil) + if tok.nil? + raise "please pass a token to yield, or use a callback" + end + + matches_empty = re.match?('') + + case next_state + when :pop! + proc do |stream| + puts " yielding: #{tok.qualname}, #{stream[0].inspect}" if @debug + @output_stream.call(tok, stream[0]) + puts " popping stack: 1" if @debug + @stack.pop or raise 'empty stack!' + end + when :push + proc do |stream| + puts " yielding: #{tok.qualname}, #{stream[0].inspect}" if @debug + @output_stream.call(tok, stream[0]) + puts " pushing :#{@stack.last.name}" if @debug + @stack.push(@stack.last) + end + when Symbol + proc do |stream| + puts " yielding: #{tok.qualname}, #{stream[0].inspect}" if @debug + @output_stream.call(tok, stream[0]) + state = @states[next_state] || self.class.get_state(next_state) + puts " pushing :#{state.name}" if @debug + @stack.push(state) + end + when Array + # [jneen] TODO: deprecate this. + proc do |stream| + puts " yielding: #{tok.qualname}, #{stream[0].inspect}" if @debug + @output_stream.call(tok, stream[0]) + for i_next_state in next_state do + next @stack.pop if i_next_state == :pop! + next @stack.push(@stack.last) if i_next_state == :push + + state = @states[i_next_state] || self.class.get_state(i_next_state) + puts " pushing :#{state.name}" if @debug + @stack.push(state) + end + end + when nil + # cannot use an empty-matching regexp with no predicate + raise InvalidRegex.new(re) if matches_empty + + proc do |stream| + puts " yielding: #{tok.qualname}, #{stream[0].inspect}" if @debug + @output_stream.call(tok, stream[0]) + end + end + end end # a State is a named set of rules that can be tested for or @@ -83,6 +144,81 @@ def inspect end class StateDSL + class KeywordRule + def initialize(covering_regex, &defn) + @covering_regex = covering_regex + @transform = nil + @default = nil + @group = 0 + @sets = [] + instance_eval(&defn) + end + + def transform(&b) + @mapper = b + end + + def group(g) + @group = g + end + + def rule(keyword_set, *args, &action) + unless keyword_set.is_a?(Set) || keyword_set.is_a?(Symbol) + raise InvalidKeywordSet + end + + raise InvalidRule if action && args.any? + action ||= Rule.make_action(@covering_regex, *args) + + @sets << [keyword_set, action] + end + + def default(*args, &block) + @default = block || Rule.make_action(@covering_regex, *args) + end + + def to_proc + mapper = @mapper + sets = @sets + default = @default + group = @group + + proc do |m| + match = m[group] + match = mapper.call(match) if mapper + puts " checking #{match.inspect} against keyword sets" if @debug + + found = false + sets.each do |(keyword_set, action)| + # [jneen] this has to be late bound because self.class here + # may be a subclass of the lexer class where this was defined + if keyword_set.is_a?(Symbol) + puts " checking keyword set #{keyword_set.inspect}" if @debug + keyword_set = self.class.get_keyword_set(keyword_set) + else + puts " checking keyword set (#{keyword_set.size} elements)" if @debug + end + + next unless keyword_set.include?(match) + + puts " matched keyword set" if @debug + instance_exec(m, &action) + + found = true + break + end + + next if found + + if default + instance_exec(m, &default) + else + fallthrough! + end + end + end + end + attr_reader :rules, :name def initialize(name, &defn) @name = name @@ -136,64 +272,12 @@ def appended(&defn) # methods, including {RegexLexer#push}, {RegexLexer#pop!}, # {RegexLexer#token}, and {RegexLexer#delegate}. The first # argument can be used to access the match groups. - def rule(re, tok=nil, next_state=nil, &callback) + def rule(re, *args, &callback) raise ClosedState.new(self) if @closed - if tok.nil? && callback.nil? - raise "please pass `rule` a token to yield or a callback" - end - matches_empty = re =~ '' - raise InvalidRule if callback && tok - - callback ||= case next_state - when :pop! - proc do |stream| - puts " yielding: #{tok.qualname}, #{stream[0].inspect}" if @debug - @output_stream.call(tok, stream[0]) - puts " popping stack: 1" if @debug - @stack.pop or raise 'empty stack!' - end - when :push - proc do |stream| - puts " yielding: #{tok.qualname}, #{stream[0].inspect}" if @debug - @output_stream.call(tok, stream[0]) - puts " pushing :#{@stack.last.name}" if @debug - @stack.push(@stack.last) - end - when Symbol - proc do |stream| - puts " yielding: #{tok.qualname}, #{stream[0].inspect}" if @debug - @output_stream.call(tok, stream[0]) - state = @states[next_state] || self.class.get_state(next_state) - puts " pushing :#{state.name}" if @debug - @stack.push(state) - end - when nil - # cannot use an empty-matching regexp with no predicate - raise InvalidRegex.new(re) if matches_empty - - proc do |stream| - puts " yielding: #{tok.qualname}, #{stream[0].inspect}" if @debug - @output_stream.call(tok, stream[0]) - end - when Array - proc do |stream| - puts " yielding: #{tok.qualname}, #{stream[0].inspect}" if @debug - @output_stream.call(tok, stream[0]) - for i_next_state in next_state do - next @stack.pop if i_next_state == :pop! - next @stack.push(@stack.last) if i_next_state == :push - - state = @states[i_next_state] || self.class.get_state(i_next_state) - puts " pushing :#{state.name}" if @debug - @stack.push(state) - end - end - else - raise "invalid next state: #{next_state.inspect}" - end + callback ||= Rule.make_action(re, *args) rules << Rule.new(re, callback) @@ -221,6 +305,13 @@ def mixin(state) rules << state.to_s end + # Define keyword sets with an overarching regular expression. + def keywords(covering_regex, &block) + keyword_rule = KeywordRule.new(covering_regex, &block) + + rule(covering_regex, &keyword_rule.to_proc) + end + private def load! return if @loaded @@ -295,6 +386,29 @@ def self.dangerous_alternation_regexp(list) # rubocop:enable Rouge/NoBuildingAlternationPatternInRegexp end + def self.get_keyword_set(name) + @keyword_sets ||= {} + + @keyword_sets[name] ||= begin + # don't recurse past the base class + if self == RegexLexer + raise "no keyword set #{name.inspect} defined - try defining one!" + end + + # Use memory sharing when applicable. If the method is not overridden, + # it is best to use the superclass method so the cache is shared. + set = if singleton_class.method_defined?(name, false) + public_send(name) + else + superclass.get_keyword_set(name) + end + + raise InvalidKeywordSet unless set.is_a?(Set) + + set + end + end + def inspect_details yield ' stack:[' first = true @@ -478,6 +592,7 @@ def recurse(text=nil) # rules, as if the current regex had not matched. Does not affect # the stack. def fallthrough! + puts "falling through" if @debug raise Fallthrough end diff --git a/spec/lexers/cpp_spec.rb b/spec/lexers/cpp_spec.rb index 60a3c8a6bb..e040677f9d 100644 --- a/spec/lexers/cpp_spec.rb +++ b/spec/lexers/cpp_spec.rb @@ -35,4 +35,12 @@ assert_guess :mimetype => 'text/x-c++src' end end + + # [jneen] this tests that inheritance is working correctly for keywords/class methods + describe 'lexing' do + include Support::Lexing + it 'lexes additional keywords' do + assert_tokens_equal('using', ['Keyword', 'using']) + end + end end From 4aab35b915777039f73a83363e4c61a971f403e0 Mon Sep 17 00:00:00 2001 From: jneen Date: Fri, 3 Apr 2026 14:01:42 -0400 Subject: [PATCH 006/154] move keywords api into the core rules section --- docs/LexerDevelopment.md | 76 ++++++++++++++++++++-------------------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/docs/LexerDevelopment.md b/docs/LexerDevelopment.md index 34ec397657..07e087be60 100644 --- a/docs/LexerDevelopment.md +++ b/docs/LexerDevelopment.md @@ -267,42 +267,6 @@ rules, as if the regular expression had not matched. You can see an example of these more complex rules in [the Ruby lexer][ruby-lexer]. -### Additional Features - -While the properties and states are the minimum elements of a lexer that need -to be implemented, a lexer can include additional features. - -#### Source Detection - -```rb -def self.detect?(text) - return true if text.shebang? 'ruby' -end -``` - -Rouge will attempt to guess the appropriate lexer if it is not otherwise clear. -If Rouge is unable to do this on the basis of any tag, associated filename or -associated mimetype, it will try to detect the appropriate lexer on the basis of -the text itself (the source). This is done by calling `self.detect?` on the -possible lexer (a default `self.detect?` method is defined in {Rouge::Lexer} -and simply returns `false`). - -A lexer can implement its own `self.detect?` method that takes a -{Rouge::TextAnalyzer} object as a parameter. If the `self.detect?` method -returns true, the lexer will be selected as the appropriate lexer. - -It is important to note that `self.detect?` should _only_ return `true` if it -is 100% sure that the language is detected. The most common ways for source -code to identify the language it's written in is with a shebang or a doctype -and Rouge provides the {Rouge::TextAnalyzer#shebang} method and the -{Rouge::TextAnalyzer#doctype} method specifically for use with `self.detect?` -to make these checks easy to perform. - -For more general disambiguation between different lexers, see [Conflicting -Filename Globs][conflict-globs] below. - -[conflict-globs]: #conflicting-filename-globs - #### Keywords, Builtins, and other word sets Most programming languages reserve certain words for use as identifiers that @@ -327,7 +291,7 @@ end If the keyword sets are very large (>75 elements or so), please put them in a constant in a separate file, which is lazily loaded: -``` +```rb # lib/rouge/lexers/my_lang.rb module Rouge module Lexers @@ -354,7 +318,7 @@ module Rouge end ``` -This way, they will not affect users of Rouge who are not using your language. These keywords can then be used with the special `#keywords` api: +This way, users of Rouge who are not using your language will not have to load the keyword sets. These keywords can then be used with the special `#keywords` api: ```rb state :my_cool_state do @@ -390,6 +354,42 @@ state :my_cool_state do end ``` +### Additional Features + +While the properties and states are the minimum elements of a lexer that need +to be implemented, a lexer can include additional features. + +#### Source Detection + +```rb +def self.detect?(text) + return true if text.shebang? 'ruby' +end +``` + +Rouge will attempt to guess the appropriate lexer if it is not otherwise clear. +If Rouge is unable to do this on the basis of any tag, associated filename or +associated mimetype, it will try to detect the appropriate lexer on the basis of +the text itself (the source). This is done by calling `self.detect?` on the +possible lexer (a default `self.detect?` method is defined in {Rouge::Lexer} +and simply returns `false`). + +A lexer can implement its own `self.detect?` method that takes a +{Rouge::TextAnalyzer} object as a parameter. If the `self.detect?` method +returns true, the lexer will be selected as the appropriate lexer. + +It is important to note that `self.detect?` should _only_ return `true` if it +is 100% sure that the language is detected. The most common ways for source +code to identify the language it's written in is with a shebang or a doctype +and Rouge provides the {Rouge::TextAnalyzer#shebang} method and the +{Rouge::TextAnalyzer#doctype} method specifically for use with `self.detect?` +to make these checks easy to perform. + +For more general disambiguation between different lexers, see [Conflicting +Filename Globs][conflict-globs] below. + +[conflict-globs]: #conflicting-filename-globs + #### Startup ```rb From a24378b450cb75f78b090b0d367c153d62829f82 Mon Sep 17 00:00:00 2001 From: jneen Date: Fri, 10 Apr 2026 23:29:42 -0400 Subject: [PATCH 007/154] use the keywords api for python --- lib/rouge/lexers/python.rb | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/lib/rouge/lexers/python.rb b/lib/rouge/lexers/python.rb index 39619363f2..3c27058a44 100644 --- a/lib/rouge/lexers/python.rb +++ b/lib/rouge/lexers/python.rb @@ -123,18 +123,12 @@ def current_string end # using negative lookbehind so we don't match property names - rule %r/(? Date: Fri, 3 Apr 2026 14:23:44 -0400 Subject: [PATCH 008/154] use the keywords api for actionscript --- lib/rouge/lexers/actionscript.rb | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/lib/rouge/lexers/actionscript.rb b/lib/rouge/lexers/actionscript.rb index 557320ebdb..7afb691934 100644 --- a/lib/rouge/lexers/actionscript.rb +++ b/lib/rouge/lexers/actionscript.rb @@ -98,7 +98,7 @@ def self.constants end def self.builtins - @builtins ||= %w( + @builtins ||= Set.new %w( void Function Math Class Object RegExp decodeURI decodeURIComponent encodeURI encodeURIComponent @@ -129,22 +129,12 @@ def self.builtins rule %r/[{}]/, Punctuation, :statement - rule id do |m| - if self.class.keywords.include? m[0] - token Keyword - push :expr_start - elsif self.class.declarations.include? m[0] - token Keyword::Declaration - push :expr_start - elsif self.class.reserved.include? m[0] - token Keyword::Reserved - elsif self.class.constants.include? m[0] - token Keyword::Constant - elsif self.class.builtins.include? m[0] - token Name::Builtin - else - token Name::Other - end + keywords id do + rule :keywords, Keyword + rule :declarations, Keyword::Declaration + rule :reserved, Keyword::Reserved + rule :builtins, Name::Builtin + default Name::Other end rule %r/\-?[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?/, Num::Float From 10272473ba0913caadbe8afa6ee600fdf0165d5a Mon Sep 17 00:00:00 2001 From: jneen Date: Fri, 3 Apr 2026 14:23:57 -0400 Subject: [PATCH 009/154] use the keywords api for ada --- lib/rouge/lexers/ada.rb | 86 +++++++++++++++++++---------------------- 1 file changed, 39 insertions(+), 47 deletions(-) diff --git a/lib/rouge/lexers/ada.rb b/lib/rouge/lexers/ada.rb index 62d4ea78a1..b4452fadf9 100644 --- a/lib/rouge/lexers/ada.rb +++ b/lib/rouge/lexers/ada.rb @@ -19,35 +19,28 @@ class Ada < RegexLexer XNUM = /\h(_?\h)*/ EXP = /(E[-+]?#{NUM})?/i - # Return a hash mapping lower-case identifiers to token classes. - def self.idents - @idents ||= Hash.new(Name).tap do |h| - %w( - abort abstract accept access aliased all array at begin body - case constant declare delay delta digits do else elsif end - exception exit for generic goto if in interface is limited - loop new null of others out overriding parallel pragma private - protected raise range record renames requeue return reverse - select separate some synchronized tagged task terminate then - until use when while with - ).each {|w| h[w] = Keyword} - - %w(abs and mod not or rem xor).each {|w| h[w] = Operator::Word} - - %w( - entry function package procedure subtype type - ).each {|w| h[w] = Keyword::Declaration} - - %w( - boolean character constraint_error duration float integer - natural positive long_float long_integer long_long_float - long_long_integer program_error short_float short_integer - short_short_integer storage_error string tasking_error - wide_character wide_string wide_wide_character - wide_wide_string - ).each {|w| h[w] = Name::Builtin} - end - end + KEYWORDS = Set.new %w( + abort abstract accept access aliased all array at begin body + case constant declare delay delta digits do else elsif end + exception exit for generic goto if in interface is limited + loop new null of others out overriding parallel pragma private + protected raise range record renames requeue return reverse + select separate some synchronized tagged task terminate then + until use when while with + ) + + OPERATORS = Set.new %w(abs and mod not or rem xor) + + DECLARATIONS = Set.new %w(entry function package procedure subtype type) + + BUILTINS = Set.new %w( + boolean character constraint_error duration float integer + natural positive long_float long_integer long_long_float + long_long_integer program_error short_float short_integer + short_short_integer storage_error string tasking_error + wide_character wide_string wide_wide_character + wide_wide_string + ) state :whitespace do rule %r{\s+}m, Text @@ -91,21 +84,19 @@ def self.idents state :libunit_name do mixin :whitespace - rule ID do |m| - t = self.class.idents[m[0].downcase] - if t <= Name - # Convert all kinds of Name to namespaces in this context. - token Name::Namespace - else + keywords ID do + transform(&:downcase) + + rule OPERATORS, Operator::Word, :pop! + rule KEYWORDS, Keyword, :pop! + rule DECLARATIONS do + token Keyword::Declaration # Yikes, we're not supposed to get a keyword in a library unit name! # We probably entered this state by mistake, so try to fix it. - token t - if t == Keyword::Declaration - goto :decl_name - else - pop! - end + goto :decl_name end + + default Name::Namespace end rule %r{[.,]}, Punctuation @@ -146,12 +137,13 @@ def self.idents rule %r{[+*/&<=>|]|-|=>|\.\.|\*\*|[:>>|<>|@}, Operator rule %r{[.,:;()\[\]]}, Punctuation - rule ID do |m| - t = self.class.idents[m[0].downcase] - token t - if t == Keyword::Declaration - push :decl_name - end + keywords ID do |m| + transform(&:downcase) + rule KEYWORDS, Keyword + rule OPERATORS, Operator::Word + rule DECLARATIONS, Keyword::Declaration, :decl_name + rule BUILTINS, Name::Builtin + default Name end # Flag word-like things that don't match the ID pattern. From 7a1f2d8381a57311b83892720908c77556830186 Mon Sep 17 00:00:00 2001 From: jneen Date: Fri, 3 Apr 2026 14:37:20 -0400 Subject: [PATCH 010/154] use the keywords api for apex --- lib/rouge/lexers/apex.rb | 37 ++++++++++++++++++------------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/lib/rouge/lexers/apex.rb b/lib/rouge/lexers/apex.rb index 51d14a8c8d..c1f1301539 100644 --- a/lib/rouge/lexers/apex.rb +++ b/lib/rouge/lexers/apex.rb @@ -29,15 +29,15 @@ def self.declarations def self.soql @soql ||= Set.new %w( - SELECT FROM WHERE UPDATE LIKE TYPEOF END USING SCOPE WITH DATA - CATEGORY GROUP BY ROLLUP CUBE HAVING ORDER ASC DESC NULLS FIRST - LAST LIMIT OFFSET FOR VIEW REFERENCE TRACKING VIEWSTAT OR AND + select from where update like typeof end using scope with data + category group by rollup cube having order asc desc nulls first + last limit offset for view reference tracking viewstat or and ) end def self.types @types ||= Set.new %w( - String boolean byte char double float int long short var void + string boolean byte char double float int long short var void ) end @@ -56,22 +56,21 @@ def self.constants rule %r/(?:class|interface)\b/, Keyword::Declaration, :class rule %r/import\b/, Keyword::Namespace, :import + keywords id do + transform(&:downcase) + + rule :keywords, Keyword + rule :soql, Keyword + rule :declarations, Keyword::Declaration + rule :types, Keyword::Type + rule :constants, Keyword::Constant + rule Set['package'], Keyword::Namespace + + default Name + end + rule %r/([@$.]?)(#{id})([:(]?)/io do |m| - lowercased = m[0].downcase - uppercased = m[0].upcase - if self.class.keywords.include? lowercased - token Keyword - elsif self.class.soql.include? uppercased - token Keyword - elsif self.class.declarations.include? lowercased - token Keyword::Declaration - elsif self.class.types.include? lowercased - token Keyword::Type - elsif self.class.constants.include? lowercased - token Keyword::Constant - elsif lowercased == 'package' - token Keyword::Namespace - elsif m[1] == "@" + if m[1] == "@" token Name::Decorator elsif m[3] == ":" groups Operator, Name::Label, Punctuation From f617d512a5e866495365cb17d69347cda9490d43 Mon Sep 17 00:00:00 2001 From: jneen Date: Sat, 4 Apr 2026 10:56:32 -0400 Subject: [PATCH 011/154] use the keywords api for applescript --- lib/rouge/lexers/applescript.rb | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/lib/rouge/lexers/applescript.rb b/lib/rouge/lexers/applescript.rb index ccc4e95a9b..054dd2037b 100644 --- a/lib/rouge/lexers/applescript.rb +++ b/lib/rouge/lexers/applescript.rb @@ -10,7 +10,6 @@ class AppleScript < RegexLexer desc "The AppleScript scripting language by Apple Inc. (https://developer.apple.com/library/archive/documentation/AppleScript/Conceptual/AppleScriptLangGuide/introduction/ASLR_intro.html)" tag 'applescript' - aliases 'applescript' filenames '*.applescript', '*.scpt' @@ -23,7 +22,7 @@ class AppleScript < RegexLexer rule %r/¬\n/, Literal::String::Escape rule %r/'s\s+/, Text rule %r/(--|#).*?$/, Comment::Single - rule %r/\(\*/, Comment::Multiline, :multicomment + rule %r/[(][*]/, Comment::Multiline, :multicomment rule %r/[\(\){}!,.:]/, Punctuation rule %r/(«)([^»]+)(»)/ do |match| token Text, match[1] @@ -51,7 +50,7 @@ class AppleScript < RegexLexer ident = %r/([a-zA-Z]\w*)\b|[|].*?[|]/ - rule /(as)([ \t]+)(#{ident})/ do |match| + rule %r/(as)([ \t]+)(#{ident})/ do |match| groups Keyword, Text, Name::Class end @@ -59,14 +58,10 @@ class AppleScript < RegexLexer rule MULTI_WORD_BUILTINS_RE, Name::Builtin - rule ident do |m| - if RESERVED.include?(m[0]) - token Keyword - elsif BUILTINS.include?(m[0]) - token Name::Builtin - else - token Name - end + keywords ident do + rule RESERVED, Keyword + rule BUILTINS, Name::Builtin + default Name end rule %r/[-+]?(\d+\.\d*|\d*\.\d+)(E[-+][0-9]+)?/, Literal::Number::Float From 6c842b87e89abb0f1d1143660417146225cd010a Mon Sep 17 00:00:00 2001 From: jneen Date: Sat, 4 Apr 2026 10:57:48 -0400 Subject: [PATCH 012/154] use the keywords api for armasm --- lib/rouge/lexers/armasm.rb | 78 +++++++++++++++++++++++--------------- 1 file changed, 48 insertions(+), 30 deletions(-) diff --git a/lib/rouge/lexers/armasm.rb b/lib/rouge/lexers/armasm.rb index 51b4a3885f..e77a5d5e39 100644 --- a/lib/rouge/lexers/armasm.rb +++ b/lib/rouge/lexers/armasm.rb @@ -9,40 +9,33 @@ class ArmAsm < RegexLexer tag 'armasm' filenames '*.s' - def self.preproc_keyword - @preproc_keyword ||= %w( - define elif else endif error if ifdef ifndef include line pragma undef warning - ) - end - def self.file_directive - @file_directive ||= %w( - BIN GET INCBIN INCLUDE LNK + @file_directive ||= Set.new %w( + bin get incbin include lnk ) end def self.general_directive - @general_directive ||= %w( - ALIAS ALIGN AOF AOUT AREA ARM ASSERT ATTR CN CODE16 CODE32 COMMON CP - DATA DCB DCD DCDO DCDU DCFD DCFDU DCFH DCFHU DCFS DCFSU DCI DCI.N DCI.W - DCQ DCQU DCW DCWU DN ELIF ELSE END ENDFUNC ENDIF ENDP ENTRY EQU EXPORT - EXPORTAS EXTERN FIELD FILL FN FRAME FUNCTION GBLA GBLL GBLS GLOBAL IF - IMPORT INFO KEEP LCLA LCLL LCLS LEADR LEAF LTORG MACRO MAP MEND MEXIT - NOFP OPT ORG PRESERVE8 PROC QN RELOC REQUIRE REQUIRE8 RLIST RN ROUT - SETA SETL SETS SN SPACE STRONG SUBT THUMB THUMBX TTL WEND WHILE - \[ \] [|!#*=%&^] + @general_directive ||= Set.new %w( + alias align aof aout area arm assert attr cn code16 code32 common cp + data dcb dcd dcdo dcdu dcfd dcfdu dcfh dcfhu dcfs dcfsu dci dci.n dci.w + dcq dcqu dcw dcwu dn elif else end endfunc endif endp entry equ export + exportas extern field fill fn frame function gbla gbll gbls global if + import info keep lcla lcll lcls leadr leaf ltorg macro map mend mexit + nofp opt org preserve8 proc qn reloc require require8 rlist rn rout + seta setl sets sn space strong subt thumb thumbx ttl wend while ) end def self.shift_or_condition - @shift_or_condition ||= %w( + @shift_or_condition ||= Set.new %w( ASR LSL LSR ROR RRX AL CC CS EQ GE GT HI HS LE LO LS LT MI NE PL VC VS asr lsl lsr ror rrx al cc cs eq ge gt hi hs le lo ls lt mi ne pl vc vs ) end def self.builtin - @builtin ||= %w( + @builtin ||= Set.new %w( ARCHITECTURE AREANAME ARMASM_VERSION CODESIZE COMMANDLINE CONFIG CPU ENDIAN FALSE FPIC FPU INPUTFILE INTER LINENUM LINENUMUP LINENUMUPPER OBJASM_VERSION OPT PC PCSTOREOFFSET REENTRANT ROPI RWPI TRUE VAR @@ -50,17 +43,16 @@ def self.builtin end def self.operator - @operator ||= %w( + @operator ||= Set.new %w( AND BASE CC CC_ENCODING CHR DEF EOR FATTR FEXEC FLOAD FSIZE INDEX LAND LEFT LEN LEOR LNOT LOR LOWERCASE MOD NOT OR RCONST REVERSE_CC RIGHT ROL - ROR SHL SHR STR TARGET_ARCH_[0-9A-Z_]+ TARGET_FEATURE_[0-9A-Z_]+ - TARGET_FPU_[A-Z_] TARGET_PROFILE_[ARM] UAL UPPERCASE + ROR SHL SHR STR UAL UPPERCASE ) end state :root do rule %r/\n/, Text - rule %r/^([ \t]*)(#[ \t]*(?:(?:#{ArmAsm.preproc_keyword.join('|')})(?:[ \t].*)?)?)(\n)/ do + rule %r/^([ \t]*)(#[ \t]*(?:\w+(?:[ \t].*)?)?)(\n)/ do groups Text, Comment::Preproc, Text end rule %r/[ \t]+/, Text, :command @@ -76,11 +68,15 @@ def self.operator goto :args end rule %r/;.*/, Comment, :pop! - rule %r/(?:#{ArmAsm.file_directive.join('|')})\b/ do |m| - token Keyword - goto :filespec + + rule %r/[\[\]|!#*=%&^](?=[; \t\n])/, Keyword + + keywords %r/\w+/ do + transform(&:downcase) + rule(:file_directive) { token Keyword; goto :filespec } + rule :general_directive, Keyword end - rule %r/(?:#{ArmAsm.general_directive.join('|')})(?=[; \t\n])/, Keyword + rule %r/(?:[A-Z][\dA-Z]*|[a-z][\da-z]*)(?:\.[NWnw])?(?:\.[DFIPSUdfipsu]?(?:8|16|32|64)?){,3}\b/, Name::Builtin # rather than attempt to list all opcodes, rely on all-uppercase or all-lowercase rule rule %r/[a-z_]\w*|\|[^|\n]+\|/i, Name::Function # probably a macro name rule %r/\$[a-z]\w*\.?/i, Name::Namespace @@ -90,7 +86,11 @@ def self.operator rule %r/\n/, Text, :pop! rule %r/[ \t]+/, Text rule %r/;.*/, Comment, :pop! - rule %r/(?:#{ArmAsm.shift_or_condition.join('|')})\b/, Name::Builtin + + keywords %r/\w+/ do + rule :shift_or_condition, Name::Builtin + end + rule %r/[a-z_]\w*|\|[^|\n]+\|/i, Name::Variable # various types of symbol rule %r/%[bf]?[at]?\d+(?:[a-z_]\w*)?/i, Name::Label rule %r/(?:&|0x)\h+(?!p)/i, Literal::Number::Hex @@ -99,13 +99,31 @@ def self.operator rule %r/(?:2_[01]+|3_[0-2]+|4_[0-3]+|5_[0-4]+|6_[0-5]+|7_[0-6]+|8_[0-7]+|9_[0-8]+|\d+)(?!e)/i, Literal::Number::Integer rule %r/(?:2_[.01]+|3_[.0-2]+|4_[.0-3]+|5_[.0-4]+|6_[.0-5]+|7_[.0-6]+|8_[.0-7]+|9_[.0-8]+|[.\d]+)(?:e[-+]?\d+)?/i, Literal::Number::Float rule %r/[@:](?=[ \t]*(?:8|16|32|64|128|256)[^\d])/, Operator - rule %r/[.@]|\{(?:#{ArmAsm.builtin.join('|')})\}/, Name::Constant - rule %r/[-!#%&()*+,\/<=>?^{|}]|\[|\]|!=|&&|\/=|<<|<=|<>|==|><|>=|>>|\|\||:(?:#{ArmAsm.operator.join('|')}):/, Operator + rule %r/[.@]/, Name::Constant + + keywords %r/[{]([A-Z_]+)[}]/ do + group 1 + rule :builtin, Name::Constant + end + + rule %r/[-!#%&()*+,\/<=>?^{|}]|\[|\]|!=|&&|\/=|<<|<=|<>|==|><|>=|>>|\|\|/, Operator + + rule %r/:TARGET_(?:ARCH|FEATURE)_[0-9A-Z_]+:/, Operator + rule %r/:TARGET_FPU_[A-Z_]:/, Operator + rule %r/:TARGET_PROFILE_[ARM]:/, Operator + + keywords %r/:([A-Z_]+):/ do + group 1 + rule :operator, Operator + end + rule %r/\$[a-z]\w*\.?/i, Name::Namespace + rule %r/'/ do |m| token Literal::String::Char goto :singlequoted end + rule %r/"/ do |m| token Literal::String::Double goto :doublequoted From 16f4d7716151c5bfca02312fbcfdf264988fd597 Mon Sep 17 00:00:00 2001 From: jneen Date: Sat, 4 Apr 2026 10:58:46 -0400 Subject: [PATCH 013/154] use the keywords api for augeas --- lib/rouge/lexers/augeas.rb | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/lib/rouge/lexers/augeas.rb b/lib/rouge/lexers/augeas.rb index 2fda76fde4..ae2da157b9 100644 --- a/lib/rouge/lexers/augeas.rb +++ b/lib/rouge/lexers/augeas.rb @@ -40,17 +40,16 @@ def self.reserved groups Punctuation, Keyword::Type end - rule %r/\w[\w']*/ do |m| - name = m[0] - if name == "module" - token Keyword::Reserved - push :module - elsif self.class.reserved.include? name - token Keyword::Reserved - elsif name =~ /\A[A-Z]/ - token Keyword::Namespace - else - token Name + keywords %r/\w[\w']*/ do + rule Set["module"], Keyword::Reserved, :module + rule :reserved, Keyword::Reserved + + default do |m| + if m[0].match?(/\A[A-Z]/) + token Keyword::Namespace + else + token Name + end end end From 314f8a915004e8765bca046793a41831ecbb6eb7 Mon Sep 17 00:00:00 2001 From: jneen Date: Sat, 4 Apr 2026 10:59:01 -0400 Subject: [PATCH 014/154] use the keywords api for awk --- lib/rouge/lexers/awk.rb | 29 ++++++++++------------------- 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/lib/rouge/lexers/awk.rb b/lib/rouge/lexers/awk.rb index 8301f86afe..9e24723c22 100644 --- a/lib/rouge/lexers/awk.rb +++ b/lib/rouge/lexers/awk.rb @@ -40,7 +40,7 @@ def self.constants end def self.builtins - @builtins ||= %w( + @builtins ||= Set.new %w( exp log sqrt sin cos atan2 length rand srand int substr index match split sub gsub sprintf system tolower toupper ) @@ -118,24 +118,15 @@ def self.builtins rule %r/[{}]/, Punctuation, :statement - rule id do |m| - if self.class.keywords.include? m[0] - token Keyword - push :expr_start - elsif self.class.declarations.include? m[0] - token Keyword::Declaration - push :expr_start - elsif self.class.reserved.include? m[0] - token Keyword::Reserved - elsif self.class.constants.include? m[0] - token Keyword::Constant - elsif self.class.builtins.include? m[0] - token Name::Builtin - elsif m[0] =~ /^\$/ - token Name::Variable - else - token Name::Other - end + rule %r/[$]\w+/, Name::Variable + + keywords id do + rule :keywords, Keyword, :expr_start + rule :declarations, Keyword::Declaration, :expr_start + rule :reserved, Keyword::Reserved + rule :constants, Keyword::Constant + rule :builtins, Name::Builtin + default Name::Other end rule %r/[0-9]+\.[0-9]+/, Num::Float From 5433aa684124b0012064e99e090259f51ada1539 Mon Sep 17 00:00:00 2001 From: jneen Date: Wed, 8 Apr 2026 11:13:14 -0400 Subject: [PATCH 015/154] use the keywords api for batchfile --- lib/rouge/lexers/batchfile.rb | 43 ++++++++++++++++------------------- 1 file changed, 19 insertions(+), 24 deletions(-) diff --git a/lib/rouge/lexers/batchfile.rb b/lib/rouge/lexers/batchfile.rb index cc61d85033..84971222ce 100644 --- a/lib/rouge/lexers/batchfile.rb +++ b/lib/rouge/lexers/batchfile.rb @@ -14,26 +14,26 @@ class Batchfile < RegexLexer mimetypes 'application/bat', 'application/x-bat', 'application/x-msdos-program' def self.keywords - @keywords ||= %w( + @keywords ||= Set.new %w( if else for in do goto call exit ) end def self.operator_words - @operator_words ||= %w( + @operator_words ||= Set.new %w( exist defined errorlevel cmdextversion not equ neq lss leq gtr geq ) end def self.devices - @devices ||= %w( + @devices ||= Set.new %w( con prn aux nul com1 com2 com3 com4 com5 com6 com7 com8 com9 lpt1 lpt2 lpt3 lpt4 lpt5 lpt6 lpt7 lpt8 lpt9 ) end def self.builtin_commands - @builtin_commands ||= %w( + @builtin_commands ||= Set.new %w( assoc attrib break bcdedit cacls cd chcp chdir chkdsk chkntfs choice cls cmd color comp compact convert copy date del dir diskpart doskey dpath driverquery echo endlocal erase fc find findstr format fsutil @@ -46,7 +46,7 @@ def self.builtin_commands end def self.other_commands - @other_commands ||= %w( + @other_commands ||= Set.new %w( addusers admodcmd ansicon arp at bcdboot bitsadmin browstat certreq certutil change cidiag cipher cleanmgr clip cmdkey compress convertcp coreinfo csccmd csvde cscript curl debug defrag delprof deltree devcon @@ -71,7 +71,7 @@ def self.other_commands end def self.attributes - @attributes ||= %w( + @attributes ||= Set.new %w( on off disable enableextensions enabledelayedexpansion ) end @@ -86,24 +86,19 @@ def self.attributes # Labels rule %r/:[a-z]+/i, Name::Label - rule %r/([a-z]\w*)(\.exe|com|bat|cmd|msi)?/i do |m| - if self.class.devices.include? m[1] - groups Keyword::Reserved, Error - elsif self.class.keywords.include? m[1] - groups Keyword, Error - elsif self.class.operator_words.include? m[1] - groups Operator::Word, Error - elsif self.class.builtin_commands.include? m[1] - token Name::Builtin - elsif self.class.other_commands.include? m[1] - token Name::Builtin - elsif self.class.attributes.include? m[1] - groups Name::Attribute, Error - elsif "set".casecmp m[1] - groups Keyword::Declaration, Error - else - token Text - end + keywords %r/([a-z]\w*)(\.exe|com|bat|cmd|msi)?/i do + group 1 + transform(&:downcase) + + rule(:devices) { groups Keyword::Reserved, Error } + rule(:keywords) { groups Keyword, Error } + rule(:operator_words) { groups Operator::Word, Error } + rule :builtin_commands, Name::Builtin + rule :other_commands, Name::Builtin + rule(:attributes) { groups Name::Attribute, Error } + rule(Set["set"]) { groups Keyword::Declaration, Error } + + default Text end rule %r/((?:[\/\+]|--?)[a-z]+)\s*/i, Name::Attribute From c01c076ea304c1b66671d2c085961666281c1809 Mon Sep 17 00:00:00 2001 From: jneen Date: Wed, 8 Apr 2026 11:20:09 -0400 Subject: [PATCH 016/154] allow joined regexes for BBCBasic --- .rubocop_todo.yml | 1 - lib/rouge/lexers/bbcbasic.rb | 17 ++++++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index 9e3f46f062..ab6e86c7ac 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -51,7 +51,6 @@ Naming/VariableName: Rouge/NoBuildingAlternationPatternInRegexp: Exclude: - 'lib/rouge/lexers/armasm.rb' - - 'lib/rouge/lexers/bbcbasic.rb' - 'lib/rouge/lexers/bicep.rb' - 'lib/rouge/lexers/cfscript.rb' - 'lib/rouge/lexers/cmhg.rb' diff --git a/lib/rouge/lexers/bbcbasic.rb b/lib/rouge/lexers/bbcbasic.rb index c64a1b6bde..22e994e5da 100644 --- a/lib/rouge/lexers/bbcbasic.rb +++ b/lib/rouge/lexers/bbcbasic.rb @@ -9,12 +9,6 @@ class BBCBASIC < RegexLexer tag 'bbcbasic' filenames '*,fd1' - def self.punctuation - @punctuation ||= %w( - [,;'~] SPC TAB - ) - end - def self.function @function ||= %w( ABS ACS ADVAL ASC ASN ATN BEATS BEAT BGET# CHR\$ COS COUNT DEG DIM @@ -49,9 +43,16 @@ def self.constant end state :expression do + # [jneen] inexplicably, BBC Basic does not require spaces or other delimiters + # around its keywords. I am allowing the use of joined regexes here as a special + # exception. + + # rubocop:disable Rouge/NoBuildingAlternationPatternInRegexp rule %r/#{BBCBASIC.function.join('|')}/o, Name::Builtin # function or pseudo-variable rule %r/#{BBCBASIC.operator.join('|')}/o, Operator rule %r/#{BBCBASIC.constant.join('|')}/o, Name::Constant + # rubocop:enable Rouge/NoBuildingAlternationPatternInRegexp + rule %r/"[^"]*"/o, Literal::String rule %r/[a-z_`][\w`]*[$%]?/io, Name::Variable rule %r/@%/o, Name::Variable @@ -75,9 +76,11 @@ def self.constant rule %r/[\[]/o, Keyword, :assembly1 rule %r/REM *>.*/o, Comment::Special rule %r/REM.*/o, Comment + # rubocop:disable Rouge/NoBuildingAlternationPatternInRegexp rule %r/(?:#{BBCBASIC.statement.join('|')}|CIRCLE(?: *FILL)?|DEF *(?:FN|PROC)|DRAW(?: *BY)?|DIM(?!\()|ELLIPSE(?: *FILL)?|ERROR(?: *EXT)?|FILL(?: *BY)?|INPUT(?:#| *LINE)?|LINE(?: *INPUT)?|LOCAL(?: *DATA| *ERROR)?|MOUSE(?: *COLOUR| *OFF| *ON| *RECTANGLE| *STEP| *TO)?|MOVE(?: *BY)?|ON(?! *ERROR)|ON *ERROR *(?:LOCAL|OFF)?|POINT(?: *BY)?(?!\()|RECTANGE(?: *FILL)?|RESTORE(?: *DATA| *ERROR)?|TRACE(?: *CLOSE| *ENDPROC| *OFF| *STEP(?: *FN| *ON| *PROC)?| *TO)?)/o, Keyword + # rubocop:enable Rouge/NoBuildingAlternationPatternInRegexp mixin :expression - rule %r/#{BBCBASIC.punctuation.join('|')}/o, Punctuation + rule %r/[,;'~]|SPC|TAB/, Punctuation end # Assembly statements are parsed as From 1fabb1a1b0b6c2db919b23139fd391603141d244 Mon Sep 17 00:00:00 2001 From: jneen Date: Wed, 8 Apr 2026 11:29:21 -0400 Subject: [PATCH 017/154] use the keywords api for Bicep --- .rubocop_todo.yml | 1 - lib/rouge/lexers/bicep.rb | 53 ++++++++++++++++++++------------------- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index ab6e86c7ac..fa1e2f0e0a 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -51,7 +51,6 @@ Naming/VariableName: Rouge/NoBuildingAlternationPatternInRegexp: Exclude: - 'lib/rouge/lexers/armasm.rb' - - 'lib/rouge/lexers/bicep.rb' - 'lib/rouge/lexers/cfscript.rb' - 'lib/rouge/lexers/cmhg.rb' - 'lib/rouge/lexers/crystal.rb' diff --git a/lib/rouge/lexers/bicep.rb b/lib/rouge/lexers/bicep.rb index 1bbbcfa429..43c87d0033 100644 --- a/lib/rouge/lexers/bicep.rb +++ b/lib/rouge/lexers/bicep.rb @@ -9,8 +9,10 @@ class Bicep < Rouge::RegexLexer def self.keywords @keywords ||= Set.new %w( - as assert existing extends extension false for from func if import in metadata module - none null output param provider resource targetScope test true type using var void with + as assert existing extends extension false for from func + if import in metadata module none null output param + provider resource targetScope test true type using var + void with ) end @@ -20,21 +22,25 @@ def self.datatypes def self.functions @functions ||= Set.new %w( - array base64 base64ToJson base64ToString bool cidrHost cidrSubnet concat contains dataUri - dataUriToString dateTimeAdd dateTimeFromEpoch dateTimeToEpoch deployer deployment empty endsWith - environment extensionResourceId fail filter first flatten format getSecret groupBy guid indexOf int - intersection items join json last lastIndexOf length list* listAccountSas listKeys listSecrets loadFileAsBase64 - loadJsonContent loadTextContent loadYamlContent managementGroup managementGroupResourceId map mapValue max min - newGuid objectKeys padLeft parseCidr pickZones range readEnvironmentVariable reduce reference replace resourceGroup - resourceId shallowMerge skip sort split startsWith string subscription subscriptionResourceId substring take tenant - tenantResourceId toLogicalZone toLower toObject toPhysicalZone toUpper trim union uniqueString uri uriComponent - uriComponentToString utcNow + array base64 base64ToJson base64ToString bool cidrHost + cidrSubnet concat contains dataUri dataUriToString + dateTimeAdd dateTimeFromEpoch dateTimeToEpoch deployer + deployment empty endsWith environment extensionResourceId + fail filter first flatten format getSecret groupBy guid + indexOf int intersection items join json last lastIndexOf + length list* listAccountSas listKeys listSecrets + loadFileAsBase64 loadJsonContent loadTextContent + loadYamlContent managementGroup managementGroupResourceId + map mapValue max min newGuid objectKeys padLeft parseCidr + pickZones range readEnvironmentVariable reduce reference + replace resourceGroup resourceId shallowMerge skip sort + split startsWith string subscription subscriptionResourceId + substring take tenant tenantResourceId toLogicalZone + toLower toObject toPhysicalZone toUpper trim union + uniqueString uri uriComponent uriComponentToString utcNow ) end - operators = %w(+ - * / % < <= > >= == != =~ !~ && || ! ?? ... .?) - - punctuations = %w(( ) { } [ ] , : ; = .) state :root do mixin :comments @@ -46,26 +52,21 @@ def self.functions rule %r/\b\d+\b/, Num # Rules for sets of reserved keywords - rule %r/\b\w+\b/ do |m| - if self.class.keywords.include? m[0] - token Keyword - elsif self.class.datatypes.include? m[0] - token Keyword::Type - elsif self.class.functions.include? m[0] - token Name::Function - else - token Name - end + keywords %r/\w+/ do + rule :keywords, Keyword + rule :datatypes, Keyword::Type + rule :functions, Name::Function + default Name end # Match operators - rule %r/#{operators.map { |o| Regexp.escape(o) }.join('|')}/, Operator + rule %r(<=|>=|==|=>|!=|=~|!~|[&][&]|[|][|]|[?][?]|[.][.][.]|[.][?]|[-+!*/%<]), Operator # Enter a state when encountering an opening curly bracket rule %r/{/, Punctuation::Indicator, :block # Match punctuation - rule %r/#{punctuations.map { |p| Regexp.escape(p) }.join('|')}/, Punctuation + rule %r/[(){}\[\],:;=.]/, Punctuation # Match identifiers rule %r/[a-zA-Z_]\w*/, Name From 4e90335c5f89bb6c38ba7093c910a60b813973f9 Mon Sep 17 00:00:00 2001 From: jneen Date: Wed, 8 Apr 2026 12:18:44 -0400 Subject: [PATCH 018/154] eliminate regex join in bpf --- lib/rouge/lexers/bpf.rb | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/lib/rouge/lexers/bpf.rb b/lib/rouge/lexers/bpf.rb index cd3307ceda..7730cf5503 100644 --- a/lib/rouge/lexers/bpf.rb +++ b/lib/rouge/lexers/bpf.rb @@ -8,13 +8,8 @@ class BPF < RegexLexer desc "BPF bytecode syntax" tag 'bpf' - TYPE_KEYWORDS = %w( - u8 u16 u32 u64 s8 s16 s32 s64 ll - ).join('|') - - MISC_KEYWORDS = %w( - be16 be32 be64 le16 le32 le64 bswap16 bswap32 bswap64 exit lock map - ).join('|') + TYPE_KEYWORDS = %r/[us](?:8|16|32|64)|ll/ + MISC_KEYWORDS = %r/(?:[bl]e|bswap)(?:16|32|64)|exit|lock|map/ state :root do # Line numbers and hexadecimal output from bpftool/objdump @@ -67,7 +62,7 @@ class BPF < RegexLexer rule %r/\/\//, Comment::Single, :linecomment rule %r/\/\*/, Comment::Multiline, :multilinescomment - rule %r/#{MISC_KEYWORDS}/i, Keyword + rule MISC_KEYWORDS, Keyword # Literals and global objects (maps) refered by name rule %r/([-]?0x\h+|[-]?\d+)(\s*)(ll)?/i do From cc10aa0cd96d79b04158fdcd37b025a9a750e50a Mon Sep 17 00:00:00 2001 From: jneen Date: Wed, 8 Apr 2026 12:27:56 -0400 Subject: [PATCH 019/154] use the keywords api for brightscript --- .rubocop_todo.yml | 1 - lib/rouge/lexers/brightscript.rb | 33 ++++++++++++++------------------ 2 files changed, 14 insertions(+), 20 deletions(-) diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index fa1e2f0e0a..4acc906e0b 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -40,7 +40,6 @@ Naming/MethodName: # SupportedStyles: snake_case, camelCase Naming/VariableName: Exclude: - - 'lib/rouge/lexers/brightscript.rb' - 'lib/rouge/lexers/dafny.rb' - 'lib/rouge/lexers/freefem.rb' - 'lib/rouge/lexers/igorpro.rb' diff --git a/lib/rouge/lexers/brightscript.rb b/lib/rouge/lexers/brightscript.rb index e1013b973e..70710c6315 100644 --- a/lib/rouge/lexers/brightscript.rb +++ b/lib/rouge/lexers/brightscript.rb @@ -120,25 +120,20 @@ def self.builtins rule %r/[()\[\],.;{}]/, Punctuation - rule id do |m| - caseSensitiveChunk = m[0] - caseInsensitiveChunk = m[0].upcase - - if self.class.builtins.include?(caseSensitiveChunk) - token Keyword::Reserved - elsif self.class.keyword_reserved.include?(caseInsensitiveChunk) - token Keyword::Reserved - elsif self.class.keyword_reserved_unsupported.include?(caseInsensitiveChunk) - token Keyword::Reserved - elsif self.class.keyword_type.include?(caseInsensitiveChunk) - token Keyword::Type - elsif self.class.name_builtin.include?(caseInsensitiveChunk) - token Name::Builtin - elsif self.class.operator_word.include?(caseInsensitiveChunk) - token Operator::Word - else - token Name - end + # separate block for case sensitivity + keywords id do + rule :builtins, Keyword::Reserved + end + + keywords id do + transform(&:upcase) + + rule :keyword_reserved, Keyword::Reserved + rule :keyword_reserved_unsupported, Keyword::Reserved + rule :keyword_type, Keyword::Type + rule :name_builtin, Name::Builtin + rule :operator_word, Operator::Word + default Name end end end From ceeb695121e6e9e45b0f0a2e29a76b9d20f22406 Mon Sep 17 00:00:00 2001 From: jneen Date: Wed, 8 Apr 2026 12:43:38 -0400 Subject: [PATCH 020/154] use the keywords api for BSL --- lib/rouge/lexers/bsl.rb | 60 +------ lib/rouge/lexers/bsl/keywords.rb | 280 +++++++++++++++++++++++++++++++ spec/visual/samples/bsl | 3 + 3 files changed, 292 insertions(+), 51 deletions(-) create mode 100644 lib/rouge/lexers/bsl/keywords.rb diff --git a/lib/rouge/lexers/bsl.rb b/lib/rouge/lexers/bsl.rb index 109d81f283..02c2f59587 100644 --- a/lib/rouge/lexers/bsl.rb +++ b/lib/rouge/lexers/bsl.rb @@ -9,55 +9,7 @@ class Bsl < RegexLexer tag 'bsl' filenames '*.bsl', '*.os' - KEYWORDS = /(?<=[^\wа-яё]|^)(?: - КонецПроцедуры | EndProcedure | КонецФункции | EndFunction - | Прервать | Break | Продолжить | Continue - | Возврат | Return | Если | If - | Иначе | Else | ИначеЕсли | ElsIf - | Тогда | Then | КонецЕсли | EndIf - | Попытка | Try | Исключение | Except - | КонецПопытки | EndTry | Raise | ВызватьИсключение - | Пока | While | Для | For - | Каждого | Each | Из | In - | По | To | Цикл | Do - | КонецЦикла | EndDo | НЕ | NOT - | И | AND | ИЛИ | OR - | Новый | New | Процедура | Procedure - | Функция | Function | Перем | Var - | Экспорт | Export | Знач | Val - )(?=[^\wа-яё]|$)/ix - - BUILTINS = /(?<=[^\wа-яё]|^)(?: - СтрДлина|StrLen|СокрЛ|TrimL|СокрП|TrimR|СокрЛП|TrimAll|Лев|Left|Прав|Right|Сред|Mid|СтрНайти|StrFind|ВРег|Upper|НРег|Lower|ТРег|Title|Символ|Char|КодСимвола|CharCode|ПустаяСтрока|IsBlankString|СтрЗаменить|StrReplace|СтрЧислоСтрок|StrLineCount|СтрПолучитьСтроку|StrGetLine|СтрЧислоВхождений|StrOccurrenceCount|СтрСравнить|StrCompare|СтрНачинаетсяС|StrStartWith|СтрЗаканчиваетсяНа|StrEndsWith|СтрРазделить|StrSplit|СтрСоединить|StrConcat - | Цел|Int|Окр|Round|ACos|ACos|ASin|ASin|ATan|ATan|Cos|Cos|Exp|Exp|Log|Log|Log10|Log10|Pow|Pow|Sin|Sin|Sqrt|Sqrt|Tan|Tan - | Год|Year|Месяц|Month|День|Day|Час|Hour|Минута|Minute|Секунда|Second|НачалоГода|BegOfYear|НачалоДня|BegOfDay|НачалоКвартала|BegOfQuarter|НачалоМесяца|BegOfMonth|НачалоМинуты|BegOfMinute|НачалоНедели|BegOfWeek|НачалоЧаса|BegOfHour|КонецГода|EndOfYear|КонецДня|EndOfDay|КонецКвартала|EndOfQuarter|КонецМесяца|EndOfMonth|КонецМинуты|EndOfMinute|КонецНедели|EndOfWeek|КонецЧаса|EndOfHour|НеделяГода|WeekOfYear|ДеньГода|DayOfYear|ДеньНедели|WeekDay|ТекущаяДата|CurrentDate|ДобавитьМесяц|AddMonth - | Тип|Type|ТипЗнч|TypeOf - | Булево|Boolean|Число|Number|Строка|String|Дата|Date - | ПоказатьВопрос|ShowQueryBox|Вопрос|DoQueryBox|ПоказатьПредупреждение|ShowMessageBox|Предупреждение|DoMessageBox|Сообщить|Message|ОчиститьСообщения|ClearMessages|ОповеститьОбИзменении|NotifyChanged|Состояние|Status|Сигнал|Beep|ПоказатьЗначение|ShowValue|ОткрытьЗначение|OpenValue|Оповестить|Notify|ОбработкаПрерыванияПользователя|UserInterruptProcessing|ОткрытьСодержаниеСправки|OpenHelpContent|ОткрытьИндексСправки|OpenHelpIndex|ОткрытьСправку|OpenHelp|ПоказатьИнформациюОбОшибке|ShowErrorInfo|КраткоеПредставлениеОшибки|BriefErrorDescription|ПодробноеПредставлениеОшибки|DetailErrorDescription|ПолучитьФорму|GetForm|ЗакрытьСправку|CloseHelp|ПоказатьОповещениеПользователя|ShowUserNotification|ОткрытьФорму|OpenForm|ОткрытьФормуМодально|OpenFormModal|АктивноеОкно|ActiveWindow|ВыполнитьОбработкуОповещения|ExecuteNotifyProcessing - | ПоказатьВводЗначения|ShowInputValue|ВвестиЗначение|InputValue|ПоказатьВводЧисла|ShowInputNumber|ВвестиЧисло|InputNumber|ПоказатьВводСтроки|ShowInputString|ВвестиСтроку|InputString|ПоказатьВводДаты|ShowInputDate|ВвестиДату|InputDate - | Формат|Format|ЧислоПрописью|NumberInWords|НСтр|NStr|ПредставлениеПериода|PeriodPresentation|СтрШаблон|StrTemplate - | ПолучитьОбщийМакет|GetCommonTemplate|ПолучитьОбщуюФорму|GetCommonForm|ПредопределенноеЗначение|PredefinedValue|ПолучитьПолноеИмяПредопределенногоЗначения|GetPredefinedValueFullName - | ПолучитьЗаголовокСистемы|GetCaption|ПолучитьСкоростьКлиентскогоСоединения|GetClientConnectionSpeed|ПодключитьОбработчикОжидания|AttachIdleHandler|УстановитьЗаголовокСистемы|SetCaption|ОтключитьОбработчикОжидания|DetachIdleHandler|ИмяКомпьютера|ComputerName|ЗавершитьРаботуСистемы|Exit|ИмяПользователя|UserName|ПрекратитьРаботуСистемы|Terminate|ПолноеИмяПользователя|UserFullName|ЗаблокироватьРаботуПользователя|LockApplication|КаталогПрограммы|BinDir|КаталогВременныхФайлов|TempFilesDir|ПравоДоступа|AccessRight|РольДоступна|IsInRole|ТекущийЯзык|CurrentLanguage|ТекущийКодЛокализации|CurrentLocaleCode|СтрокаСоединенияИнформационнойБазы|InfoBaseConnectionString|ПодключитьОбработчикОповещения|AttachNotificationHandler|ОтключитьОбработчикОповещения|DetachNotificationHandler|ПолучитьСообщенияПользователю|GetUserMessages|ПараметрыДоступа|AccessParameters|ПредставлениеПриложения|ApplicationPresentation|ТекущийЯзыкСистемы|CurrentSystemLanguage|ЗапуститьСистему|RunSystem|ТекущийРежимЗапуска|CurrentRunMode|УстановитьЧасовойПоясСеанса|SetSessionTimeZone|ЧасовойПоясСеанса|SessionTimeZone|ТекущаяДатаСеанса|CurrentSessionDate|УстановитьКраткийЗаголовокПриложения|SetShortApplicationCaption|ПолучитьКраткийЗаголовокПриложения|GetShortApplicationCaption|ПредставлениеПрава|RightPresentation|ВыполнитьПроверкуПравДоступа|VerifyAccessRights|РабочийКаталогДанныхПользователя|UserDataWorkDir|КаталогДокументов|DocumentsDir|ПолучитьИнформациюЭкрановКлиента|GetClientDisplaysInformation|ТекущийВариантОсновногоШрифтаКлиентскогоПриложения|ClientApplicationBaseFontCurrentVariant|ТекущийВариантИнтерфейсаКлиентскогоПриложения|ClientApplicationInterfaceCurrentVariant|УстановитьЗаголовокКлиентскогоПриложения|SetClientApplicationCaption|ПолучитьЗаголовокКлиентскогоПриложения|GetClientApplicationCaption|НачатьПолучениеКаталогаВременныхФайлов|BeginGettingTempFilesDir|НачатьПолучениеКаталогаДокументов|BeginGettingDocumentsDir|НачатьПолучениеРабочегоКаталогаДанныхПользователя|BeginGettingUserDataWorkDir|ПодключитьОбработчикЗапросаНастроекКлиентаЛицензирования|AttachLicensingClientParametersRequestHandler|ОтключитьОбработчикЗапросаНастроекКлиентаЛицензирования|DetachLicensingClientParametersRequestHandler - | ЗначениеВСтрокуВнутр|ValueToStringInternal|ЗначениеИзСтрокиВнутр|ValueFromStringInternal|ЗначениеВФайл|ValueToFile|ЗначениеИзФайла|ValueFromFile - | КомандаСистемы|System|ЗапуститьПриложение|RunApp|ПолучитьCOMОбъект|GetCOMObject|ПользователиОС|OSUsers|НачатьЗапускПриложения|BeginRunningApplication - | ПодключитьВнешнююКомпоненту|AttachAddIn|НачатьУстановкуВнешнейКомпоненты|BeginInstallAddIn|УстановитьВнешнююКомпоненту|InstallAddIn|НачатьПодключениеВнешнейКомпоненты|BeginAttachingAddIn - | КопироватьФайл|FileCopy|ПереместитьФайл|MoveFile|УдалитьФайлы|DeleteFiles|НайтиФайлы|FindFiles|СоздатьКаталог|CreateDirectory|ПолучитьИмяВременногоФайла|GetTempFileName|РазделитьФайл|SplitFile|ОбъединитьФайлы|MergeFiles|ПолучитьФайл|GetFile|НачатьПомещениеФайла|BeginPutFile|ПоместитьФайл|PutFile|ЭтоАдресВременногоХранилища|IsTempStorageURL|УдалитьИзВременногоХранилища|DeleteFromTempStorage|ПолучитьИзВременногоХранилища|GetFromTempStorage|ПоместитьВоВременноеХранилище|PutToTempStorage|ПодключитьРасширениеРаботыСФайлами|AttachFileSystemExtension|НачатьУстановкуРасширенияРаботыСФайлами|BeginInstallFileSystemExtension|УстановитьРасширениеРаботыСФайлами|InstallFileSystemExtension|ПолучитьФайлы|GetFiles|ПоместитьФайлы|PutFiles|ЗапроситьРазрешениеПользователя|RequestUserPermission|ПолучитьМаскуВсеФайлы|GetAllFilesMask|ПолучитьМаскуВсеФайлыКлиента|GetClientAllFilesMask|ПолучитьМаскуВсеФайлыСервера|GetServerAllFilesMask|ПолучитьРазделительПути|GetPathSeparator|ПолучитьРазделительПутиКлиента|GetClientPathSeparator|ПолучитьРазделительПутиСервера|GetServerPathSeparator|НачатьПодключениеРасширенияРаботыСФайлами|BeginAttachingFileSystemExtension|НачатьЗапросРазрешенияПользователя|BeginRequestingUserPermission|НачатьПоискФайлов|BeginFindingFiles|НачатьСозданиеКаталога|BeginCreatingDirectory|НачатьКопированиеФайла|BeginCopyingFile|НачатьПеремещениеФайла|BeginMovingFile|НачатьУдалениеФайлов|BeginDeletingFiles|НачатьПолучениеФайлов|BeginGettingFiles|НачатьПомещениеФайлов|BeginPuttingFiles - | НачатьТранзакцию|BeginTransaction|ЗафиксироватьТранзакцию|CommitTransaction|ОтменитьТранзакцию|RollbackTransaction|УстановитьМонопольныйРежим|SetExclusiveMode|МонопольныйРежим|ExclusiveMode|ПолучитьОперативнуюОтметкуВремени|GetRealTimeTimestamp|ПолучитьСоединенияИнформационнойБазы|GetInfoBaseConnections|НомерСоединенияИнформационнойБазы|InfoBaseConnectionNumber|КонфигурацияИзменена|ConfigurationChanged|КонфигурацияБазыДанныхИзмененаДинамически|DataBaseConfigurationChangedDynamically|УстановитьВремяОжиданияБлокировкиДанных|SetLockWaitTime|ОбновитьНумерациюОбъектов|RefreshObjectsNumbering|ПолучитьВремяОжиданияБлокировкиДанных|GetLockWaitTime|КодЛокализацииИнформационнойБазы|InfoBaseLocaleCode|УстановитьМинимальнуюДлинуПаролейПользователей|SetUserPasswordMinLength|ПолучитьМинимальнуюДлинуПаролейПользователей|GetUserPasswordMinLength|ИнициализироватьПредопределенныеДанные|InitializePredefinedData|УдалитьДанныеИнформационнойБазы|EraseInfoBaseData|УстановитьПроверкуСложностиПаролейПользователей|SetUserPasswordStrengthCheck|ПолучитьПроверкуСложностиПаролейПользователей|GetUserPasswordStrengthCheck|ПолучитьСтруктуруХраненияБазыДанных|GetDBStorageStructureInfo|УстановитьПривилегированныйРежим|SetPrivilegedMode|ПривилегированныйРежим|PrivilegedMode|ТранзакцияАктивна|TransactionActive|НеобходимостьЗавершенияСоединения|ConnectionStopRequest|НомерСеансаИнформационнойБазы|InfoBaseSessionNumber|ПолучитьСеансыИнформационнойБазы|GetInfoBaseSessions|ЗаблокироватьДанныеДляРедактирования|LockDataForEdit|УстановитьСоединениеСВнешнимИсточникомДанных|ConnectExternalDataSource|РазблокироватьДанныеДляРедактирования|UnlockDataForEdit|РазорватьСоединениеСВнешнимИсточникомДанных|DisconnectExternalDataSource|ПолучитьБлокировкуСеансов|GetSessionsLock|УстановитьБлокировкуСеансов|SetSessionsLock|ОбновитьПовторноИспользуемыеЗначения|RefreshReusableValues|УстановитьБезопасныйРежим|SetSafeMode|БезопасныйРежим|SafeMode|ПолучитьДанныеВыбора|GetChoiceData|УстановитьЧасовойПоясИнформационнойБазы|SetInfoBaseTimeZone|ПолучитьЧасовойПоясИнформационнойБазы|GetInfoBaseTimeZone|ПолучитьОбновлениеКонфигурацииБазыДанных|GetDataBaseConfigurationUpdate|УстановитьБезопасныйРежимРазделенияДанных|SetDataSeparationSafeMode|БезопасныйРежимРазделенияДанных|DataSeparationSafeMode|УстановитьВремяЗасыпанияПассивногоСеанса|SetPassiveSessionHibernateTime|ПолучитьВремяЗасыпанияПассивногоСеанса|GetPassiveSessionHibernateTime|УстановитьВремяЗавершенияСпящегоСеанса|SetHibernateSessionTerminateTime|ПолучитьВремяЗавершенияСпящегоСеанса|GetHibernateSessionTerminateTime|ПолучитьТекущийСеансИнформационнойБазы|GetCurrentInfoBaseSession|ПолучитьИдентификаторКонфигурации|GetConfigurationID|УстановитьНастройкиКлиентаЛицензирования|SetLicensingClientParameters|ПолучитьИмяКлиентаЛицензирования|GetLicensingClientName|ПолучитьДополнительныйПараметрКлиентаЛицензирования|GetLicensingClientAdditionalParameter - | НайтиПомеченныеНаУдаление|FindMarkedForDeletion|НайтиПоСсылкам|FindByRef|УдалитьОбъекты|DeleteObjects|УстановитьОбновлениеПредопределенныхДанныхИнформационнойБазы|SetInfoBasePredefinedDataUpdate|ПолучитьОбновлениеПредопределенныхДанныхИнформационнойБазы|GetInfoBasePredefinedData - | XMLСтрока|XMLString|XMLЗначение|XMLValue|XMLТип|XMLType|XMLТипЗнч|XMLTypeOf|ИзXMLТипа|FromXMLType|ВозможностьЧтенияXML|CanReadXML|ПолучитьXMLТип|GetXMLType|ПрочитатьXML|ReadXML|ЗаписатьXML|WriteXML|НайтиНедопустимыеСимволыXML|FindDisallowedXMLCharacters|ИмпортМоделиXDTO|ImportXDTOModel|СоздатьФабрикуXDTO|CreateXDTOFactory - | ЗаписатьJSON|WriteJSON|ПрочитатьJSON|ReadJSON|ПрочитатьДатуJSON|ReadJSONDate|ЗаписатьДатуJSON|WriteJSONDate - | ЗаписьЖурналаРегистрации|WriteLogEvent|ПолучитьИспользованиеЖурналаРегистрации|GetEventLogUsing|УстановитьИспользованиеЖурналаРегистрации|SetEventLogUsing|ПредставлениеСобытияЖурналаРегистрации|EventLogEventPresentation|ВыгрузитьЖурналРегистрации|UnloadEventLog|ПолучитьЗначенияОтбораЖурналаРегистрации|GetEventLogFilterValues|УстановитьИспользованиеСобытияЖурналаРегистрации|SetEventLogEventUse|ПолучитьИспользованиеСобытияЖурналаРегистрации|GetEventLogEventUse|СкопироватьЖурналРегистрации|CopyEventLog|ОчиститьЖурналРегистрации|ClearEventLog - | ЗначениеВДанныеФормы|ValueToFormData|ДанныеФормыВЗначение|FormDataToValue|КопироватьДанныеФормы|CopyFormData|УстановитьСоответствиеОбъектаИФормы|SetObjectAndFormConformity|ПолучитьСоответствиеОбъектаИФормы|GetObjectAndFormConformity - | ПолучитьФункциональнуюОпцию|GetFunctionalOption|ПолучитьФункциональнуюОпциюИнтерфейса|GetInterfaceFunctionalOption|УстановитьПараметрыФункциональныхОпцийИнтерфейса|SetInterfaceFunctionalOptionParameters|ПолучитьПараметрыФункциональныхОпцийИнтерфейса|GetInterfaceFunctionalOptionParameters|ОбновитьИнтерфейс|RefreshInterface - | УстановитьРасширениеРаботыСКриптографией|InstallCryptoExtension|НачатьУстановкуРасширенияРаботыСКриптографией|BeginInstallCryptoExtension|ПодключитьРасширениеРаботыСКриптографией|AttachCryptoExtension|НачатьПодключениеРасширенияРаботыСКриптографией|BeginAttachingCryptoExtension - | УстановитьСоставСтандартногоИнтерфейсаOData|SetStandardODataInterfaceContent|ПолучитьСоставСтандартногоИнтерфейсаOData|GetStandardODataInterfaceContent - | Мин|Min|Макс|Max|ОписаниеОшибки|ErrorDescription|Вычислить|Eval|ИнформацияОбОшибке|ErrorInfo|Base64Значение|Base64Value|Base64Строка|Base64String|ЗаполнитьЗначенияСвойств|FillPropertyValues|ЗначениеЗаполнено|ValueIsFilled|ПолучитьПредставленияНавигационныхСсылок|GetURLsPresentations|НайтиОкноПоНавигационнойСсылке|FindWindowByURL|ПолучитьОкна|GetWindows|ПерейтиПоНавигационнойСсылке|GotoURL|ПолучитьНавигационнуюСсылку|GetURL|ПолучитьДопустимыеКодыЛокализации|GetAvailableLocaleCodes|ПолучитьНавигационнуюСсылкуИнформационнойБазы|GetInfoBaseURL|ПредставлениеКодаЛокализации|LocaleCodePresentation|ПолучитьДопустимыеЧасовыеПояса|GetAvailableTimeZones|ПредставлениеЧасовогоПояса|TimeZonePresentation|ТекущаяУниверсальнаяДата|CurrentUniversalDate|ТекущаяУниверсальнаяДатаВМиллисекундах|CurrentUniversalDateInMilliseconds|МестноеВремя|ToLocalTime|УниверсальноеВремя|ToUniversalTime|ЧасовойПояс|TimeZone|СмещениеЛетнегоВремени|DaylightTimeOffset|СмещениеСтандартногоВремени|StandardTimeOffset|КодироватьСтроку|EncodeString|РаскодироватьСтроку|DecodeString|Найти|Find - | ПередНачаломРаботыСистемы|BeforeStart|ПриНачалеРаботыСистемы|OnStart|ПередЗавершениемРаботыСистемы|BeforeExit|ПриЗавершенииРаботыСистемы|OnExit|ОбработкаВнешнегоСобытия|ExternEventProcessing|УстановкаПараметровСеанса|SessionParametersSetting|ПриИзмененииПараметровЭкрана|OnChangeDisplaySettings - | WSСсылки|WSReferences|БиблиотекаКартинок|PictureLib|БиблиотекаМакетовОформленияКомпоновкиДанных|DataCompositionAppearanceTemplateLib|БиблиотекаСтилей|StyleLib|БизнесПроцессы|BusinessProcesses|ВнешниеИсточникиДанных|ExternalDataSources|ВнешниеОбработки|ExternalDataProcessors|ВнешниеОтчеты|ExternalReports|Документы|Documents|ДоставляемыеУведомления|DeliverableNotifications|ЖурналыДокументов|DocumentJournals|Задачи|Tasks|ИспользованиеРабочейДаты|WorkingDateUse|ИсторияРаботыПользователя|UserWorkHistory|Константы|Constants|КритерииОтбора|FilterCriteria|Метаданные|Metadata|Обработки|DataProcessors|ОтправкаДоставляемыхУведомлений|DeliverableNotificationSend|Отчеты|Reports|ПараметрыСеанса|SessionParameters|Перечисления|Enums|ПланыВидовРасчета|ChartsOfCalculationTypes|ПланыВидовХарактеристик|ChartsOfCharacteristicTypes|ПланыОбмена|ExchangePlans|ПланыСчетов|ChartsOfAccounts|ПолнотекстовыйПоиск|FullTextSearch|ПользователиИнформационнойБазы|InfoBaseUsers|Последовательности|Sequences|РасширенияКонфигурации|ConfigurationExtensions|РегистрыБухгалтерии|AccountingRegisters|РегистрыНакопления|AccumulationRegisters|РегистрыРасчета|CalculationRegisters|РегистрыСведений|InformationRegisters|РегламентныеЗадания|ScheduledJobs|СериализаторXDTO|XDTOSerializer|Справочники|Catalogs|СредстваГеопозиционирования|LocationTools|СредстваКриптографии|CryptoToolsManager|СредстваМультимедиа|MultimediaTools|СредстваПочты|MailTools|СредстваТелефонии|TelephonyTools|ФабрикаXDTO|XDTOFactory|ФоновыеЗадания|BackgroundJobs|ХранилищаНастроек - | ГлавныйИнтерфейс|MainInterface|ГлавныйСтиль|MainStyle|ПараметрЗапуска|LaunchParameter|РабочаяДата|WorkingDate|SettingsStorages|ХранилищеВариантовОтчетов|ReportsVariantsStorage|ХранилищеНастроекДанныхФорм|FormDataSettingsStorage|ХранилищеОбщихНастроек|CommonSettingsStorage|ХранилищеПользовательскихНастроекДинамическихСписков|DynamicListsUserSettingsStorage|ХранилищеПользовательскихНастроекОтчетов|ReportsUserSettingsStorage|ХранилищеСистемныхНастроек|SystemSettingsStorage - | Если|If|ИначеЕсли|ElsIf|Иначе|Else|КонецЕсли|EndIf|Тогда|Then - | Неопределено|Undefined|Истина|True|Ложь|False|NULL - )\s*(?=\()/ix + lazy { require_relative 'bsl/keywords' } state :root do rule %r/\n/, Text @@ -67,8 +19,14 @@ class Bsl < RegexLexer rule %r/(?<=[^\wа-яё]|^)\&.*$/, Keyword::Declaration rule %r/[-+\/*%=<>.?&]/, Operator rule %r/(?<=[^\wа-яё]|^)\#.*$/, Keyword::Declaration - rule KEYWORDS, Keyword - rule BUILTINS, Name::Builtin + keywords %r/\p{Word}+/ do + rule KEYWORDS, Keyword + end + + keywords %r/\p{Word}+(?=\s*[(])/ do + rule BUILTINS, Name::Builtin + end + rule %r/[\wа-яё][\wа-яё]*/i, Name::Variable #literals diff --git a/lib/rouge/lexers/bsl/keywords.rb b/lib/rouge/lexers/bsl/keywords.rb new file mode 100644 index 0000000000..b5b78f1055 --- /dev/null +++ b/lib/rouge/lexers/bsl/keywords.rb @@ -0,0 +1,280 @@ +module Rouge + module Lexers + class Bsl + KEYWORDS = Set.new %w( + КонецПроцедуры EndProcedure КонецФункции EndFunction + Прервать Break Продолжить Continue + Возврат Return Если If + Иначе Else ИначеЕсли ElsIf + Тогда Then КонецЕсли EndIf + Попытка Try Исключение Except + КонецПопытки EndTry Raise ВызватьИсключение + Пока While Для For + Каждого Each Из In + По To Цикл Do + КонецЦикла EndDo НЕ NOT + И AND ИЛИ OR + Новый New Процедура Procedure + Функция Function Перем Var + Экспорт Export Знач Val + ) + + BUILTINS = Set.new %w( + Лев Цел Окр Год Час Тип Мин Прав Сред ВРег НРег ТРег День Дата НСтр + Макс Если Ложь СокрЛ СокрП Месяц Число Найти Иначе Тогда СокрЛП + Символ Минута ТипЗнч Булево Строка Вопрос Сигнал Формат Задачи + Отчеты Истина Секунда СтрДлина СтрНайти КонецДня ДеньГода Сообщить + НачалоДня КонецГода КонецЧаса Состояние СтрШаблон Вычислить Документы + Константы Обработки ИначеЕсли КонецЕсли КодСимвола НачалоГода + НачалоЧаса НеделяГода ДеньНедели Оповестить ВвестиДату НайтиФайлы + Метаданные СтрЗаменить СтрСравнить КонецМесяца КонецМинуты КонецНедели + ТекущаяДата ВвестиЧисло ТекущийЯзык ЧасовойПояс ПланыОбмена ПланыСчетов + Справочники РабочаяДата ПустаяСтрока СтрРазделить СтрСоединить + НачалоМесяца НачалоМинуты НачалоНедели ОткрытьФорму АктивноеОкно + ВвестиСтроку ПравоДоступа РольДоступна УдалитьФайлы ПолучитьФайл + ПолучитьОкна МестноеВремя Перечисления ГлавныйСтиль Неопределено + СтрЧислоСтрок КонецКвартала ДобавитьМесяц ПолучитьФорму ЧислоПрописью + ИмяКомпьютера ЗначениеВФайл РазделитьФайл ПоместитьФайл ПолучитьФайлы + ВнешниеОтчеты СредстваПочты СтрНачинаетсяС НачалоКвартала ПоказатьВопрос + Предупреждение ОткрытьСправку ЗакрытьСправку ВвестиЗначение + КомандаСистемы ПользователиОС КопироватьФайл СоздатьКаталог + ПоместитьФайлы НайтиПоСсылкам УдалитьОбъекты ОписаниеОшибки + БизнесПроцессы КритерииОтбора ФоновыеЗадания ОткрытьЗначение + ИмяПользователя ЗначениеИзФайла ПереместитьФайл ОбъединитьФайлы + БезопасныйРежим ПараметрыСеанса РегистрыРасчета ПараметрЗапуска + ПоказатьЗначение ПоказатьВводДаты КаталогПрограммы ПараметрыДоступа + ЗапуститьСистему НачатьТранзакцию МонопольныйРежим КодироватьСтроку + БиблиотекаСтилей ВнешниеОбработки РегистрыСведений ГлавныйИнтерфейс + СтрПолучитьСтроку СтрЧислоВхождений ОчиститьСообщения ПоказатьВводЧисла + ЧасовойПоясСеанса ТекущаяДатаСеанса КаталогДокументов НачатьПоискФайлов + ТранзакцияАктивна ОбновитьИнтерфейс ЗначениеЗаполнено ЖурналыДокументов + ПланыВидовРасчета СредстваТелефонии ХранилищаНастроек СтрЗаканчиваетсяНа + ПоказатьВводСтроки ПолучитьОбщийМакет ПолучитьОбщуюФорму + ТекущийЯзыкСистемы ПредставлениеПрава ОтменитьТранзакцию + ИнформацияОбОшибке УниверсальноеВремя БиблиотекаКартинок + Последовательности РегистрыНакопления ТекущийРежимЗапуска + ЗапуститьПриложение РаскодироватьСтроку ПолнотекстовыйПоиск + РегистрыБухгалтерии РегламентныеЗадания СредстваМультимедиа + ОткрытьИндексСправки ОткрытьФормуМодально ПоказатьВводЗначения + ПредставлениеПериода ЗначениеВСтрокуВнутр НачатьПомещениеФайла + НачатьУдалениеФайлов КонфигурацияИзменена ПолучитьДанныеВыбора + ЗначениеВДанныеФормы ДанныеФормыВЗначение СредстваКриптографии + ОповеститьОбИзменении ПолноеИмяПользователя ТекущийКодЛокализации + ЗначениеИзСтрокиВнутр ПолучитьМаскуВсеФайлы НачатьПолучениеФайлов + НачатьПомещениеФайлов КопироватьДанныеФормы ПоказатьПредупреждение + ЗавершитьРаботуСистемы КаталогВременныхФайлов НачатьЗапускПриложения + НачатьСозданиеКаталога НачатьКопированиеФайла НачатьПеремещениеФайла + ПривилегированныйРежим СмещениеЛетнегоВремени ПриНачалеРаботыСистемы + ВнешниеИсточникиДанных РасширенияКонфигурации ХранилищеОбщихНастроек + ПрекратитьРаботуСистемы ПредставлениеПриложения ПолучитьРазделительПути + ЗафиксироватьТранзакцию ДоставляемыеУведомления ПланыВидовХарактеристик + ОткрытьСодержаниеСправки ПредопределенноеЗначение ПолучитьЗаголовокСистемы + ЗаписьЖурналаРегистрации ЗаполнитьЗначенияСвойств ТекущаяУниверсальнаяДата + ОбработкаВнешнегоСобытия ИспользованиеРабочейДаты ОбновитьНумерациюОбъектов + ПолучитьБлокировкуСеансов УстановитьБезопасныйРежим + НайтиПомеченныеНаУдаление ОчиститьЖурналРегистрации + ПередНачаломРаботыСистемы УстановкаПараметровСеанса + ИсторияРаботыПользователя ХранилищеВариантовОтчетов + ПоказатьИнформациюОбОшибке КраткоеПредставлениеОшибки + УстановитьЗаголовокСистемы ПолучитьИмяВременногоФайла + УстановитьМонопольныйРежим ВыгрузитьЖурналРегистрации + ПредставлениеЧасовогоПояса ПриЗавершенииРаботыСистемы + ХранилищеСистемныхНастроек ОтключитьОбработчикОжидания + УстановитьЧасовойПоясСеанса ПодключитьВнешнююКомпоненту + УстановитьВнешнююКомпоненту ЭтоАдресВременногоХранилища + УстановитьБлокировкуСеансов ПолучитьФункциональнуюОпцию + ПолучитьНавигационнуюСсылку СмещениеСтандартногоВремени + СредстваГеопозиционирования ХранилищеНастроекДанныхФорм + ПодробноеПредставлениеОшибки ВыполнитьОбработкуОповещения + ПодключитьОбработчикОжидания ВыполнитьПроверкуПравДоступа + УдалитьИзВременногоХранилища ПолучитьМаскуВсеФайлыКлиента + ПолучитьМаскуВсеФайлыСервера СкопироватьЖурналРегистрации + ПерейтиПоНавигационнойСсылке ПредставлениеКодаЛокализации + ПриИзмененииПараметровЭкрана ОтключитьОбработчикОповещения + ПолучитьСообщенияПользователю ПолучитьИзВременногоХранилища + ПоместитьВоВременноеХранилище НомерСеансаИнформационнойБазы + ПередЗавершениемРаботыСистемы ПоказатьОповещениеПользователя + ПодключитьОбработчикОповещения ПолучитьРазделительПутиКлиента + ПолучитьРазделительПутиСервера НайтиОкноПоНавигационнойСсылке + ПолучитьДопустимыеЧасовыеПояса ПользователиИнформационнойБазы + ОбработкаПрерыванияПользователя ЗаблокироватьРаботуПользователя + ЗапроситьРазрешениеПользователя УдалитьДанныеИнформационнойБазы + БезопасныйРежимРазделенияДанных ОтправкаДоставляемыхУведомлений + РабочийКаталогДанныхПользователя ПолучитьИнформациюЭкрановКлиента + НачатьУстановкуВнешнейКомпоненты КодЛокализацииИнформационнойБазы + УстановитьПривилегированныйРежим ПолучитьСеансыИнформационнойБазы + ПолучитьИмяКлиентаЛицензирования НачатьПолучениеКаталогаДокументов + ПолучитьОперативнуюОтметкуВремени НомерСоединенияИнформационнойБазы + НеобходимостьЗавершенияСоединения ПолучитьИдентификаторКонфигурации + ПолучитьСоответствиеОбъектаИФормы ПолучитьДопустимыеКодыЛокализации + СтрокаСоединенияИнформационнойБазы ПолучитьКраткийЗаголовокПриложения + НачатьПодключениеВнешнейКомпоненты ПодключитьРасширениеРаботыСФайлами + УстановитьРасширениеРаботыСФайлами НачатьЗапросРазрешенияПользователя + ПолучитьСтруктуруХраненияБазыДанных УстановитьСоответствиеОбъектаИФормы + УстановитьКраткийЗаголовокПриложения ПолучитьСоединенияИнформационнойБазы + ЗаблокироватьДанныеДляРедактирования ОбновитьПовторноИспользуемыеЗначения + ПолучитьВремяЗавершенияСпящегоСеанса ПолучитьСкоростьКлиентскогоСоединения + ПолучитьВремяОжиданияБлокировкиДанных РазблокироватьДанныеДляРедактирования + ПолучитьЧасовойПоясИнформационнойБазы ПолучитьФункциональнуюОпциюИнтерфейса + ПолучитьЗаголовокКлиентскогоПриложения + НачатьПолучениеКаталогаВременныхФайлов + ИнициализироватьПредопределенныеДанные + ПолучитьВремяЗасыпанияПассивногоСеанса + УстановитьВремяЗавершенияСпящегоСеанса + ПолучитьТекущийСеансИнформационнойБазы + ПредставлениеСобытияЖурналаРегистрации + ТекущаяУниверсальнаяДатаВМиллисекундах + НачатьУстановкуРасширенияРаботыСФайлами + УстановитьВремяОжиданияБлокировкиДанных + УстановитьЧасовойПоясИнформационнойБазы + ПолучитьИспользованиеЖурналаРегистрации + УстановитьЗаголовокКлиентскогоПриложения + ПолучитьОбновлениеКонфигурацииБазыДанных + УстановитьВремяЗасыпанияПассивногоСеанса + УстановитьНастройкиКлиентаЛицензирования + ПолучитьЗначенияОтбораЖурналаРегистрации + УстановитьРасширениеРаботыСКриптографией + ПодключитьРасширениеРаботыСКриптографией + ПолучитьПредставленияНавигационныхСсылок + ХранилищеПользовательскихНастроекОтчетов + НачатьПодключениеРасширенияРаботыСФайлами + КонфигурацияБазыДанныхИзмененаДинамически + УстановитьБезопасныйРежимРазделенияДанных + УстановитьИспользованиеЖурналаРегистрации + ПолучитьПолноеИмяПредопределенногоЗначения + РазорватьСоединениеСВнешнимИсточникомДанных + БиблиотекаМакетовОформленияКомпоновкиДанных + ПолучитьМинимальнуюДлинуПаролейПользователей + УстановитьСоединениеСВнешнимИсточникомДанных + ТекущийВариантИнтерфейсаКлиентскогоПриложения + ПолучитьПроверкуСложностиПаролейПользователей + НачатьУстановкуРасширенияРаботыСКриптографией + ПолучитьНавигационнуюСсылкуИнформационнойБазы + УстановитьМинимальнуюДлинуПаролейПользователей + ПолучитьИспользованиеСобытияЖурналаРегистрации + ПолучитьПараметрыФункциональныхОпцийИнтерфейса + УстановитьПроверкуСложностиПаролейПользователей + НачатьПодключениеРасширенияРаботыСКриптографией + УстановитьИспользованиеСобытияЖурналаРегистрации + УстановитьПараметрыФункциональныхОпцийИнтерфейса + НачатьПолучениеРабочегоКаталогаДанныхПользователя + ТекущийВариантОсновногоШрифтаКлиентскогоПриложения + ПолучитьДополнительныйПараметрКлиентаЛицензирования + ХранилищеПользовательскихНастроекДинамическихСписков + ОтключитьОбработчикЗапросаНастроекКлиентаЛицензирования + ПодключитьОбработчикЗапросаНастроекКлиентаЛицензирования + ПолучитьОбновлениеПредопределенныхДанныхИнформационнойБазы + УстановитьОбновлениеПредопределенныхДанныхИнформационнойБазы + AccessParameters AccessRight AccountingRegisters AccumulationRegisters + ACos ACos ActiveWindow AddMonth ApplicationPresentation ASin ASin + ATan ATan AttachAddIn AttachCryptoExtension AttachFileSystemExtension + AttachIdleHandler AttachLicensingClientParametersRequestHandler + AttachNotificationHandler BackgroundJobs Base64Строка Base64Значение + Base64String Base64Value Beep BeforeExit BeforeStart BeginAttachingAddIn + BeginAttachingCryptoExtension BeginAttachingFileSystemExtension + BeginCopyingFile BeginCreatingDirectory BeginDeletingFiles + BeginFindingFiles BeginGettingDocumentsDir BeginGettingFiles + BeginGettingTempFilesDir BeginGettingUserDataWorkDir BeginInstallAddIn + BeginInstallCryptoExtension BeginInstallFileSystemExtension + BeginMovingFile BeginPutFile BeginPuttingFiles + BeginRequestingUserPermission BeginRunningApplication BeginTransaction + BegOfDay BegOfHour BegOfMinute BegOfMonth BegOfQuarter BegOfWeek + BegOfYear BinDir Boolean BriefErrorDescription BusinessProcesses + CalculationRegisters CanReadXML Catalogs Char CharCode ChartsOfAccounts + ChartsOfCalculationTypes ChartsOfCharacteristicTypes ClearEventLog + ClearMessages ClientApplicationBaseFontCurrentVariant + ClientApplicationInterfaceCurrentVariant CloseHelp ПолучитьCOMОбъект + CommitTransaction CommonSettingsStorage ComputerName ConfigurationChanged + ConfigurationExtensions ConnectExternalDataSource ConnectionStopRequest + Constants CopyEventLog CopyFormData Cos Cos CreateDirectory + CreateXDTOFactory CryptoToolsManager CurrentDate CurrentLanguage + CurrentLocaleCode CurrentRunMode CurrentSessionDate CurrentSystemLanguage + CurrentUniversalDate CurrentUniversalDateInMilliseconds + DataBaseConfigurationChangedDynamically + DataCompositionAppearanceTemplateLib DataProcessors DataSeparationSafeMode + Date Day DaylightTimeOffset DayOfYear DecodeString DeleteFiles + DeleteFromTempStorage DeleteObjects DeliverableNotifications + DeliverableNotificationSend DetachIdleHandler + DetachLicensingClientParametersRequestHandler DetachNotificationHandler + DetailErrorDescription DisconnectExternalDataSource DocumentJournals + Documents DocumentsDir DoMessageBox DoQueryBox + DynamicListsUserSettingsStorage Else ElsIf EncodeString EndIf + EndOfDay EndOfHour EndOfMinute EndOfMonth EndOfQuarter EndOfWeek + EndOfYear Enums EraseInfoBaseData ErrorDescription ErrorInfo Eval + EventLogEventPresentation ExchangePlans ExclusiveMode + ExecuteNotifyProcessing Exit Exp Exp ExternalDataProcessors + ExternalDataSources ExternalReports ExternEventProcessing False + FileCopy FillPropertyValues FilterCriteria Find FindByRef + FindDisallowedXMLCharacters FindFiles FindMarkedForDeletion + FindWindowByURL Format FormDataSettingsStorage FormDataToValue + FromXMLType FullTextSearch GetAllFilesMask GetAvailableLocaleCodes + GetAvailableTimeZones GetCaption GetChoiceData GetClientAllFilesMask + GetClientApplicationCaption GetClientConnectionSpeed + GetClientDisplaysInformation GetClientPathSeparator GetCommonForm + GetCommonTemplate GetCOMObject GetConfigurationID GetCurrentInfoBaseSession + GetDataBaseConfigurationUpdate GetDBStorageStructureInfo + GetEventLogEventUse GetEventLogFilterValues GetEventLogUsing GetFile + GetFiles GetForm GetFromTempStorage GetFunctionalOption + GetHibernateSessionTerminateTime GetInfoBaseConnections + GetInfoBasePredefinedData GetInfoBaseSessions GetInfoBaseTimeZone + GetInfoBaseURL GetInterfaceFunctionalOption + GetInterfaceFunctionalOptionParameters + GetLicensingClientAdditionalParameter GetLicensingClientName + GetLockWaitTime GetObjectAndFormConformity GetPassiveSessionHibernateTime + GetPathSeparator GetPredefinedValueFullName GetRealTimeTimestamp + GetServerAllFilesMask GetServerPathSeparator GetSessionsLock + GetShortApplicationCaption GetStandardODataInterfaceContent + GetTempFileName GetURL GetURLsPresentations GetUserMessages + GetUserPasswordMinLength GetUserPasswordStrengthCheck GetWindows + GetXMLType GotoURL Hour If ImportXDTOModel InfoBaseConnectionNumber + InfoBaseConnectionString InfoBaseLocaleCode InfoBaseSessionNumber + InfoBaseUsers InformationRegisters InitializePredefinedData InputDate + InputNumber InputString InputValue InstallAddIn InstallCryptoExtension + InstallFileSystemExtension Int IsBlankString IsInRole IsTempStorageURL + ЗаписатьJSON ПрочитатьJSON ЗаписатьДатуJSON ПрочитатьДатуJSON + LaunchParameter Left LocaleCodePresentation LocationTools LockApplication + LockDataForEdit Log Log Log10 Log10 Lower MailTools MainInterface + MainStyle Max MergeFiles Message Metadata Mid Min Minute Month + MoveFile MultimediaTools Notify NotifyChanged NStr NULL Number + NumberInWords ПолучитьСоставСтандартногоИнтерфейсаOData + УстановитьСоставСтандартногоИнтерфейсаOData OnChangeDisplaySettings + OnExit OnStart OpenForm OpenFormModal OpenHelp OpenHelpContent + OpenHelpIndex OpenValue OSUsers PeriodPresentation PictureLib Pow + Pow PredefinedValue PrivilegedMode PutFile PutFiles PutToTempStorage + ReadJSON ReadJSONDate ReadXML RefreshInterface RefreshObjectsNumbering + RefreshReusableValues Reports ReportsUserSettingsStorage + ReportsVariantsStorage RequestUserPermission Right RightPresentation + RollbackTransaction Round RunApp RunSystem SafeMode ScheduledJobs + Second Sequences SessionParameters SessionParametersSetting + SessionTimeZone SetCaption SetClientApplicationCaption + SetDataSeparationSafeMode SetEventLogEventUse SetEventLogUsing + SetExclusiveMode SetHibernateSessionTerminateTime + SetInfoBasePredefinedDataUpdate SetInfoBaseTimeZone + SetInterfaceFunctionalOptionParameters SetLicensingClientParameters + SetLockWaitTime SetObjectAndFormConformity SetPassiveSessionHibernateTime + SetPrivilegedMode SetSafeMode SetSessionsLock SetSessionTimeZone + SetShortApplicationCaption SetStandardODataInterfaceContent + SettingsStorages SetUserPasswordMinLength SetUserPasswordStrengthCheck + ShowErrorInfo ShowInputDate ShowInputNumber ShowInputString + ShowInputValue ShowMessageBox ShowQueryBox ShowUserNotification + ShowValue Sin Sin SplitFile Sqrt Sqrt StandardTimeOffset Status + StrCompare StrConcat StrEndsWith StrFind StrGetLine String StrLen + StrLineCount StrOccurrenceCount StrReplace StrSplit StrStartWith + StrTemplate StyleLib System SystemSettingsStorage Tan Tan Tasks + TelephonyTools TempFilesDir Terminate Then TimeZone TimeZonePresentation + Title ToLocalTime ToUniversalTime TransactionActive TrimAll TrimL + TrimR True Type TypeOf Undefined UnloadEventLog UnlockDataForEdit + Upper UserDataWorkDir UserFullName UserInterruptProcessing UserName + UserWorkHistory ValueFromFile ValueFromStringInternal ValueIsFilled + ValueToFile ValueToFormData ValueToStringInternal VerifyAccessRights + WeekDay WeekOfYear WorkingDate WorkingDateUse WriteJSON WriteJSONDate + WriteLogEvent WriteXML WSСсылки WSReferences ФабрикаXDTO ИмпортМоделиXDTO + СериализаторXDTO СоздатьФабрикуXDTO XDTOFactory XDTOSerializer + XMLТип XMLСтрока XMLТипЗнч ИзXMLТипа XMLЗначение ЗаписатьXML + ПрочитатьXML ПолучитьXMLТип ВозможностьЧтенияXML + НайтиНедопустимыеСимволыXML XMLString XMLType XMLTypeOf XMLValue + Year + ) + end + end +end diff --git a/spec/visual/samples/bsl b/spec/visual/samples/bsl index 5597e2c08c..ce43c0b229 100644 --- a/spec/visual/samples/bsl +++ b/spec/visual/samples/bsl @@ -83,3 +83,6 @@ #КонецЕсли #КонецОбласти + +// [jneen] test full capture +Upper (thing) From 870f8d1c1df30b8e078f044392451eb1a0e4e152 Mon Sep 17 00:00:00 2001 From: jneen Date: Wed, 8 Apr 2026 12:48:32 -0400 Subject: [PATCH 021/154] use the keywords api for c --- lib/rouge/lexers/c.rb | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/lib/rouge/lexers/c.rb b/lib/rouge/lexers/c.rb index c3a1eea40b..ad03d454c8 100644 --- a/lib/rouge/lexers/c.rb +++ b/lib/rouge/lexers/c.rb @@ -60,8 +60,9 @@ def self.reserved ) end + # for subclasses def self.builtins - @builtins ||= [] + @builtins ||= Set.new end start { push :bol } @@ -113,20 +114,13 @@ def self.builtins rule %r/[()\[\],.;]/, Punctuation rule %r/\bcase\b/, Keyword, :case rule %r/(?:true|false|NULL|nullptr)\b/, Name::Builtin - rule id do |m| - name = m[0] - - if self.class.keywords.include? name - token Keyword - elsif self.class.keywords_type.include? name - token Keyword::Type - elsif self.class.reserved.include? name - token Keyword::Reserved - elsif self.class.builtins.include? name - token Name::Builtin - else - token Name - end + + keywords id do + rule :keywords, Keyword + rule :keywords_type, Keyword::Type + rule :reserved, Keyword::Reserved + rule :builtins, Name::Builtin + default Name end end From a7e8508b33a3074f24f421cbe080ac51416d373f Mon Sep 17 00:00:00 2001 From: jneen Date: Wed, 8 Apr 2026 13:04:32 -0400 Subject: [PATCH 022/154] use the keywords api in cfscript --- .rubocop_todo.yml | 1 - lib/rouge/lexers/cfscript.rb | 66 ++++++++++++++++++++---------------- 2 files changed, 36 insertions(+), 31 deletions(-) diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index 4acc906e0b..93a5fc1015 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -50,7 +50,6 @@ Naming/VariableName: Rouge/NoBuildingAlternationPatternInRegexp: Exclude: - 'lib/rouge/lexers/armasm.rb' - - 'lib/rouge/lexers/cfscript.rb' - 'lib/rouge/lexers/cmhg.rb' - 'lib/rouge/lexers/crystal.rb' - 'lib/rouge/lexers/csharp.rb' diff --git a/lib/rouge/lexers/cfscript.rb b/lib/rouge/lexers/cfscript.rb index 6b083ce7be..6d2445c514 100644 --- a/lib/rouge/lexers/cfscript.rb +++ b/lib/rouge/lexers/cfscript.rb @@ -12,32 +12,44 @@ class Cfscript < RegexLexer filenames '*.cfc' def self.keywords - @keywords ||= %w( + @keywords ||= Set.new %w( if else var xml default break switch do try catch throw in continue for return while required ) end def self.declarations - @declarations ||= %w( + @declarations ||= Set.new %w( component property function remote public package private ) end def self.types - @types ||= %w( + @types ||= Set.new %w( any array binary boolean component date guid numeric query string struct uuid void xml ) end - constants = %w(application session client cookie super this variables arguments cgi) + CONSTANTS = Set.new %w( + application session client cookie super this variables arguments cgi + ) - operators = %w(\+\+ -- && \|\| <= >= < > == != mod eq lt gt lte gte not is and or xor eqv imp equal contains \? ) dotted_id = /[$a-zA-Z_][a-zA-Z0-9_.]*/ state :root do mixin :comments_and_whitespace - rule %r/(?:#{operators.join('|')}|does not contain|greater than(?: or equal to)?|less than(?: or equal to)?)\b/i, Operator, :expr_start + + rule %r( + [+][+] | -- | [|][|] | <= | >= | == | != + | [<>?] + | (?: + mod | eq | gte? | lte? | not | is | and | x?or | eqv | imp | equal + | contains + | does\s+not\s+contain + | (?:greater|less)\s+than(?:\s+or\s+equal\s+to)? + )\b + )x, Operator, :expr_start + rule %r([-<>+*%&|\^/!=]=?), Operator, :expr_start rule %r/[(\[,]/, Punctuation, :expr_start @@ -52,7 +64,10 @@ def self.types rule %r/[{}]/, Punctuation, :statement - rule %r/(?:#{constants.join('|')})\b/, Name::Constant + keywords %r/\w+/ do + rule CONSTANTS, Name::Constant + end + rule %r/(?:true|false|null)\b/, Keyword::Constant rule %r/import\b/, Keyword::Namespace, :import rule %r/(#{dotted_id})(\s*)(:)(\s*)/ do @@ -60,39 +75,30 @@ def self.types push :expr_start end - rule %r/([A-Za-z_$][\w.]*)(\s*)(\()/ do |m| - if self.class.keywords.include? m[1] - token Keyword, m[1] - token Text, m[2] - token Punctuation, m[3] - else - token Name::Function, m[1] - token Text, m[2] - token Punctuation, m[3] + keywords %r/([a-z_$][\w.]*)(\s*)(\()/i do |m| + group 1 + + rule :keywords do |m| + groups Keyword, Text, Punctuation end - end - rule dotted_id do |m| - if self.class.declarations.include? m[0] - token Keyword::Declaration - push :expr_start - elsif self.class.keywords.include? m[0] - token Keyword - push :expr_start - elsif self.class.types.include? m[0] - token Keyword::Type - push :expr_start - else - token Name::Other + default do |m| + groups Name::Function, Text, Punctuation end end + keywords dotted_id do + rule :declarations, Keyword::Declaration, :expr_start + rule :keywords, Keyword, :expr_start + rule :types, Keyword::Type, :expr_start + default Name::Other + end + rule %r/[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?/, Num::Float rule %r/0x[0-9a-fA-F]+/, Num::Hex rule %r/[0-9]+/, Num::Integer rule %r/"(\\\\|\\"|[^"])*"/, Str::Double rule %r/'(\\\\|\\'|[^'])*'/, Str::Single - end # same as java, broken out From 98e8db5e5b4437e7295a6f500c617e90fa17a23c Mon Sep 17 00:00:00 2001 From: jneen Date: Wed, 8 Apr 2026 13:07:27 -0400 Subject: [PATCH 023/154] use the keywords api for Clean --- lib/rouge/lexers/clean.rb | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/lib/rouge/lexers/clean.rb b/lib/rouge/lexers/clean.rb index 031cd61776..52f1c60743 100644 --- a/lib/rouge/lexers/clean.rb +++ b/lib/rouge/lexers/clean.rb @@ -79,20 +79,14 @@ class instance rule %r/code(\s+inline)?\s*{/, Comment::Preproc, :abc - rule %r/_*[a-z][\w`]*/ do |m| - if self.class.keywords.include?(m[0]) - token Keyword - else - token Name - end + keywords %r/_*[a-z][\w`]*/ do + rule :keywords, Keyword + default Name end - rule %r/_*[A-Z][\w`]*/ do |m| - if m[0]=='True' || m[0]=='False' - token Keyword::Constant - else - token Keyword::Type - end + keywords %r/_*[A-Z][\w`]*/ do |m| + rule Set['True', 'False'], Keyword::Constant + default Keyword::Type end rule %r/[^\w\s`]/, Punctuation From fead18d25a5de1a7e4ae50b3957522023f42c9e3 Mon Sep 17 00:00:00 2001 From: jneen Date: Wed, 8 Apr 2026 13:14:12 -0400 Subject: [PATCH 024/154] use the keywords api for cmake --- lib/rouge/lexers/cmake.rb | 131 +++-------------------------- lib/rouge/lexers/cmake/builtins.rb | 38 +++++++++ 2 files changed, 50 insertions(+), 119 deletions(-) create mode 100644 lib/rouge/lexers/cmake/builtins.rb diff --git a/lib/rouge/lexers/cmake.rb b/lib/rouge/lexers/cmake.rb index a22169cde3..3944698012 100644 --- a/lib/rouge/lexers/cmake.rb +++ b/lib/rouge/lexers/cmake.rb @@ -10,8 +10,8 @@ class CMake < RegexLexer filenames 'CMakeLists.txt', '*.cmake' mimetypes 'text/x-cmake' - SPACE = '[ \t]' - BRACKET_OPEN = '\[=*\[' + SPACE = /[ \t]/ + BRACKET_OPEN = /\[=*\[/ STATES_MAP = { :root => Text, @@ -21,118 +21,7 @@ class CMake < RegexLexer :variable_reference => Name::Variable, } - BUILTIN_COMMANDS = Set.new %w[ - add_compile_definitions - add_compile_options - add_custom_command - add_custom_target - add_definitions - add_dependencies - add_executable - add_library - add_link_options - add_subdirectory - add_test - aux_source_directory - break - build_command - build_name - cmake_host_system_information - cmake_language - cmake_minimum_required - cmake_parse_arguments - cmake_policy - configure_file - create_test_sourcelist - define_property - else - elseif - enable_language - enable_testing - endforeach - endfunction - endif - endmacro - endwhile - exec_program - execute_process - export - export_library_dependencies - file - find_file - find_library - find_package - find_path - find_program - fltk_wrap_ui - foreach - function - get_cmake_property - get_directory_property - get_filename_component - get_property - get_source_file_property - get_target_property - get_test_property - if - include - include_directories - include_external_msproject - include_guard - include_regular_expression - install - install_files - install_programs - install_targets - link_directories - link_libraries - list - load_cache - load_command - macro - make_directory - mark_as_advanced - math - message - option - output_required_files - project - qt_wrap_cpp - qt_wrap_ui - remove - remove_definitions - return - separate_arguments - set - set_directory_properties - set_property - set_source_files_properties - set_target_properties - set_tests_properties - site_name - source_group - string - subdir_depends - subdirs - target_compile_definitions - target_compile_features - target_compile_options - target_include_directories - target_link_directories - target_link_libraries - target_link_options - target_precompile_headers - target_sources - try_compile - try_run - unset - use_mangled_mesa - utility_source - variable_requires - variable_watch - while - write_file - ] + lazy { require_relative 'cmake/builtins' } state :default do rule %r/\r\n?|\n/ do @@ -160,7 +49,7 @@ class CMake < RegexLexer state :root do mixin :variable_interpolation - rule %r/#{SPACE}/, Text + rule SPACE, Text rule %r/[()]/, Punctuation rule %r/##{BRACKET_OPEN}/ do |m| @@ -168,17 +57,21 @@ class CMake < RegexLexer @bracket_len = m[0].length - 1 # decount '#' goto :bracket_comment end - rule %r/#{BRACKET_OPEN}/ do |m| + + rule BRACKET_OPEN do |m| token Str::Double @bracket_len = m[0].length goto :bracket_string end - rule %r/\\"/, Text + rule %r/\\"/, Str::Escape rule %r/"/, Str::Double, :quoted_argument - rule %r/([A-Za-z_][A-Za-z0-9_]*)(#{SPACE}*)(\()/ do |m| - groups BUILTIN_COMMANDS.include?(m[1]) ? Name::Builtin : Name::Function, Text, Punctuation + keywords %r/([A-Za-z_][A-Za-z0-9_]*)(#{SPACE}*)(\()/ do + group 1 + + rule(BUILTIN_COMMANDS) { groups Name::Builtin, Text, Punctuation } + default { groups Name::Function, Text, Punctuation } end rule %r/#.*/, Comment::Single diff --git a/lib/rouge/lexers/cmake/builtins.rb b/lib/rouge/lexers/cmake/builtins.rb new file mode 100644 index 0000000000..b50f922525 --- /dev/null +++ b/lib/rouge/lexers/cmake/builtins.rb @@ -0,0 +1,38 @@ +module Rouge + module Lexers + class CMake + BUILTIN_COMMANDS = Set.new %w[ + add_compile_definitions add_compile_options add_custom_command + add_custom_target add_definitions add_dependencies + add_executable add_library add_link_options add_subdirectory + add_test aux_source_directory break build_command build_name + cmake_host_system_information cmake_language cmake_minimum_required + cmake_parse_arguments cmake_policy configure_file + create_test_sourcelist define_property else elseif + enable_language enable_testing endforeach endfunction endif + endmacro endwhile exec_program execute_process export + export_library_dependencies file find_file find_library + find_package find_path find_program fltk_wrap_ui foreach + function get_cmake_property get_directory_property + get_filename_component get_property get_source_file_property + get_target_property get_test_property if include + include_directories include_external_msproject include_guard + include_regular_expression install install_files install_programs + install_targets link_directories link_libraries list + load_cache load_command macro make_directory mark_as_advanced + math message option output_required_files project qt_wrap_cpp + qt_wrap_ui remove remove_definitions return separate_arguments + set set_directory_properties set_property + set_source_files_properties set_target_properties + set_tests_properties site_name source_group string + subdir_depends subdirs target_compile_definitions + target_compile_features target_compile_options + target_include_directories target_link_directories + target_link_libraries target_link_options target_precompile_headers + target_sources try_compile try_run unset use_mangled_mesa + utility_source variable_requires variable_watch while + write_file + ] + end + end +end From 2a6b5f6ee9580c25c52296a9438e68cdd0550112 Mon Sep 17 00:00:00 2001 From: jneen Date: Wed, 8 Apr 2026 13:55:12 -0400 Subject: [PATCH 025/154] eliminate preproc keyword list from cmhg --- .rubocop_todo.yml | 1 - lib/rouge/lexers/cmhg.rb | 18 ++++++++++-------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index 93a5fc1015..521f2a893a 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -50,7 +50,6 @@ Naming/VariableName: Rouge/NoBuildingAlternationPatternInRegexp: Exclude: - 'lib/rouge/lexers/armasm.rb' - - 'lib/rouge/lexers/cmhg.rb' - 'lib/rouge/lexers/crystal.rb' - 'lib/rouge/lexers/csharp.rb' - 'lib/rouge/lexers/d.rb' diff --git a/lib/rouge/lexers/cmhg.rb b/lib/rouge/lexers/cmhg.rb index 338a3faec9..c9c56c136a 100644 --- a/lib/rouge/lexers/cmhg.rb +++ b/lib/rouge/lexers/cmhg.rb @@ -9,16 +9,11 @@ class CMHG < RegexLexer tag 'cmhg' filenames '*.cmhg' - def self.preproc_keyword - @preproc_keyword ||= %w( - define elif else endif error if ifdef ifndef include line pragma undef warning - ) - end - state :root do rule %r/;[^\n]*/, Comment - rule %r/^([ \t]*)(#[ \t]*(?:(?:#{CMHG.preproc_keyword.join('|')})(?:[ \t].*)?)?)(?=\n)/ do - groups Text, Comment::Preproc + rule %r/^([ \t]*)(#[ \t]*)(\w*)/ do + groups Text, Comment::Preproc, Comment::Preproc + push :preproc end rule %r/[-a-z]+:/, Keyword::Declaration rule %r/[a-z_]\w+/i, Name::Entity @@ -29,6 +24,13 @@ def self.preproc_keyword rule %r/[ \t]+/, Text rule %r/\n+/, Text end + + state :preproc do + rule %r/[^\n\\]+/, Comment::Preproc + rule %r/\\\n/, Comment::Preproc + rule %r/\n/, Comment::Preproc, :pop! + rule %r/\\/, Comment::Preproc + end end end end From fce5467b3e6b409489dfc21147a8ae54447544cf Mon Sep 17 00:00:00 2001 From: jneen Date: Wed, 8 Apr 2026 14:05:19 -0400 Subject: [PATCH 026/154] use the keywords api for cobol --- .rubocop_todo.yml | 1 - lib/rouge/lexers/cobol.rb | 81 +++++----------------------- lib/rouge/lexers/cobol/keywords.rb | 85 ++++++++++++++++++++++++++++++ 3 files changed, 99 insertions(+), 68 deletions(-) create mode 100644 lib/rouge/lexers/cobol/keywords.rb diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index 521f2a893a..b45489bd50 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -85,7 +85,6 @@ Rouge/NoBuildingAlternationPatternInRegexp: # Offense count: 20 Rouge/NoHugeCollections: Exclude: - - 'lib/rouge/lexers/cobol.rb' - 'lib/rouge/lexers/common_lisp.rb' - 'lib/rouge/lexers/css.rb' - 'lib/rouge/lexers/datastudio.rb' diff --git a/lib/rouge/lexers/cobol.rb b/lib/rouge/lexers/cobol.rb index cceb046ca4..51c0b9371d 100644 --- a/lib/rouge/lexers/cobol.rb +++ b/lib/rouge/lexers/cobol.rb @@ -10,66 +10,17 @@ class COBOL < RegexLexer filenames '*.cob', '*.cbl', '*.cpy', '*.cpb' mimetypes 'text/x-cobol' + lazy { require_relative 'cobol/keywords' } + identifier = /\p{Alpha}[\p{Alnum}-]*/ - def self.divisions - @divisions ||= %w( - IDENTIFICATION ENVIRONMENT DATA PROCEDURE DIVISION - ) - end + DIVISIONS = Set.new %w( + IDENTIFICATION ENVIRONMENT DATA PROCEDURE DIVISION + ) - def self.sections - @sections ||= %w( - CONFIGURATION INPUT-OUTPUT FILE WORKING-STORAGE LOCAL-STORAGE LINKAGE SECTION - ) - end - - # List of COBOL keywords - # sourced from https://www.ibm.com/docs/en/cobol-zos/6.4?topic=appendixes-reserved-words - def self.keywords - @keywords ||= Set.new(%w( - ACCEPT ACCESS ACTIVE-CLASS ADD ADDRESS ADVANCING AFTER ALIGNED ALL ALLOCATE ALPHABET ALPHABETIC ALPHABETIC-LOWER - ALPHABETIC-UPPER ALPHANUMERIC ALPHANUMERIC-EDITED ALSO ALTER ALTERNATE AND ANYCASE ANY APPLY ARE AREA AREAS - ASCENDING ASSIGN AT AUTHOR B-AND B-NOT B-OR B-XOR BASED BASIS BEFORE BEGINNING BINARY BINARY-CHAR BINARY-DOUBLE - BINARY-LONG BINARY-SHORT BIT BLANK BLOCK BOOLEAN BOTTOM BY BYTE-LENGTH CALL CANCEL CBL CD CF CH CHARACTER - CHARACTERS CLASS CLASS-ID CLOCK-UNITS CLOSE COBOL CODE CODE-SET COL COLLATING COLS COLUMN COLUMNS COM-REG COMMA - COMMON COMMUNICATION COMP-1 COMP-2 COMP-3 COMP-4 COMP-5 COMP COMPUTATIONAL-1 COMPUTATIONAL-2 - COMPUTATIONAL-3 COMPUTATIONAL-4 COMPUTATIONAL-5 COMPUTATIONAL COMPUTE CONDITION CONSTANT CONTAINS CONTENT - CONTINUE CONTROL CONTROLS CONVERTING COPY CORR CORRESPONDING COUNT CRT CURRENCY CURSOR DATA-POINTER DATE - DATE-COMPILED DATE-WRITTEN DAY DAY-OF-WEEK DBCS DE DEBUG-CONTENTS DEBUG-ITEM DEBUG-LINE DEBUG-NAME DEBUG-SUB-1 - DEBUG-SUB-2 DEBUG-SUB-3 DEBUGGING DECIMAL-POINT DECLARATIVES DEFAULT DELETE DELIMITED DELIMITER DEPENDING - DESCENDING DESTINATION DETAIL DISABLE DISPLAY-1 DISPLAY DIVIDE DOWN DUPLICATES DYNAMIC EC EGCS EGI - EJECT ELSE EMI ENABLE END-ACCEPT END-ADD END-CALL END-COMPUTE END-DELETE END-DISPLAY END-DIVIDE END-EVALUATE - END-EXEC END-IF END-INVOKE END-JSON END-MULTIPLY END-OF-PAGE END-PERFORM END-READ END-RECEIVE END-RETURN - END-REWRITE END-SEARCH END-START END-STRING END-SUBTRACT END-UNSTRING END-WRITE END-XML ENDING END ENTER ENTRY - EO EOP EQUAL ERROR ESI EVALUATE EVERY EXCEPTION EXCEPTION-OBJECT EXEC EXECUTE EXIT EXTEND EXTERNAL - FACTORY FALSE FD FILE-CONTROL FILLER FINAL FIRST FLOAT-EXTENDED FLOAT-LONG FLOAT-SHORT FOOTING FOR FORMAT - FREE FROM FUNCTION FUNCTION-ID FUNCTION-POINTER GENERATE GET GIVING GLOBAL GO GOBACK GREATER GROUP GROUP-USAGE - HEADING HIGH-VALUE HIGH-VALUES I-O-CONTROL I-O ID IF IN INDEX INDEXED INDICATE INHERITS INITIAL - INITIALIZE INITIATE INPUT INSERT INSPECT INSTALLATION INTERFACE INTERFACE-ID INTO INVALID INVOKE - IS JAVA JNIENVPTR JSON JSON-CODE JSON-STATUS JUST JUSTIFIED KANJI KEY LABEL LAST LEADING LEFT LENGTH LESS LIMIT - LIMITS LINAGE-COUNTER LINAGE LINE-COUNTER LINES LINE LOCALE LOCK LOW-VALUE LOW-VALUES - MEMORY MERGE MESSAGE METHOD METHOD-ID MINUS MODE MODULES MORE-LABELS MOVE MULTIPLE MULTIPLY NATIONAL - NATIONAL-EDITED NATIVE NEGATIVE NESTED NEXT NO NOT NULL NULLS NUMBER NUMERIC NUMERIC-EDITED OBJECT - OBJECT-COMPUTER OBJECT-REFERENCE OCCURS OF OFF OMITTED ON OPEN OPTIONAL OPTIONS OR ORDER ORGANIZATION - OTHER OUTPUT OVERFLOW OVERRIDE PACKED-DECIMAL PADDING PAGE PAGE-COUNTER PASSWORD PERFORM PF PH PIC PICTURE - PLUS POINTER- POINTER-31 POINTER-32 POINTER-64 POINTER POSITION POSITIVE PRESENT PRINTING - PROCEDURE-POINTER PROCEDURES PROCEED PROCESSING PROGRAM-ID PROGRAM-POINTER PROGRAM PROPERTY PROTOTYPE - PURGE QUEUE QUOTE QUOTES RAISE RAISING RANDOM RD READ READY RECEIVE RECORD RECORDING RECORDS RECURSIVE REDEFINES - REEL REFERENCE REFERENCES RELATIVE RELEASE RELOAD REMAINDER REMOVAL RENAMES REPLACE REPLACING REPORT REPORTING - REPORTS REPOSITORY RERUN RESERVE RESET RESUME RETRY RETURN RETURN-CODE RETURNING REVERSED REWIND REWRITE RF RH - RIGHT ROUNDED RUN SAME SCREEN SD SEARCH SECTION SECURITY SEGMENT SEGMENT-LIMIT SELECT SELF SEND SENTENCE - SEPARATE SEQUENCE SEQUENTIAL SERVICE SET SHARING SHIFT-IN SHIFT-OUT SIGN SIZE SKIP1 SKIP2 SKIP3 - SORT-CONTROL SORT-CORE-SIZE SORT-FILE-SIZE SORT-MERGE SORT-MESSAGE SORT-MODE-SIZE SORT-RETURN SORT - SOURCE-COMPUTER SOURCES SOURCE SPACE SPACES SPECIAL-NAMES SQL SQLIMS STANDARD-1 STANDARD-2 STANDARD START STATUS STOP - STRING SUB-QUEUE-1 SUB-QUEUE-2 SUB-QUEUE-3 SUBTRACT SUM SUPER SUPPRESS SYMBOLIC SYNC SYNCHRONIZED SYSTEM-DEFAULT - TABLE TALLY TALLYING TAPE TERMINAL TERMINATE TEST TEXT THAN THEN THROUGH THRU TIME TIMES TITLE TO TOP TRACE - TRAILING TRUE TYPE TYPEDEF UNIT UNIVERSAL UNLOCK UNSTRING UNTIL UP UPON USAGE USE USER-DEFAULT USING UTF-8 - VAL-STATUS VALID VALIDATE VALIDATE-STATUS VALUE VALUES VARYING VOLATILE WHEN WHEN-COMPILED WITH WORDS - WRITE WRITE-ONLY XML-CODE XML-EVENT XML-INFORMATION XML-NAMESPACE XML-NAMESPACE-PREFIX - XML-NNAMESPACE XML-NNAMESPACE-PREFIX XML-NTEXT XML-SCHEMA XML-TEXT XML ZERO ZEROES ZEROS - )) - end + SECTIONS = Set.new %w( + CONFIGURATION INPUT-OUTPUT FILE WORKING-STORAGE LOCAL-STORAGE LINKAGE SECTION + ) state :root do # First detect the comments @@ -80,16 +31,12 @@ def self.keywords rule %r/'/, Str::Single, :string_single # Keywords and divisions - rule %r/(? Date: Wed, 8 Apr 2026 14:05:27 -0400 Subject: [PATCH 027/154] use the keywords api for coffeescript --- lib/rouge/lexers/coffeescript.rb | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/lib/rouge/lexers/coffeescript.rb b/lib/rouge/lexers/coffeescript.rb index 032943b931..1f8931ce7b 100644 --- a/lib/rouge/lexers/coffeescript.rb +++ b/lib/rouge/lexers/coffeescript.rb @@ -126,18 +126,11 @@ def self.builtins rule %r/#{id}(?=\s*:)/, Name::Attribute, :slash_starts_regex - rule %r/#{id}/ do |m| - if self.class.keywords.include? m[0] - token Keyword - elsif self.class.constants.include? m[0] - token Name::Constant - elsif self.class.builtins.include? m[0] - token Name::Builtin - else - token Name::Other - end - - push :slash_starts_regex + keywords id do + rule :keywords, Keyword, :slash_starts_regex + rule :constants, Name::Constant, :slash_starts_regex + rule :builtins, Name::Builtin, :slash_starts_regex + default Name::Other, :slash_starts_regex end rule %r/[{(\[;,]/, Punctuation, :slash_starts_regex From 8b31e32a354af7aebb0387eb578d7cc0103bd88e Mon Sep 17 00:00:00 2001 From: jneen Date: Wed, 8 Apr 2026 14:08:52 -0400 Subject: [PATCH 028/154] use the keywords api for common lisp --- lib/rouge/lexers/common_lisp.rb | 225 +---------------------- lib/rouge/lexers/common_lisp/builtins.rb | 202 ++++++++++++++++++++ 2 files changed, 212 insertions(+), 215 deletions(-) create mode 100644 lib/rouge/lexers/common_lisp/builtins.rb diff --git a/lib/rouge/lexers/common_lisp.rb b/lib/rouge/lexers/common_lisp.rb index 1cc171664f..72c98898d7 100644 --- a/lib/rouge/lexers/common_lisp.rb +++ b/lib/rouge/lexers/common_lisp.rb @@ -12,202 +12,7 @@ class CommonLisp < RegexLexer filenames '*.cl', '*.lisp', '*.asd', '*.el' # used for Elisp too mimetypes 'text/x-common-lisp' - # 638 functions - BUILTIN_FUNCTIONS = Set.new %w( - < <= = > >= - / /= * + 1- 1+ abort abs acons acos acosh add-method - adjoin adjustable-array-p adjust-array allocate-instance - alpha-char-p alphanumericp append apply apropos apropos-list - aref arithmetic-error-operands arithmetic-error-operation - array-dimension array-dimensions array-displacement - array-element-type array-has-fill-pointer-p array-in-bounds-p - arrayp array-rank array-row-major-index array-total-size - ash asin asinh assoc assoc-if assoc-if-not atan atanh atom - bit bit-and bit-andc1 bit-andc2 bit-eqv bit-ior bit-nand - bit-nor bit-not bit-orc1 bit-orc2 bit-vector-p bit-xor boole - both-case-p boundp break broadcast-stream-streams butlast - byte byte-position byte-size caaaar caaadr caaar caadar - caaddr caadr caar cadaar cadadr cadar caddar cadddr caddr - cadr call-next-method car cdaaar cdaadr cdaar cdadar cdaddr - cdadr cdar cddaar cddadr cddar cdddar cddddr cdddr cddr cdr - ceiling cell-error-name cerror change-class char char< char<= - char= char> char>= char/= character characterp char-code - char-downcase char-equal char-greaterp char-int char-lessp - char-name char-not-equal char-not-greaterp char-not-lessp - char-upcase cis class-name class-of clear-input clear-output - close clrhash code-char coerce compile compiled-function-p - compile-file compile-file-pathname compiler-macro-function - complement complex complexp compute-applicable-methods - compute-restarts concatenate concatenated-stream-streams conjugate - cons consp constantly constantp continue copy-alist copy-list - copy-pprint-dispatch copy-readtable copy-seq copy-structure - copy-symbol copy-tree cos cosh count count-if count-if-not - decode-float decode-universal-time delete delete-duplicates - delete-file delete-if delete-if-not delete-package denominator - deposit-field describe describe-object digit-char digit-char-p - directory directory-namestring disassemble documentation dpb - dribble echo-stream-input-stream echo-stream-output-stream - ed eighth elt encode-universal-time endp enough-namestring - ensure-directories-exist ensure-generic-function eq - eql equal equalp error eval evenp every exp export expt - fboundp fceiling fdefinition ffloor fifth file-author - file-error-pathname file-length file-namestring file-position - file-string-length file-write-date fill fill-pointer find - find-all-symbols find-class find-if find-if-not find-method - find-package find-restart find-symbol finish-output first - float float-digits floatp float-precision float-radix - float-sign floor fmakunbound force-output format fourth - fresh-line fround ftruncate funcall function-keywords - function-lambda-expression functionp gcd gensym gentemp get - get-decoded-time get-dispatch-macro-character getf gethash - get-internal-real-time get-internal-run-time get-macro-character - get-output-stream-string get-properties get-setf-expansion - get-universal-time graphic-char-p hash-table-count hash-table-p - hash-table-rehash-size hash-table-rehash-threshold - hash-table-size hash-table-test host-namestring identity - imagpart import initialize-instance input-stream-p inspect - integer-decode-float integer-length integerp interactive-stream-p - intern intersection invalid-method-error invoke-debugger - invoke-restart invoke-restart-interactively isqrt keywordp - last lcm ldb ldb-test ldiff length lisp-implementation-type - lisp-implementation-version list list* list-all-packages listen - list-length listp load load-logical-pathname-translations - log logand logandc1 logandc2 logbitp logcount logeqv - logical-pathname logical-pathname-translations logior - lognand lognor lognot logorc1 logorc2 logtest logxor - long-site-name lower-case-p machine-instance machine-type - machine-version macroexpand macroexpand-1 macro-function - make-array make-broadcast-stream make-concatenated-stream - make-condition make-dispatch-macro-character make-echo-stream - make-hash-table make-instance make-instances-obsolete make-list - make-load-form make-load-form-saving-slots make-package - make-pathname make-random-state make-sequence make-string - make-string-input-stream make-string-output-stream make-symbol - make-synonym-stream make-two-way-stream makunbound map mapc - mapcan mapcar mapcon maphash map-into mapl maplist mask-field - max member member-if member-if-not merge merge-pathnames - method-combination-error method-qualifiers min minusp mismatch mod - muffle-warning name-char namestring nbutlast nconc next-method-p - nintersection ninth no-applicable-method no-next-method not notany - notevery nreconc nreverse nset-difference nset-exclusive-or - nstring-capitalize nstring-downcase nstring-upcase nsublis - nsubst nsubst-if nsubst-if-not nsubstitute nsubstitute-if - nsubstitute-if-not nth nthcdr null numberp numerator nunion - oddp open open-stream-p output-stream-p package-error-package - package-name package-nicknames packagep package-shadowing-symbols - package-used-by-list package-use-list pairlis parse-integer - parse-namestring pathname pathname-device pathname-directory - pathname-host pathname-match-p pathname-name pathnamep - pathname-type pathname-version peek-char phase plusp - position position-if position-if-not pprint pprint-dispatch - pprint-fill pprint-indent pprint-linear pprint-newline pprint-tab - pprint-tabular prin1 prin1-to-string princ princ-to-string print - print-object probe-file proclaim provide random random-state-p - rassoc rassoc-if rassoc-if-not rational rationalize rationalp - read read-byte read-char read-char-no-hang read-delimited-list - read-from-string read-line read-preserving-whitespace - read-sequence readtable-case readtablep realp realpart - reduce reinitialize-instance rem remhash remove - remove-duplicates remove-if remove-if-not remove-method - remprop rename-file rename-package replace require rest - restart-name revappend reverse room round row-major-aref - rplaca rplacd sbit scale-float schar search second set - set-difference set-dispatch-macro-character set-exclusive-or - set-macro-character set-pprint-dispatch set-syntax-from-char - seventh shadow shadowing-import shared-initialize - short-site-name signal signum simple-bit-vector-p - simple-condition-format-arguments simple-condition-format-control - simple-string-p simple-vector-p sin sinh sixth sleep slot-boundp - slot-exists-p slot-makunbound slot-missing slot-unbound slot-value - software-type software-version some sort special-operator-p - sqrt stable-sort standard-char-p store-value stream-element-type - stream-error-stream stream-external-format streamp string string< - string<= string= string> string>= string/= string-capitalize - string-downcase string-equal string-greaterp string-left-trim - string-lessp string-not-equal string-not-greaterp string-not-lessp - stringp string-right-trim string-trim string-upcase sublis subseq - subsetp subst subst-if subst-if-not substitute substitute-if - substitute-if-not subtypepsvref sxhash symbol-function - symbol-name symbolp symbol-package symbol-plist symbol-value - synonym-stream-symbol syntax: tailp tan tanh tenth terpri third - translate-logical-pathname translate-pathname tree-equal truename - truncate two-way-stream-input-stream two-way-stream-output-stream - type-error-datum type-error-expected-type type-of - typep unbound-slot-instance unexport unintern union - unread-char unuse-package update-instance-for-different-class - update-instance-for-redefined-class upgraded-array-element-type - upgraded-complex-part-type upper-case-p use-package - user-homedir-pathname use-value values values-list vector vectorp - vector-pop vector-push vector-push-extend warn wild-pathname-p - write write-byte write-char write-line write-sequence write-string - write-to-string yes-or-no-p y-or-n-p zerop - ).freeze - - SPECIAL_FORMS = Set.new %w( - block catch declare eval-when flet function go if labels lambda - let let* load-time-value locally macrolet multiple-value-call - multiple-value-prog1 progn progv quote return-from setq - symbol-macrolet tagbody the throw unwind-protect - ) - - MACROS = Set.new %w( - and assert call-method case ccase check-type cond ctypecase decf - declaim defclass defconstant defgeneric define-compiler-macro - define-condition define-method-combination define-modify-macro - define-setf-expander define-symbol-macro defmacro defmethod - defpackage defparameter defsetf defstruct defsystem deftype defun defvar - destructuring-bind do do* do-all-symbols do-external-symbols - dolist do-symbols dotimes ecase etypecase formatter - handler-bind handler-case ignore-errors incf in-package - lambda loop loop-finish make-method multiple-value-bind - multiple-value-list multiple-value-setq nth-value or pop - pprint-exit-if-list-exhausted pprint-logical-block pprint-pop - print-unreadable-object prog prog* prog1 prog2 psetf psetq - push pushnew remf restart-bind restart-case return rotatef - setf shiftf step time trace typecase unless untrace when - with-accessors with-compilation-unit with-condition-restarts - with-hash-table-iterator with-input-from-string with-open-file - with-open-stream with-output-to-string with-package-iterator - with-simple-restart with-slots with-standard-io-syntax - ) - - LAMBDA_LIST_KEYWORDS = Set.new %w( - &allow-other-keys &aux &body &environment &key &optional &rest - &whole - ) - - DECLARATIONS = Set.new %w( - dynamic-extent ignore optimize ftype inline special ignorable - notinline type - ) - - BUILTIN_TYPES = Set.new %w( - atom boolean base-char base-string bignum bit compiled-function - extended-char fixnum keyword nil signed-byte short-float - single-float double-float long-float simple-array - simple-base-string simple-bit-vector simple-string simple-vector - standard-char unsigned-byte - - arithmetic-error cell-error condition control-error - division-by-zero end-of-file error file-error - floating-point-inexact floating-point-overflow - floating-point-underflow floating-point-invalid-operation - parse-error package-error print-not-readable program-error - reader-error serious-condition simple-condition simple-error - simple-type-error simple-warning stream-error storage-condition - style-warning type-error unbound-variable unbound-slot - undefined-function warning - ) - - BUILTIN_CLASSES = Set.new %w( - array broadcast-stream bit-vector built-in-class character - class complex concatenated-stream cons echo-stream file-stream - float function generic-function hash-table integer list - logical-pathname method-combination method null number package - pathname ratio rational readtable real random-state restart - sequence standard-class standard-generic-function standard-method - standard-object string-stream stream string structure-class - structure-object symbol synonym-stream t two-way-stream vector - ) + lazy { require_relative 'common_lisp/builtins' } nonmacro = /\\.|[a-zA-Z0-9!$%&*+-\/<=>?@\[\]^_{}~]/ constituent = /#{nonmacro}|[#.:]/ @@ -296,26 +101,16 @@ class complex concatenated-stream cons echo-stream file-stream # functions and variables # note that these get filtered through in stream_tokens rule %r/\*#{symbol}\*/, Name::Variable::Global - rule symbol do |m| - sym = m[0] - if BUILTIN_FUNCTIONS.include? sym - token Name::Builtin - elsif SPECIAL_FORMS.include? sym - token Keyword - elsif MACROS.include? sym - token Name::Builtin - elsif LAMBDA_LIST_KEYWORDS.include? sym - token Keyword - elsif DECLARATIONS.include? sym - token Keyword - elsif BUILTIN_TYPES.include? sym - token Keyword::Type - elsif BUILTIN_CLASSES.include? sym - token Name::Class - else - token Name::Variable - end + keywords symbol do + rule BUILTIN_FUNCTIONS, Name::Builtin + rule SPECIAL_FORMS, Keyword + rule MACROS, Name::Builtin + rule LAMBDA_LIST_KEYWORDS, Keyword + rule DECLARATIONS, Keyword + rule BUILTIN_TYPES, Keyword + rule BUILTIN_CLASSES, Keyword + default Name end rule %r/\(/, Punctuation, :root diff --git a/lib/rouge/lexers/common_lisp/builtins.rb b/lib/rouge/lexers/common_lisp/builtins.rb new file mode 100644 index 0000000000..cfdfc7e6ac --- /dev/null +++ b/lib/rouge/lexers/common_lisp/builtins.rb @@ -0,0 +1,202 @@ +module Rouge + module Lexers + class CommonLisp + # 638 functions + BUILTIN_FUNCTIONS = Set.new %w( + < <= = > >= - / /= * + 1- 1+ abort abs acons acos acosh add-method + adjoin adjustable-array-p adjust-array allocate-instance + alpha-char-p alphanumericp append apply apropos apropos-list + aref arithmetic-error-operands arithmetic-error-operation + array-dimension array-dimensions array-displacement + array-element-type array-has-fill-pointer-p array-in-bounds-p + arrayp array-rank array-row-major-index array-total-size + ash asin asinh assoc assoc-if assoc-if-not atan atanh atom + bit bit-and bit-andc1 bit-andc2 bit-eqv bit-ior bit-nand + bit-nor bit-not bit-orc1 bit-orc2 bit-vector-p bit-xor boole + both-case-p boundp break broadcast-stream-streams butlast + byte byte-position byte-size caaaar caaadr caaar caadar + caaddr caadr caar cadaar cadadr cadar caddar cadddr caddr + cadr call-next-method car cdaaar cdaadr cdaar cdadar cdaddr + cdadr cdar cddaar cddadr cddar cdddar cddddr cdddr cddr cdr + ceiling cell-error-name cerror change-class char char< char<= + char= char> char>= char/= character characterp char-code + char-downcase char-equal char-greaterp char-int char-lessp + char-name char-not-equal char-not-greaterp char-not-lessp + char-upcase cis class-name class-of clear-input clear-output + close clrhash code-char coerce compile compiled-function-p + compile-file compile-file-pathname compiler-macro-function + complement complex complexp compute-applicable-methods + compute-restarts concatenate concatenated-stream-streams conjugate + cons consp constantly constantp continue copy-alist copy-list + copy-pprint-dispatch copy-readtable copy-seq copy-structure + copy-symbol copy-tree cos cosh count count-if count-if-not + decode-float decode-universal-time delete delete-duplicates + delete-file delete-if delete-if-not delete-package denominator + deposit-field describe describe-object digit-char digit-char-p + directory directory-namestring disassemble documentation dpb + dribble echo-stream-input-stream echo-stream-output-stream + ed eighth elt encode-universal-time endp enough-namestring + ensure-directories-exist ensure-generic-function eq + eql equal equalp error eval evenp every exp export expt + fboundp fceiling fdefinition ffloor fifth file-author + file-error-pathname file-length file-namestring file-position + file-string-length file-write-date fill fill-pointer find + find-all-symbols find-class find-if find-if-not find-method + find-package find-restart find-symbol finish-output first + float float-digits floatp float-precision float-radix + float-sign floor fmakunbound force-output format fourth + fresh-line fround ftruncate funcall function-keywords + function-lambda-expression functionp gcd gensym gentemp get + get-decoded-time get-dispatch-macro-character getf gethash + get-internal-real-time get-internal-run-time get-macro-character + get-output-stream-string get-properties get-setf-expansion + get-universal-time graphic-char-p hash-table-count hash-table-p + hash-table-rehash-size hash-table-rehash-threshold + hash-table-size hash-table-test host-namestring identity + imagpart import initialize-instance input-stream-p inspect + integer-decode-float integer-length integerp interactive-stream-p + intern intersection invalid-method-error invoke-debugger + invoke-restart invoke-restart-interactively isqrt keywordp + last lcm ldb ldb-test ldiff length lisp-implementation-type + lisp-implementation-version list list* list-all-packages listen + list-length listp load load-logical-pathname-translations + log logand logandc1 logandc2 logbitp logcount logeqv + logical-pathname logical-pathname-translations logior + lognand lognor lognot logorc1 logorc2 logtest logxor + long-site-name lower-case-p machine-instance machine-type + machine-version macroexpand macroexpand-1 macro-function + make-array make-broadcast-stream make-concatenated-stream + make-condition make-dispatch-macro-character make-echo-stream + make-hash-table make-instance make-instances-obsolete make-list + make-load-form make-load-form-saving-slots make-package + make-pathname make-random-state make-sequence make-string + make-string-input-stream make-string-output-stream make-symbol + make-synonym-stream make-two-way-stream makunbound map mapc + mapcan mapcar mapcon maphash map-into mapl maplist mask-field + max member member-if member-if-not merge merge-pathnames + method-combination-error method-qualifiers min minusp mismatch mod + muffle-warning name-char namestring nbutlast nconc next-method-p + nintersection ninth no-applicable-method no-next-method not notany + notevery nreconc nreverse nset-difference nset-exclusive-or + nstring-capitalize nstring-downcase nstring-upcase nsublis + nsubst nsubst-if nsubst-if-not nsubstitute nsubstitute-if + nsubstitute-if-not nth nthcdr null numberp numerator nunion + oddp open open-stream-p output-stream-p package-error-package + package-name package-nicknames packagep package-shadowing-symbols + package-used-by-list package-use-list pairlis parse-integer + parse-namestring pathname pathname-device pathname-directory + pathname-host pathname-match-p pathname-name pathnamep + pathname-type pathname-version peek-char phase plusp + position position-if position-if-not pprint pprint-dispatch + pprint-fill pprint-indent pprint-linear pprint-newline pprint-tab + pprint-tabular prin1 prin1-to-string princ princ-to-string print + print-object probe-file proclaim provide random random-state-p + rassoc rassoc-if rassoc-if-not rational rationalize rationalp + read read-byte read-char read-char-no-hang read-delimited-list + read-from-string read-line read-preserving-whitespace + read-sequence readtable-case readtablep realp realpart + reduce reinitialize-instance rem remhash remove + remove-duplicates remove-if remove-if-not remove-method + remprop rename-file rename-package replace require rest + restart-name revappend reverse room round row-major-aref + rplaca rplacd sbit scale-float schar search second set + set-difference set-dispatch-macro-character set-exclusive-or + set-macro-character set-pprint-dispatch set-syntax-from-char + seventh shadow shadowing-import shared-initialize + short-site-name signal signum simple-bit-vector-p + simple-condition-format-arguments simple-condition-format-control + simple-string-p simple-vector-p sin sinh sixth sleep slot-boundp + slot-exists-p slot-makunbound slot-missing slot-unbound slot-value + software-type software-version some sort special-operator-p + sqrt stable-sort standard-char-p store-value stream-element-type + stream-error-stream stream-external-format streamp string string< + string<= string= string> string>= string/= string-capitalize + string-downcase string-equal string-greaterp string-left-trim + string-lessp string-not-equal string-not-greaterp string-not-lessp + stringp string-right-trim string-trim string-upcase sublis subseq + subsetp subst subst-if subst-if-not substitute substitute-if + substitute-if-not subtypepsvref sxhash symbol-function + symbol-name symbolp symbol-package symbol-plist symbol-value + synonym-stream-symbol syntax: tailp tan tanh tenth terpri third + translate-logical-pathname translate-pathname tree-equal truename + truncate two-way-stream-input-stream two-way-stream-output-stream + type-error-datum type-error-expected-type type-of + typep unbound-slot-instance unexport unintern union + unread-char unuse-package update-instance-for-different-class + update-instance-for-redefined-class upgraded-array-element-type + upgraded-complex-part-type upper-case-p use-package + user-homedir-pathname use-value values values-list vector vectorp + vector-pop vector-push vector-push-extend warn wild-pathname-p + write write-byte write-char write-line write-sequence write-string + write-to-string yes-or-no-p y-or-n-p zerop + ).freeze + + SPECIAL_FORMS = Set.new %w( + block catch declare eval-when flet function go if labels lambda + let let* load-time-value locally macrolet multiple-value-call + multiple-value-prog1 progn progv quote return-from setq + symbol-macrolet tagbody the throw unwind-protect + ) + + MACROS = Set.new %w( + and assert call-method case ccase check-type cond ctypecase decf + declaim defclass defconstant defgeneric define-compiler-macro + define-condition define-method-combination define-modify-macro + define-setf-expander define-symbol-macro defmacro defmethod + defpackage defparameter defsetf defstruct defsystem deftype defun defvar + destructuring-bind do do* do-all-symbols do-external-symbols + dolist do-symbols dotimes ecase etypecase formatter + handler-bind handler-case ignore-errors incf in-package + lambda loop loop-finish make-method multiple-value-bind + multiple-value-list multiple-value-setq nth-value or pop + pprint-exit-if-list-exhausted pprint-logical-block pprint-pop + print-unreadable-object prog prog* prog1 prog2 psetf psetq + push pushnew remf restart-bind restart-case return rotatef + setf shiftf step time trace typecase unless untrace when + with-accessors with-compilation-unit with-condition-restarts + with-hash-table-iterator with-input-from-string with-open-file + with-open-stream with-output-to-string with-package-iterator + with-simple-restart with-slots with-standard-io-syntax + ) + + LAMBDA_LIST_KEYWORDS = Set.new %w( + &allow-other-keys &aux &body &environment &key &optional &rest + &whole + ) + + DECLARATIONS = Set.new %w( + dynamic-extent ignore optimize ftype inline special ignorable + notinline type + ) + + BUILTIN_TYPES = Set.new %w( + atom boolean base-char base-string bignum bit compiled-function + extended-char fixnum keyword nil signed-byte short-float + single-float double-float long-float simple-array + simple-base-string simple-bit-vector simple-string simple-vector + standard-char unsigned-byte + + arithmetic-error cell-error condition control-error + division-by-zero end-of-file error file-error + floating-point-inexact floating-point-overflow + floating-point-underflow floating-point-invalid-operation + parse-error package-error print-not-readable program-error + reader-error serious-condition simple-condition simple-error + simple-type-error simple-warning stream-error storage-condition + style-warning type-error unbound-variable unbound-slot + undefined-function warning + ) + + BUILTIN_CLASSES = Set.new %w( + array broadcast-stream bit-vector built-in-class character + class complex concatenated-stream cons echo-stream file-stream + float function generic-function hash-table integer list + logical-pathname method-combination method null number package + pathname ratio rational readtable real random-state restart + sequence standard-class standard-generic-function standard-method + standard-object string-stream stream string structure-class + structure-object symbol synonym-stream t two-way-stream vector + ) + end + end +end From dfa0ebbda7eb563ccac3d2f4175132c60034df95 Mon Sep 17 00:00:00 2001 From: jneen Date: Thu, 9 Apr 2026 11:04:48 -0400 Subject: [PATCH 029/154] remove extraneous set creation in cop --- lib/rubocop/cop/rouge/no_huge_collections.rb | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/rubocop/cop/rouge/no_huge_collections.rb b/lib/rubocop/cop/rouge/no_huge_collections.rb index 26036ad3bb..6f8ce72446 100644 --- a/lib/rubocop/cop/rouge/no_huge_collections.rb +++ b/lib/rubocop/cop/rouge/no_huge_collections.rb @@ -14,9 +14,6 @@ class NoHugeCollections < RuboCop::Cop::Base RESTRICT_ON_SEND = [:[]] MAX_SIZE = 300 - Set["1", "2", "3"] - Set.new %w(1 2 3) - def on_send(node) return unless set_literal?(node) From f1d3aed9958ecd0e829ddc8f1e4e44ccf534321d Mon Sep 17 00:00:00 2001 From: jneen Date: Thu, 9 Apr 2026 12:00:22 -0400 Subject: [PATCH 030/154] add a spec that keywords don't get leaked up the chain --- spec/lexers/c_spec.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/spec/lexers/c_spec.rb b/spec/lexers/c_spec.rb index c8acc301fd..5f3f49da39 100644 --- a/spec/lexers/c_spec.rb +++ b/spec/lexers/c_spec.rb @@ -24,5 +24,9 @@ it 'recognizes one-line comments not followed by a newline (#796)' do assert_tokens_equal '// comment', ['Comment.Single', '// comment'] end + + it "doesn't leak c++ keywords" do + assert_tokens_equal 'using', ['Name', 'using'] + end end end From 6a01350b39c3b35d65f728772b173a75ac8a7ff3 Mon Sep 17 00:00:00 2001 From: jneen Date: Thu, 9 Apr 2026 12:00:33 -0400 Subject: [PATCH 031/154] use the keywords api for crystal --- .rubocop_todo.yml | 1 - lib/rouge/lexers/crystal.rb | 108 +++++++++++++++++++----------------- 2 files changed, 58 insertions(+), 51 deletions(-) diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index b45489bd50..7b0efad451 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -50,7 +50,6 @@ Naming/VariableName: Rouge/NoBuildingAlternationPatternInRegexp: Exclude: - 'lib/rouge/lexers/armasm.rb' - - 'lib/rouge/lexers/crystal.rb' - 'lib/rouge/lexers/csharp.rb' - 'lib/rouge/lexers/d.rb' - 'lib/rouge/lexers/dart.rb' diff --git a/lib/rouge/lexers/crystal.rb b/lib/rouge/lexers/crystal.rb index 91d45297c1..afb7b06aab 100644 --- a/lib/rouge/lexers/crystal.rb +++ b/lib/rouge/lexers/crystal.rb @@ -16,6 +16,40 @@ def self.detect?(text) return true if text.shebang? 'crystal' end + KEYWORDS = Set.new %w( + BEGIN END alias begin break case defined\? do else elsif end + ensure for if ifdef in next redo rescue raise retry return super then + undef unless until when while yield lib fun type of as + + abstract union enum + ) + + KEYWORDS_PSEUDO = Set.new %w( + initialize new loop include extend raise attr_reader attr_writer + attr_accessor alias_method attr catch throw private module_function + public protected true false nil __FILE__ __LINE__ + getter getter? getter! property property? property! struct record + ) + + BUILTIN_GLOBALS = Set.new %w( + abort ancestors at_exit + caller catch chomp chop clone constants + display dup eval exec exit extend fail fork format freeze + getc gets global_variables gsub hash id included_modules + inspect instance_eval instance_method instance_methods + lambda loop method method_missing + methods module_eval object_id open p print printf + proc putc puts raise rand + readline readlines require scan select self send + sleep split sprintf srand sub syscall system + test throw to_a to_s trap warn flag? + ) + + BUILTIN_METHODS = Set.new %w( + eql? equal? include? is_a? iterator? kind_of? nil? + chomp! chop! exit! gsub! sub! + ) + state :symbols do # symbols rule %r( @@ -90,9 +124,11 @@ def self.detect?(text) end # double-quoted string and symbol - [[:string, Str::Double, '"'], - [:sym, Str::Symbol, '"'], - [:backtick, Str::Backtick, '`']].each do |name, tok, fin| + [ + [:string, Str::Double, '"'], + [:sym, Str::Symbol, '"'], + [:backtick, Str::Backtick, '`'] + ].each do |name, tok, fin| state :"simple_#{name}" do mixin :string_intp_escaped rule %r/[^\\#{fin}#]+/m, tok @@ -101,39 +137,6 @@ def self.detect?(text) end end - keywords = %w( - BEGIN END alias begin break case defined\? do else elsif end - ensure for if ifdef in next redo rescue raise retry return super then - undef unless until when while yield lib fun type of as - ) - - keywords_pseudo = %w( - initialize new loop include extend raise attr_reader attr_writer - attr_accessor alias_method attr catch throw private module_function - public protected true false nil __FILE__ __LINE__ - getter getter? getter! property property? property! struct record - ) - - builtins_g = %w( - abort ancestors at_exit - caller catch chomp chop clone constants - display dup eval exec exit extend fail fork format freeze - getc gets global_variables gsub hash id included_modules - inspect instance_eval instance_method instance_methods - lambda loop method method_missing - methods module_eval name object_id open p print printf - proc putc puts raise rand - readline readlines require scan select self send - sleep split sprintf srand sub syscall system - test throw to_a to_s trap warn - ) - - builtins_q = %w( - eql equal include is_a iterator kind_of nil - ) - - builtins_b = %w(chomp chop exit gsub sub) - start do push :expr_start @heredoc_queue = [] @@ -175,9 +178,6 @@ def self.detect?(text) mixin :strings - rule %r/(?:#{keywords.join('|')})\b/, Keyword, :expr_start - rule %r/(?:#{keywords_pseudo.join('|')})\b/, Keyword::Pseudo, :expr_start - rule %r( (module) (\s+) @@ -196,25 +196,33 @@ def self.detect?(text) push :classname end - rule %r/(?:#{builtins_q.join('|')})[?]/, Name::Builtin, :expr_start - rule %r/(?:#{builtins_b.join('|')})!/, Name::Builtin, :expr_start - rule %r/(?=])/ do - groups Punctuation, Text, Name::Function - push :method_call + keywords %r/(\.|::)(\s*)([\p{Ll}_]\p{Word}*[!?]?|[*%&^`~+-\/\[<>=])/ do + group 3 + rule BUILTIN_METHODS do + groups Punctuation, Text, Name::Builtin + push :method_call + end + + default do + groups Punctuation, Text, Name::Function + push :method_call + end + end + + keywords %r/[\p{L}_]\p{Word}*[?!]?/ do + rule KEYWORDS, Keyword, :expr_start + rule KEYWORDS_PSEUDO, Keyword::Pseudo, :expr_start + rule BUILTIN_GLOBALS, Name::Builtin, :method_call + default Name, :expr_start end - rule %r/[\p{L}_]\p{Word}*[?!]/, Name, :expr_start - rule %r/[\p{L}_]\p{Word}*/, Name, :method_call rule %r/\*\*|\/\/|>=|<=|<=>|<>?|=~|={3}|!~|&&?|\|\||\./, Operator, :expr_start rule %r/{%|%}/, Punctuation From fa8026b7af7c1ff254c056e73ecae4ad545df736 Mon Sep 17 00:00:00 2001 From: jneen Date: Thu, 9 Apr 2026 12:11:31 -0400 Subject: [PATCH 032/154] use the keywords api for csharp (and refactor preproc) --- .rubocop_todo.yml | 1 - lib/rouge/lexers/csharp.rb | 23 +++++++++++++++++------ spec/visual/samples/csharp | 6 ++++++ 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index 7b0efad451..f572369838 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -50,7 +50,6 @@ Naming/VariableName: Rouge/NoBuildingAlternationPatternInRegexp: Exclude: - 'lib/rouge/lexers/armasm.rb' - - 'lib/rouge/lexers/csharp.rb' - 'lib/rouge/lexers/d.rb' - 'lib/rouge/lexers/dart.rb' - 'lib/rouge/lexers/eiffel.rb' diff --git a/lib/rouge/lexers/csharp.rb b/lib/rouge/lexers/csharp.rb index a83fb58d83..e198c85c13 100644 --- a/lib/rouge/lexers/csharp.rb +++ b/lib/rouge/lexers/csharp.rb @@ -18,7 +18,7 @@ class CSharp < RegexLexer # Contextual Keywords # LINQ Query Expressions def self.keywords - @keywords ||= %w( + @keywords ||= Set.new %w( abstract add alias and as ascending async await base break by case catch checked const continue default delegate descending do else enum equals event explicit extern false @@ -34,20 +34,22 @@ def self.keywords end def self.keywords_type - @keywords_type ||= %w( + @keywords_type ||= Set.new %w( bool byte char decimal double dynamic float int long nint nuint object sbyte short string uint ulong ushort var ) end def self.cpp_keywords - @cpp_keywords ||= %w( + @cpp_keywords ||= Set.new %w( if endif else elif define undef line error warning region endregion pragma nullable ) end state :whitespace do + rule %r/^[ \t]*#/, Comment::Preproc, :preproc + rule %r/\s+/m, Text rule %r(//.*?$), Comment::Single rule %r(/[*].*?[*]/)m, Comment::Multiline @@ -96,13 +98,22 @@ def self.cpp_keywords )ix, Num rule %r/\b(?:class|record|struct|interface)\b/, Keyword, :class rule %r/\b(?:namespace|using)\b/, Keyword, :namespace - rule %r/^#[ \t]*(#{CSharp.cpp_keywords.join('|')})\b.*?\n/, Comment::Preproc - rule %r/\b(#{CSharp.keywords.join('|')})\b/, Keyword - rule %r/\b(#{CSharp.keywords_type.join('|')})\b/, Keyword::Type + + keywords %r/\w+/ do + rule :keywords, Keyword + rule :keywords_type, Keyword::Type + end + rule %r/#{id}(?=\s*[(])/, Name::Function rule id, Name end + state :preproc do + rule %r/[^\\\n]+/, Comment::Preproc + rule %r/\\.?/m, Comment::Preproc + rule %r/\n/, Comment::Preproc, :pop! + end + state :class do mixin :whitespace rule id, Name::Class, :pop! diff --git a/spec/visual/samples/csharp b/spec/visual/samples/csharp index 09c1b24143..9f2a9475a9 100644 --- a/spec/visual/samples/csharp +++ b/spec/visual/samples/csharp @@ -7,6 +7,12 @@ class Foo { // stuff } +#ifdef FOO + #ifdef BAR + #endif +#endif + + //////////////////////////////////////////////////////////////////////////////// // // // MIT X11 license, Copyright (c) 2005-2006 by: // From 16f26dae808ba016280735dfac1f9984b354f233 Mon Sep 17 00:00:00 2001 From: jneen Date: Thu, 9 Apr 2026 15:14:23 -0400 Subject: [PATCH 033/154] use the keywords api for CSS --- .rubocop_todo.yml | 1 - lib/rouge/lexers/css.rb | 232 +++---------------------------- lib/rouge/lexers/css/builtins.rb | 195 ++++++++++++++++++++++++++ lib/rouge/lexers/sass/common.rb | 6 +- 4 files changed, 215 insertions(+), 219 deletions(-) create mode 100644 lib/rouge/lexers/css/builtins.rb diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index f572369838..f8659f0215 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -84,7 +84,6 @@ Rouge/NoBuildingAlternationPatternInRegexp: Rouge/NoHugeCollections: Exclude: - 'lib/rouge/lexers/common_lisp.rb' - - 'lib/rouge/lexers/css.rb' - 'lib/rouge/lexers/datastudio.rb' - 'lib/rouge/lexers/freefem.rb' - 'lib/rouge/lexers/hql.rb' diff --git a/lib/rouge/lexers/css.rb b/lib/rouge/lexers/css.rb index 67b8ba0393..ec3cd6f64d 100644 --- a/lib/rouge/lexers/css.rb +++ b/lib/rouge/lexers/css.rb @@ -11,6 +11,8 @@ class CSS < RegexLexer filenames '*.css' mimetypes 'text/css' + lazy { require_relative 'css/builtins' } + # Documentation: https://www.w3.org/TR/CSS21/syndata.html#characters # [jneen] workaround for: @@ -28,211 +30,11 @@ class CSS < RegexLexer number = /-?(?:[0-9]+(\.[0-9]+)?|\.[0-9]+)/ - def self.properties - @properties ||= Set.new %w( - additive-symbols align-content align-items align-self - alignment-adjust alignment-baseline all anchor-point animation - animation-composition animation-delay animation-direction - animation-duration animation-fill-mode animation-iteration-count - animation-name animation-play-state animation-timing-function - appearance aspect-ratio azimuth backface-visibility background - background-attachment background-blend-mode - background-clip background-color background-image - background-origin background-position - background-repeat background-size baseline-shift - binding bleed bookmark-label bookmark-level bookmark-state - bookmark-target border border-bottom border-bottom-color - border-bottom-left-radius border-bottom-right-radius - border-bottom-style border-bottom-width border-collapse - border-color border-image border-image-outset border-image-repeat - border-image-slice border-image-source border-image-width - border-left border-left-color border-left-style border-left-width - border-radius border-right border-right-color border-right-style - border-right-width border-spacing border-style border-top - border-top-color border-top-left-radius border-top-right-radius - border-top-style border-top-width border-width bottom box-align - box-decoration-break box-direction box-flex box-flex-group - box-lines box-ordinal-group box-orient box-pack box-shadow - box-sizing break-after break-before break-inside caption-side - clear clip clip-path clip-rule color color-profile column-count - column-fill column-gap column-rule column-rule-color - column-rule-style column-rule-width column-span column-width - columns content container container-name container-type - counter-increment counter-reset counter-set crop cue - cue-after cue-before cursor direction display dominant-baseline - drop-initial-after-adjust drop-initial-after-align - drop-initial-before-adjust drop-initial-before-align - drop-initial-size drop-initial-value elevation empty-cells fallback - filter fit fit-position flex flex-basis flex-direction flex-flow - flex-grow flex-shrink flex-wrap float float-offset font - font-display font-family font-feature-settings font-kerning - font-language-override font-size font-size-adjust font-stretch - font-style font-synthesis font-variant font-variant-alternates - font-variant-caps font-variant-east-asian font-variant-ligatures - font-variant-numeric font-variant-position font-weight gap - grid-area grid-auto-columns grid-auto-flow grid-auto-rows - grid-column grid-column-end grid-column-start grid-row - grid-row-end grid-row-start grid-template grid-template-areas - grid-template-columns grid-template-rows hanging-punctuation - height hyphenate-after hyphenate-before hyphenate-character - hyphenate-lines hyphenate-resource hyphens icon image-orientation - image-rendering image-resolution ime-mode inherits initial-value - inline-box-align inset isolation justify-content justify-items - justify-self left letter-spacing line-break line-height - line-stacking line-stacking-ruby line-stacking-shift - line-stacking-strategy list-style list-style-image - list-style-position list-style-type margin margin-bottom - margin-left margin-right margin-top mark mark-after mark-before - marker-offset marks marquee-direction marquee-loop - marquee-play-count marquee-speed marquee-style - mask mask-clip mask-composite mask-image mask-mode - mask-origin mask-position mask-repeat mask-size mask-type - max-height max-width min-height min-width mix-blend-mode - move-to nav-down nav-index nav-left nav-right nav-up negative - object-fit object-position offset offset-anchor offset-distance - offset-path offset-position offset-rotate opacity order orphans - outline outline-color outline-offset outline-style outline-width - overflow overflow-style overflow-wrap overflow-x overflow-y pad - padding padding-bottom padding-left padding-right padding-top page - page-break-after page-break-before page-break-inside page-policy - pause pause-after pause-before perspective perspective-origin - phonemes pitch pitch-range place-content place-items place-self - play-during pointer-events position prefix presentation-level - punctuation-trim quotes range rendering-intent resize rest - rest-after rest-before richness right rotate rotation rotation-point - row-gap ruby-align ruby-overhang ruby-position ruby-span scale - scroll-behavior scroll-margin scroll-margin-block - scroll-margin-block-end scroll-margin-block-start - scroll-margin-bottom scroll-margin-inline scroll-margin-inline-end - scroll-margin-inline-start scroll-margin-left scroll-margin-right - scroll-margin-top scroll-padding-top scroll-padding-right - scroll-padding-bottom scroll-padding-left scroll-padding - scroll-padding-block-end scroll-padding-block-start - scroll-padding-block scroll-padding-inline-end - scroll-padding-inline-start scroll-padding-inline - scroll-snap-type scroll-snap-align scroll-snap-stop - shape-outside shape-margin shape-image-threshold shape-rendering - size speak speak-as speak-header speak-numeral speak-punctuation - speech-rate src stress string-set suffix symbols syntax system - tab-size table-layout target target-name target-new target-position - text-align text-align-last text-combine-horizontal - text-decoration text-decoration-color text-decoration-line - text-decoration-skip text-decoration-style text-emphasis - text-emphasis-color text-emphasis-position text-emphasis-style - text-height text-indent text-justify text-orientation - text-outline text-overflow text-rendering text-shadow - text-space-collapse text-transform text-underline-position - text-wrap top transform transform-origin transform-style - transition transition-delay transition-duration - transition-property transition-timing-function translate - unicode-bidi vertical-align visibility voice-balance - voice-duration voice-family voice-pitch voice-pitch-range - voice-range voice-rate voice-stress voice-volume volume - white-space widows width word-break word-spacing word-wrap - writing-mode z-index - ) - end - - def self.builtins - @builtins ||= Set.new %w( - above absolute accumulate add additive all alpha alphabetic - alternate alternate-reverse always armenian aural auto auto-fill - auto-fit avoid backwards balance baseline behind below bidi-override - blink block bold bolder border-box both bottom break-spaces - capitalize center center-left center-right circle cjk-ideographic - close-quote closest-corner closest-side collapse - color color-burn color-dodge column column-reverse - condensed contain content content-box continuous cover crop cross - crosshair cursive cyclic darken dashed decimal decimal-leading-zero - default difference digits disc dotted double e-resize - ease ease-in ease-in-out ease-out embed end exclude exclusion - expanded extends extra-condensed extra-expanded fantasy - farthest-corner farthest-side far-left far-right - fast faster fill fixed flat flex flex-end flex-start - forwards georgian grid groove hard-light hebrew help hidden - hide high higher hiragana hiragana-iroha horizontal hue icon - infinite inherit inline inline-block inline-flex inline-grid - inline-size inline-table inset inside intersect isolate italic - justify katakana katakana-iroha landscape large larger left - left-side leftwards level lighten lighter line-through linear - list-item loud low lower lower-alpha lower-greek lower-roman - lowercase ltr luminance luminosity mandatory match-source medium - message-box middle mix monospace multiply n-resize narrower - ne-resize no-close-quote no-open-quote no-repeat none normal - nowrap numeric nw-resize oblique once open-quote outset outside - overlay overline paused pointer portrait pre preserve-3d pre-line - pre-wrap proximity px relative repeat-x repeat-y replace reverse - ridge right right-side rightwards row row-reverse rtl running - s-resize sans-serif saturation scale-down screen scroll - se-resize semi-condensed semi-expanded separate serif show - sides silent size slow slower small-caps small-caption smaller - smooth soft soft-light solid space-around space-between - space-evenly span spell-out square start static status-bar sticky - stretch sub subtract super sw-resize swap symbolic table - table-caption table-cell table-column table-column-group - table-footer-group table-header-group table-row table-row-group - text text-bottom text-top thick thin top transparent - ultra-condensed ultra-expanded underline upper-alpha upper-latin - upper-roman uppercase vertical visible w-resize wait wider wrap - wrap-reverse x x-fast x-high x-large x-loud x-low x-small x-soft - xx-large xx-small yes y z - ) - end - - def self.colors - @colors ||= Set.new %w( - aliceblue antiquewhite aqua aquamarine azure beige bisque black - blanchedalmond blue blueviolet brown burlywood cadetblue - chartreuse chocolate coral cornflowerblue cornsilk crimson cyan - darkblue darkcyan darkgoldenrod darkgray darkgreen darkkhaki - darkmagenta darkolivegreen darkorange darkorchid darkred - darksalmon darkseagreen darkslateblue darkslategray darkturquoise - darkviolet deeppink deepskyblue dimgray dodgerblue firebrick - floralwhite forestgreen fuchsia gainsboro ghostwhite gold - goldenrod gray green greenyellow honeydew hotpink indianred - indigo ivory khaki lavender lavenderblush lawngreen lemonchiffon - lightblue lightcoral lightcyan lightgoldenrodyellow lightgreen - lightgrey lightpink lightsalmon lightseagreen lightskyblue - lightslategray lightsteelblue lightyellow lime limegreen linen - magenta maroon mediumaquamarine mediumblue mediumorchid - mediumpurple mediumseagreen mediumslateblue mediumspringgreen - mediumturquoise mediumvioletred midnightblue mintcream mistyrose - moccasin navajowhite navy oldlace olive olivedrab orange - orangered orchid palegoldenrod palegreen paleturquoise - palevioletred papayawhip peachpuff peru pink plum powderblue - purple red rosybrown royalblue saddlebrown salmon sandybrown - seagreen seashell sienna silver skyblue slateblue slategray snow - springgreen steelblue tan teal thistle tomato - turquoise violet wheat white whitesmoke yellow yellowgreen - rebeccapurple - ) - end - - def self.functions - @functions ||= Set.new %w( - abs acos annotation asin atan atan2 attr blur brightness calc - character-variant circle clamp color color-mix conic-gradient - contrast cos counter counters cubic-bezier drop-shadow ellipse - env exp fit-content format grayscale hsl hsla hue-rotate hwb hypot - image-set inset invert lab lch linear linear-gradient log matrix - matrix3d max min minmax mod oklab oklch opacity ornaments path - perspective polygon pow radial-gradient ray rect rem repeat - repeating-conic-gradient repeating-linear-gradient - repeating-radial-gradient rgb rgba rotate rotate3d rotatex - rotatey rotatez round saturate scale scale3d scalex scaley scalez - sepia sign sin skewx skewy sqrt steps styleset - stylistic swash tan translate translate3d translatex translatey - translatez url var xywh - ) - end - # source: http://www.w3.org/TR/CSS21/syndata.html#vendor-keyword-history - def self.vendor_prefixes - @vendor_prefixes ||= Set.new %w( - -ah- -atsc- -hp- -khtml- -moz- -ms- -o- -rim- -ro- -tc- -wap- - -webkit- -xv- mso- prince- - ) - end + VENDOR_PREFIXES = %w( + -ah- -atsc- -hp- -khtml- -moz- -ms- -o- -rim- -ro- -tc- -wap- + -webkit- -xv- mso- prince- + ) state :root do mixin :basics @@ -262,16 +64,13 @@ def self.vendor_prefixes groups Name::Function, Punctuation, Str::Other, Punctuation end - rule(identifier) do |m| - if self.class.colors.include? m[0].downcase - token Name::Other - elsif self.class.builtins.include? m[0].downcase - token Name::Builtin - elsif self.class.functions.include? m[0].downcase - token Name::Function - else - token Name - end + keywords identifier do + transform(&:downcase) + + rule COLORS, Name::Other + rule BUILTINS, Name::Builtin + rule FUNCTIONS, Name::Function + default Name end end @@ -308,10 +107,11 @@ def self.vendor_prefixes state :stanza do mixin :basics rule %r/}/, Punctuation, :pop! + rule %r/(#{identifier})(\s*)(:)/m do |m| - name_tok = if self.class.properties.include? m[1] + name_tok = if PROPERTIES.include? m[1] Name::Label - elsif self.class.vendor_prefixes.any? { |p| m[1].start_with?(p) } + elsif VENDOR_PREFIXES.any? { |p| m[1].start_with?(p) } Name::Label else Name::Property diff --git a/lib/rouge/lexers/css/builtins.rb b/lib/rouge/lexers/css/builtins.rb new file mode 100644 index 0000000000..a8dc6da713 --- /dev/null +++ b/lib/rouge/lexers/css/builtins.rb @@ -0,0 +1,195 @@ +module Rouge + module Lexers + class CSS < RegexLexer + PROPERTIES = Set.new %w( + additive-symbols align-content align-items align-self + alignment-adjust alignment-baseline all anchor-point animation + animation-composition animation-delay animation-direction + animation-duration animation-fill-mode animation-iteration-count + animation-name animation-play-state animation-timing-function + appearance aspect-ratio azimuth backface-visibility background + background-attachment background-blend-mode + background-clip background-color background-image + background-origin background-position + background-repeat background-size baseline-shift + binding bleed bookmark-label bookmark-level bookmark-state + bookmark-target border border-bottom border-bottom-color + border-bottom-left-radius border-bottom-right-radius + border-bottom-style border-bottom-width border-collapse + border-color border-image border-image-outset border-image-repeat + border-image-slice border-image-source border-image-width + border-left border-left-color border-left-style border-left-width + border-radius border-right border-right-color border-right-style + border-right-width border-spacing border-style border-top + border-top-color border-top-left-radius border-top-right-radius + border-top-style border-top-width border-width bottom box-align + box-decoration-break box-direction box-flex box-flex-group + box-lines box-ordinal-group box-orient box-pack box-shadow + box-sizing break-after break-before break-inside caption-side + clear clip clip-path clip-rule color color-profile column-count + column-fill column-gap column-rule column-rule-color + column-rule-style column-rule-width column-span column-width + columns content container container-name container-type + counter-increment counter-reset counter-set crop cue + cue-after cue-before cursor direction display dominant-baseline + drop-initial-after-adjust drop-initial-after-align + drop-initial-before-adjust drop-initial-before-align + drop-initial-size drop-initial-value elevation empty-cells fallback + filter fit fit-position flex flex-basis flex-direction flex-flow + flex-grow flex-shrink flex-wrap float float-offset font + font-display font-family font-feature-settings font-kerning + font-language-override font-size font-size-adjust font-stretch + font-style font-synthesis font-variant font-variant-alternates + font-variant-caps font-variant-east-asian font-variant-ligatures + font-variant-numeric font-variant-position font-weight gap + grid-area grid-auto-columns grid-auto-flow grid-auto-rows + grid-column grid-column-end grid-column-start grid-row + grid-row-end grid-row-start grid-template grid-template-areas + grid-template-columns grid-template-rows hanging-punctuation + height hyphenate-after hyphenate-before hyphenate-character + hyphenate-lines hyphenate-resource hyphens icon image-orientation + image-rendering image-resolution ime-mode inherits initial-value + inline-box-align inset isolation justify-content justify-items + justify-self left letter-spacing line-break line-height + line-stacking line-stacking-ruby line-stacking-shift + line-stacking-strategy list-style list-style-image + list-style-position list-style-type margin margin-bottom + margin-left margin-right margin-top mark mark-after mark-before + marker-offset marks marquee-direction marquee-loop + marquee-play-count marquee-speed marquee-style + mask mask-clip mask-composite mask-image mask-mode + mask-origin mask-position mask-repeat mask-size mask-type + max-height max-width min-height min-width mix-blend-mode + move-to nav-down nav-index nav-left nav-right nav-up negative + object-fit object-position offset offset-anchor offset-distance + offset-path offset-position offset-rotate opacity order orphans + outline outline-color outline-offset outline-style outline-width + overflow overflow-style overflow-wrap overflow-x overflow-y pad + padding padding-bottom padding-left padding-right padding-top page + page-break-after page-break-before page-break-inside page-policy + pause pause-after pause-before perspective perspective-origin + phonemes pitch pitch-range place-content place-items place-self + play-during pointer-events position prefix presentation-level + punctuation-trim quotes range rendering-intent resize rest + rest-after rest-before richness right rotate rotation rotation-point + row-gap ruby-align ruby-overhang ruby-position ruby-span scale + scroll-behavior scroll-margin scroll-margin-block + scroll-margin-block-end scroll-margin-block-start + scroll-margin-bottom scroll-margin-inline scroll-margin-inline-end + scroll-margin-inline-start scroll-margin-left scroll-margin-right + scroll-margin-top scroll-padding-top scroll-padding-right + scroll-padding-bottom scroll-padding-left scroll-padding + scroll-padding-block-end scroll-padding-block-start + scroll-padding-block scroll-padding-inline-end + scroll-padding-inline-start scroll-padding-inline + scroll-snap-type scroll-snap-align scroll-snap-stop + shape-outside shape-margin shape-image-threshold shape-rendering + size speak speak-as speak-header speak-numeral speak-punctuation + speech-rate src stress string-set suffix symbols syntax system + tab-size table-layout target target-name target-new target-position + text-align text-align-last text-combine-horizontal + text-decoration text-decoration-color text-decoration-line + text-decoration-skip text-decoration-style text-emphasis + text-emphasis-color text-emphasis-position text-emphasis-style + text-height text-indent text-justify text-orientation + text-outline text-overflow text-rendering text-shadow + text-space-collapse text-transform text-underline-position + text-wrap top transform transform-origin transform-style + transition transition-delay transition-duration + transition-property transition-timing-function translate + unicode-bidi vertical-align visibility voice-balance + voice-duration voice-family voice-pitch voice-pitch-range + voice-range voice-rate voice-stress voice-volume volume + white-space widows width word-break word-spacing word-wrap + writing-mode z-index + ) + + BUILTINS = Set.new %w( + above absolute accumulate add additive all alpha alphabetic + alternate alternate-reverse always armenian aural auto auto-fill + auto-fit avoid backwards balance baseline behind below bidi-override + blink block bold bolder border-box both bottom break-spaces + capitalize center center-left center-right circle cjk-ideographic + close-quote closest-corner closest-side collapse + color color-burn color-dodge column column-reverse + condensed contain content content-box continuous cover crop cross + crosshair cursive cyclic darken dashed decimal decimal-leading-zero + default difference digits disc dotted double e-resize + ease ease-in ease-in-out ease-out embed end exclude exclusion + expanded extends extra-condensed extra-expanded fantasy + farthest-corner farthest-side far-left far-right + fast faster fill fixed flat flex flex-end flex-start + forwards georgian grid groove hard-light hebrew help hidden + hide high higher hiragana hiragana-iroha horizontal hue icon + infinite inherit inline inline-block inline-flex inline-grid + inline-size inline-table inset inside intersect isolate italic + justify katakana katakana-iroha landscape large larger left + left-side leftwards level lighten lighter line-through linear + list-item loud low lower lower-alpha lower-greek lower-roman + lowercase ltr luminance luminosity mandatory match-source medium + message-box middle mix monospace multiply n-resize narrower + ne-resize no-close-quote no-open-quote no-repeat none normal + nowrap numeric nw-resize oblique once open-quote outset outside + overlay overline paused pointer portrait pre preserve-3d pre-line + pre-wrap proximity px relative repeat-x repeat-y replace reverse + ridge right right-side rightwards row row-reverse rtl running + s-resize sans-serif saturation scale-down screen scroll + se-resize semi-condensed semi-expanded separate serif show + sides silent size slow slower small-caps small-caption smaller + smooth soft soft-light solid space-around space-between + space-evenly span spell-out square start static status-bar sticky + stretch sub subtract super sw-resize swap symbolic table + table-caption table-cell table-column table-column-group + table-footer-group table-header-group table-row table-row-group + text text-bottom text-top thick thin top transparent + ultra-condensed ultra-expanded underline upper-alpha upper-latin + upper-roman uppercase vertical visible w-resize wait wider wrap + wrap-reverse x x-fast x-high x-large x-loud x-low x-small x-soft + xx-large xx-small yes y z + ) + + COLORS = Set.new %w( + aliceblue antiquewhite aqua aquamarine azure beige bisque black + blanchedalmond blue blueviolet brown burlywood cadetblue + chartreuse chocolate coral cornflowerblue cornsilk crimson cyan + darkblue darkcyan darkgoldenrod darkgray darkgreen darkkhaki + darkmagenta darkolivegreen darkorange darkorchid darkred + darksalmon darkseagreen darkslateblue darkslategray darkturquoise + darkviolet deeppink deepskyblue dimgray dodgerblue firebrick + floralwhite forestgreen fuchsia gainsboro ghostwhite gold + goldenrod gray green greenyellow honeydew hotpink indianred + indigo ivory khaki lavender lavenderblush lawngreen lemonchiffon + lightblue lightcoral lightcyan lightgoldenrodyellow lightgreen + lightgrey lightpink lightsalmon lightseagreen lightskyblue + lightslategray lightsteelblue lightyellow lime limegreen linen + magenta maroon mediumaquamarine mediumblue mediumorchid + mediumpurple mediumseagreen mediumslateblue mediumspringgreen + mediumturquoise mediumvioletred midnightblue mintcream mistyrose + moccasin navajowhite navy oldlace olive olivedrab orange + orangered orchid palegoldenrod palegreen paleturquoise + palevioletred papayawhip peachpuff peru pink plum powderblue + purple red rosybrown royalblue saddlebrown salmon sandybrown + seagreen seashell sienna silver skyblue slateblue slategray snow + springgreen steelblue tan teal thistle tomato + turquoise violet wheat white whitesmoke yellow yellowgreen + rebeccapurple + ) + + FUNCTIONS = Set.new %w( + abs acos annotation asin atan atan2 attr blur brightness calc + character-variant circle clamp color color-mix conic-gradient + contrast cos counter counters cubic-bezier drop-shadow ellipse + env exp fit-content format grayscale hsl hsla hue-rotate hwb hypot + image-set inset invert lab lch linear linear-gradient log matrix + matrix3d max min minmax mod oklab oklch opacity ornaments path + perspective polygon pow radial-gradient ray rect rem repeat + repeating-conic-gradient repeating-linear-gradient + repeating-radial-gradient rgb rgba rotate rotate3d rotatex + rotatey rotatez round saturate scale scale3d scalex scaley scalez + sepia sign sin skewx skewy sqrt steps styleset + stylistic swash tan translate translate3d translatex translatey + translatez url var xywh + ) + end + end +end diff --git a/lib/rouge/lexers/sass/common.rb b/lib/rouge/lexers/sass/common.rb index 04c40947b3..317557f465 100644 --- a/lib/rouge/lexers/sass/common.rb +++ b/lib/rouge/lexers/sass/common.rb @@ -5,6 +5,8 @@ module Rouge module Lexers # shared states with SCSS class SassCommon < RegexLexer + lazy { require_relative '../css/builtins' } + id = /[\w-]+/ state :content_common do @@ -65,9 +67,9 @@ class SassCommon < RegexLexer # identifiers rule(id) do |m| - if CSS.builtins.include? m[0] + if CSS::BUILTINS.include? m[0] token Name::Builtin - elsif CSS.colors.include? m[0] + elsif CSS::COLORS.include? m[0] token Name::Constant else token Name From ac1cb43269e580d1576b06a5b1ed37405ec1adde Mon Sep 17 00:00:00 2001 From: jneen Date: Thu, 9 Apr 2026 15:21:33 -0400 Subject: [PATCH 034/154] remove common_lisp from rubocop_todo --- .rubocop_todo.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index f8659f0215..8109e5673b 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -83,7 +83,6 @@ Rouge/NoBuildingAlternationPatternInRegexp: # Offense count: 20 Rouge/NoHugeCollections: Exclude: - - 'lib/rouge/lexers/common_lisp.rb' - 'lib/rouge/lexers/datastudio.rb' - 'lib/rouge/lexers/freefem.rb' - 'lib/rouge/lexers/hql.rb' From 2623df4d4ad1b85f12dec975fcaeaf97cf5ddc58 Mon Sep 17 00:00:00 2001 From: jneen Date: Thu, 9 Apr 2026 15:21:48 -0400 Subject: [PATCH 035/154] use the keywords API for cypher, and use Punctuation not Str::Symbol --- lib/rouge/lexers/cypher.rb | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/lib/rouge/lexers/cypher.rb b/lib/rouge/lexers/cypher.rb index 2bc90f261c..160d4fa5f9 100644 --- a/lib/rouge/lexers/cypher.rb +++ b/lib/rouge/lexers/cypher.rb @@ -48,7 +48,7 @@ def self.keywords rule %r(/\*), Comment::Multiline, :multiline_comments rule %r([*+\-<>=&|~%^]), Operator - rule %r/[{}),;\[\]]/, Str::Symbol + rule %r/[{}),;\[\]]/, Punctuation # literal number rule %r/(\w+)(:)(\s*)(-?[._\d]+)/ do @@ -59,15 +59,13 @@ def self.keywords # - "name(" # - "name (" # - "name (" - rule %r/(\w+)(\s*)(\()/ do |m| - name = m[1].upcase - if self.class.functions.include? name - groups Name::Function, Text::Whitespace, Str::Symbol - elsif self.class.keywords.include? name - groups Keyword, Text::Whitespace, Str::Symbol - else - groups Name, Text::Whitespace, Str::Symbol - end + keywords %r/(\w+)(?=\s*[(])/ do |m| + group 1 + transform(&:upcase) + + rule :functions, Name::Function + rule :keywords, Keyword + default Name end rule %r/:\w+/, Name::Class @@ -87,7 +85,7 @@ def self.keywords rule %r([.\w]+:), Name::Property # remaining "(" - rule %r/\(/, Str::Symbol + rule %r/\(/, Punctuation rule %r/[.\w$]+/ do |m| match = m[0].upcase From c16bcaee6869aa67eafc327480d7f614954fbd91 Mon Sep 17 00:00:00 2001 From: jneen Date: Fri, 10 Apr 2026 21:12:45 -0400 Subject: [PATCH 036/154] fix sasscommon --- lib/rouge/lexers/sass/common.rb | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/lib/rouge/lexers/sass/common.rb b/lib/rouge/lexers/sass/common.rb index 317557f465..6cb75b248c 100644 --- a/lib/rouge/lexers/sass/common.rb +++ b/lib/rouge/lexers/sass/common.rb @@ -156,12 +156,9 @@ class SassCommon < RegexLexer state :attr_common do mixin :has_interp - rule id do |m| - if CSS.properties.include? m[0] - token Name::Label - else - token Name::Attribute - end + keywords id do + rule CSS::PROPERTIES, Name::Label + default Name::Attribute end end From a60c807ff9d5cb09f319abfd4ddf4bf55882f49c Mon Sep 17 00:00:00 2001 From: jneen Date: Fri, 10 Apr 2026 21:12:58 -0400 Subject: [PATCH 037/154] use the keywords api for cython --- lib/rouge/lexers/cython.rb | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/lib/rouge/lexers/cython.rb b/lib/rouge/lexers/cython.rb index b4206b4e43..f2ac8131b9 100644 --- a/lib/rouge/lexers/cython.rb +++ b/lib/rouge/lexers/cython.rb @@ -74,22 +74,12 @@ def self.builtins # This rule matches identifiers that could be type declarations. The # lookahead matches (1) pointers, (2) arrays and (3) variable names. - rule %r/#{identifier}(?=(?:\*+)|(?:[ \t]*\[)|(?:[ \t]+\w))/ do |m| - if self.class.keywords.include?(m[0]) - token Keyword - pop! - elsif m[0] == 'def' - token Keyword - goto :funcname - elsif %w(struct class).include?(m[0]) - token Keyword - goto :classname - elsif self.class.c_keywords.include?(m[0]) - token Keyword::Reserved - else - token Keyword::Type - pop! - end + keywords %r/#{identifier}(?=(?:\*+)|(?:[ \t]*\[)|(?:[ \t]+\w))/ do |m| + rule :keywords, Keyword, :pop! + rule(Set['def']) { token Keyword; goto :funcname } + rule(Set['struct', 'class']) { token Keyword::Reserved; goto :classname } + rule :c_keywords, Keyword::Reserved + default Keyword::Type, :pop! end rule(//) { pop! } From c743cec612c16215b24430ac94c7c2c27b98c570 Mon Sep 17 00:00:00 2001 From: jneen Date: Fri, 10 Apr 2026 23:44:45 -0400 Subject: [PATCH 038/154] use the keywords API for dlang --- .rubocop_todo.yml | 1 - lib/rouge/lexers/d.rb | 27 ++++++++++++++++----------- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index 8109e5673b..d7a6f73c84 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -50,7 +50,6 @@ Naming/VariableName: Rouge/NoBuildingAlternationPatternInRegexp: Exclude: - 'lib/rouge/lexers/armasm.rb' - - 'lib/rouge/lexers/d.rb' - 'lib/rouge/lexers/dart.rb' - 'lib/rouge/lexers/eiffel.rb' - 'lib/rouge/lexers/elm.rb' diff --git a/lib/rouge/lexers/d.rb b/lib/rouge/lexers/d.rb index de52e30a27..6a15c30b99 100644 --- a/lib/rouge/lexers/d.rb +++ b/lib/rouge/lexers/d.rb @@ -12,7 +12,7 @@ class D < RegexLexer title "D" desc 'The D programming language(dlang.org)' - keywords = %w( + KEYWORDS = Set.new %w( abstract alias align asm assert auto body break case cast catch class const continue debug default delegate delete deprecated do else @@ -27,14 +27,14 @@ module new nothrow out override package pragma __gshared __traits __vector __parameters ) - keywords_type = %w( + KEYWORDS_TYPE = Set.new %w( bool byte cdouble cent cfloat char creal dchar double float idouble ifloat int ireal long real short ubyte ucent uint ulong ushort void wchar ) - keywords_pseudo = %w( + KEYWORDS_PSEUDO = Set.new %w( __FILE__ __FILE_FULL_PATH__ __MODULE__ __LINE__ __FUNCTION__ __PRETTY_FUNCTION__ __DATE__ __EOF__ __TIME__ __TIMESTAMP__ __VENDOR__ __VERSION__ @@ -51,13 +51,7 @@ module new nothrow out override package pragma rule %r(//.*), Comment::Single rule %r(/(\\\n)?[*](.|\n)*?[*](\\\n)?/), Comment::Multiline rule %r(/\+), Comment::Multiline, :nested_comment - # Keywords - rule %r/(#{keywords.join('|')})\b/, Keyword - rule %r/(#{keywords_type.join('|')})\b/, Keyword::Type - rule %r/(false|true|null)\b/, Keyword::Constant - rule %r/(#{keywords_pseudo.join('|')})\b/, Keyword::Pseudo - rule %r/macro\b/, Keyword::Reserved - rule %r/(string|wstring|dstring|size_t|ptrdiff_t)\b/, Name::Builtin + # Literals # HexFloat rule %r/0[xX]([0-9a-fA-F_]*\.[0-9a-fA-F_]+|[0-9a-fA-F_]+)[pP][+\-]?[0-9_]+[fFL]?[i]?/, Num::Float @@ -100,8 +94,19 @@ module new nothrow out override package pragma rule %r/@([a-zA-Z_]\w*)?/, Name::Decorator # Tokens rule %r`(~=|\^=|%=|\*=|==|!>=|!<=|!<>=|!<>|!<|!>|!=|>>>=|>>>|>>=|>>|>=|<>=|<>|<<=|<<|<=|\+\+|\+=|--|-=|\|\||\|=|&&|&=|\.\.\.|\.\.|/=)|[/.&|\-+<>!()\[\]{}?,;:$=*%^~]`, Punctuation + # Identifier - rule %r/[a-zA-Z_]\w*/, Name + # Keywords, identifiers + keywords %r/[a-zA-Z_]\w*/ do + rule KEYWORDS, Keyword + rule KEYWORDS_TYPE, Keyword::Type + rule KEYWORDS_PSEUDO, Keyword::Pseudo + rule Set['false', 'true', 'nil'], Keyword::Constant + rule Set['macro'], Keyword::Reserved + rule Set['string', 'wstring', 'dstring', 'size_t', 'ptrdiff_t'], Name::Builtin + default Name + end + # Line rule %r/#line\s.*\n/, Comment::Special end From b0e15e84ee21a0645bd5a71190d8491ea7bba117 Mon Sep 17 00:00:00 2001 From: jneen Date: Fri, 10 Apr 2026 23:49:32 -0400 Subject: [PATCH 039/154] use keywords api for dafny, and add some keywords from the spec --- lib/rouge/lexers/dafny.rb | 38 +++++++++++++++++--------------------- 1 file changed, 17 insertions(+), 21 deletions(-) diff --git a/lib/rouge/lexers/dafny.rb b/lib/rouge/lexers/dafny.rb index 21317f472c..4e7cbbc023 100644 --- a/lib/rouge/lexers/dafny.rb +++ b/lib/rouge/lexers/dafny.rb @@ -10,15 +10,15 @@ class Dafny < RegexLexer filenames "*.dfy" mimetypes "text/x-dafny" - keywords = %w( + KEYWORDS = Set.new %w( abstract allocated assert assume break by calc case class codatatype const constructor - datatype decreases downto + datatype decreases default downto else ensures exists expect export extends false for forall fresh function ghost greatest - if import include invariant iterator + if import include inductive invariant iterator label least lemma match method modifies modify module nameonly new newtype null @@ -29,17 +29,19 @@ class Dafny < RegexLexer then this to trait true twostate type unchanged var - while witness + where while witness yield yields ) - literals = %w{ true false null } + LITERALS = Set.new %w{ true false null } - textOperators = %w{ as is in } + TEXT_OPERATORS = Set.new %w{ as is in } - types = %w(bool char int real string nat - array array? object object? ORDINAL - seq set iset map imap multiset ) + TYPES = Set.new %w( + bool char int real string nat + array array? object object? ORDINAL + seq set iset map imap multiset + ) idstart = /[0-9a-zA-Z?]/ idchar = /[0-9a-zA-Z_'?]/ @@ -93,18 +95,12 @@ class Dafny < RegexLexer rule %r/#{arrayType}/, Keyword::Type rule %r/#{bvType}/, Keyword::Type - rule id do |m| - if types.include?(m[0]) - token Keyword::Type - elsif literals.include?(m[0]) - token Keyword::Constant - elsif textOperators.include?(m[0]) - token Operator::Word - elsif keywords.include?(m[0]) - token Keyword::Reserved - else - token Name - end + keywords id do + rule TYPES, Keyword::Type + rule LITERALS, Keyword::Constant + rule TEXT_OPERATORS, Operator::Word + rule KEYWORDS, Keyword::Reserved + default Name end rule %r/\.\./, Operator From be85ef226bb2de1a75ec0b3bf47967a11ab9a3b4 Mon Sep 17 00:00:00 2001 From: jneen Date: Sun, 12 Apr 2026 18:19:47 -0400 Subject: [PATCH 040/154] use the keywords api for dart --- .rubocop_todo.yml | 1 - lib/rouge/lexers/dart.rb | 34 ++++++++++++++++++++++------------ 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index d7a6f73c84..82ab1d7688 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -50,7 +50,6 @@ Naming/VariableName: Rouge/NoBuildingAlternationPatternInRegexp: Exclude: - 'lib/rouge/lexers/armasm.rb' - - 'lib/rouge/lexers/dart.rb' - 'lib/rouge/lexers/eiffel.rb' - 'lib/rouge/lexers/elm.rb' - 'lib/rouge/lexers/erlang.rb' diff --git a/lib/rouge/lexers/dart.rb b/lib/rouge/lexers/dart.rb index 7dcffc474a..286ec220d5 100644 --- a/lib/rouge/lexers/dart.rb +++ b/lib/rouge/lexers/dart.rb @@ -11,19 +11,24 @@ class Dart < RegexLexer filenames '*.dart' mimetypes 'text/x-dart' - keywords = %w( + KEYWORDS = Set.new %w( as assert await break case catch continue default do else finally for if in is new rethrow return super switch this throw try while when with yield ) - declarations = %w( - abstract base async dynamic const covariant external extends factory final get implements - interface late native on operator required sealed set static sync typedef var + DECLARATIONS = Set.new %w( + abstract base async dynamic const covariant external extends + factory final get implements interface late native on + operator required sealed set static sync typedef var ) - types = %w(bool Comparable double Dynamic enum Function int List Map Never Null num Object Pattern Record Set String Symbol Type Uri void) + TYPES = Set.new %w( + bool Comparable double Dynamic enum Function int List Map + Never Null num Object Pattern Record Set String Symbol Type + Uri void + ) - imports = %w(import deferred export library part\s*of part source) + IMPORTS = Set.new %w(import deferred export library part source) id = /[a-zA-Z_]\w*/ @@ -51,12 +56,17 @@ class Dart < RegexLexer rule %r/r'[^']*'/, Str::Other rule %r/##{id}*/i, Str::Symbol rule %r/@#{id}/, Name::Decorator - rule %r/(?:#{keywords.join('|')})\b/, Keyword - rule %r/(?:#{declarations.join('|')})\b/, Keyword::Declaration - rule %r/(?:#{types.join('|')})\b/, Keyword::Type - rule %r/(?:true|false|null)\b/, Keyword::Constant - rule %r/(?:class|mixin)\b/, Keyword::Declaration, :class - rule %r/(?:#{imports.join('|')})\b/, Keyword::Namespace, :import + rule %r/part\s*of\b/, Keyword::Namespace, :import + + keywords id do + rule KEYWORDS, Keyword + rule DECLARATIONS, Keyword::Declaration + rule TYPES, Keyword::Type + rule Set['true', 'false', 'nil'], Keyword::Constant + rule Set['class', 'mixin'], Keyword::Declaration, :class + rule IMPORTS, Keyword::Namespace, :import + end + rule %r/(\.)(#{id})/ do groups Operator, Name::Attribute end From f7e35be8812aa871d0b069f33de1adaec222ecec Mon Sep 17 00:00:00 2001 From: jneen Date: Sun, 12 Apr 2026 18:24:39 -0400 Subject: [PATCH 041/154] use the keywords api for datastudio --- .rubocop_todo.yml | 1 - lib/rouge/lexers/datastudio.rb | 93 ++----------------------- lib/rouge/lexers/datastudio/keywords.rb | 81 +++++++++++++++++++++ 3 files changed, 88 insertions(+), 87 deletions(-) create mode 100644 lib/rouge/lexers/datastudio/keywords.rb diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index 82ab1d7688..d140dbd7ce 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -81,7 +81,6 @@ Rouge/NoBuildingAlternationPatternInRegexp: # Offense count: 20 Rouge/NoHugeCollections: Exclude: - - 'lib/rouge/lexers/datastudio.rb' - 'lib/rouge/lexers/freefem.rb' - 'lib/rouge/lexers/hql.rb' - 'lib/rouge/lexers/idlang.rb' diff --git a/lib/rouge/lexers/datastudio.rb b/lib/rouge/lexers/datastudio.rb index 91cd998804..e3e46672cc 100644 --- a/lib/rouge/lexers/datastudio.rb +++ b/lib/rouge/lexers/datastudio.rb @@ -11,85 +11,9 @@ class Datastudio < RegexLexer title "Datastudio" desc 'Datastudio scripting language' - id = /@?[_a-z]\w*/i + lazy { require_relative 'datastudio/keywords' } - def self.sql_keywords - @sql_keywords ||= %w( - ABORT ABS ABSOLUTE ACCESS ADA ADD ADMIN AFTER AGGREGATE ALIAS - ALL ALLOCATE ALTER ANALYSE ANALYZE AND ANY ARE AS ASC ASENSITIVE - ASSERTION ASSIGNMENT ASYMMETRIC AT ATOMIC AUTHORIZATION - AVG BACKWARD BEFORE BEGIN BETWEEN BITVAR BIT_LENGTH BOTH - BREADTH BY C CACHE CALL CALLED CARDINALITY CASCADE CASCADED - CASE CAST CATALOG CATALOG_NAME CHAIN CHARACTERISTICS - CHARACTER_LENGTH CHARACTER_SET_CATALOG CHARACTER_SET_NAME - CHARACTER_SET_SCHEMA CHAR_LENGTH CHECK CHECKED CHECKPOINT - CLASS CLASS_ORIGIN CLOB CLOSE CLUSTER COALSECE COBOL COLLATE - COLLATION COLLATION_CATALOG COLLATION_NAME COLLATION_SCHEMA - COLUMN COLUMN_NAME COMMAND_FUNCTION COMMAND_FUNCTION_CODE - COMMENT COMMIT COMMITTED COMPLETION CONDITION_NUMBER - CONNECT CONNECTION CONNECTION_NAME CONSTRAINT CONSTRAINTS - CONSTRAINT_CATALOG CONSTRAINT_NAME CONSTRAINT_SCHEMA - CONSTRUCTOR CONTAINS CONTINUE CONVERSION CONVERT COPY - CORRESPONTING COUNT CREATE CREATEDB CREATEUSER CROSS CUBE - CURRENT CURRENT_DATE CURRENT_PATH CURRENT_ROLE CURRENT_TIME - CURRENT_TIMESTAMP CURRENT_USER CURSOR CURSOR_NAME CYCLE DATA - DATABASE DATETIME_INTERVAL_CODE DATETIME_INTERVAL_PRECISION - DAY DEALLOCATE DECLARE DEFAULT DEFAULTS DEFERRABLE DEFERRED - DEFINED DEFINER DELETE DELIMITER DELIMITERS DEREF DESC DESCRIBE - DESCRIPTOR DESTROY DESTRUCTOR DETERMINISTIC DIAGNOSTICS - DICTIONARY DISCONNECT DISPATCH DISTINCT DO DOMAIN DROP - DYNAMIC DYNAMIC_FUNCTION DYNAMIC_FUNCTION_CODE EACH ELSE - ENCODING ENCRYPTED END END-EXEC EQUALS ESCAPE EVERY EXCEPT - ESCEPTION EXCLUDING EXCLUSIVE EXEC EXECUTE EXISTING EXISTS - EXPLAIN EXTERNAL EXTRACT FALSE FETCH FINAL FIRST FOR FORCE - FOREIGN FORTRAN FORWARD FOUND FREE FREEZE FROM FULL FUNCTION - G GENERAL GENERATED GET GLOBAL GO GOTO GRANT GRANTED GROUP - GROUPING HANDLER HAVING HIERARCHY HOLD HOST IDENTITY IGNORE - ILIKE IMMEDIATE IMMUTABLE IMPLEMENTATION IMPLICIT IN INCLUDING - INCREMENT INDEX INDITCATOR INFIX INHERITS INITIALIZE INITIALLY - INNER INOUT INPUT INSENSITIVE INSERT INSTANTIABLE INSTEAD - INTERSECT INTO INVOKER IS ISNULL ISOLATION ITERATE JOIN KEY - KEY_MEMBER KEY_TYPE LANCOMPILER LANGUAGE LARGE LAST LATERAL - LEADING LEFT LENGTH LESS LEVEL LIKE LIMIT LISTEN LOAD LOCAL - LOCALTIME LOCALTIMESTAMP LOCATION LOCATOR LOCK LOWER MAP MATCH - MAX MAXVALUE MESSAGE_LENGTH MESSAGE_OCTET_LENGTH MESSAGE_TEXT - METHOD MIN MINUTE MINVALUE MOD MODE MODIFIES MODIFY MONTH - MORE MOVE MUMPS NAMES NATURAL NCLOB NEW NEXT - NO NOCREATEDB NOCREATEUSER NONE NOT NOTHING NOTIFY NOTNULL - NULL NULLABLE NULLIF OBJECT OCTET_LENGTH OF OFF OFFSET OIDS - OLD ON ONLY OPEN OPERATION OPERATOR OPTION OPTIONS OR ORDER - ORDINALITY OUT OUTER OUTPUT OVERLAPS OVERLAY OVERRIDING - OWNER PAD PARAMETER PARAMETERS PARAMETER_MODE PARAMATER_NAME - PARAMATER_ORDINAL_POSITION PARAMETER_SPECIFIC_CATALOG - PARAMETER_SPECIFIC_NAME PARAMATER_SPECIFIC_SCHEMA PARTIAL PASCAL - PENDANT PLACING PLI POSITION POSTFIX PREFIX PREORDER - PREPARE PRESERVE PRIMARY PRIOR PRIVILEGES PROCEDURAL PROCEDURE - PUBLIC READ READS RECHECK RECURSIVE REF REFERENCES REFERENCING - REINDEX RELATIVE RENAME REPEATABLE REPLACE RESET RESTART - RESTRICT RESULT RETURN RETURNED_LENGTH RETURNED_OCTET_LENGTH - RETURNED_SQLSTATE RETURNS REVOKE RIGHT ROLE ROLLBACK ROLLUP - ROUTINE ROUTINE_CATALOG ROUTINE_NAME ROUTINE_SCHEMA ROW ROWS - ROW_COUNT RULE SAVE_POINT SCALE SCHEMA SCHEMA_NAME SCOPE SCROLL - SEARCH SECOND SECURITY SELECT SELF SENSITIVE SERIALIZABLE - SERVER_NAME SESSION SESSION_USER SET SETOF SETS SHARE SHOW - SIMILAR SIMPLE SIZE SOME SOURCE SPACE SPECIFIC SPECIFICTYPE - SPECIFIC_NAME SQL SQLCODE SQLERROR SQLEXCEPTION SQLSTATE - SQLWARNINIG STABLE START STATE STATEMENT STATIC STATISTICS - STDIN STDOUT STORAGE STRICT STRUCTURE STYPE SUBCLASS_ORIGIN - SUBLIST SUBSTRING SUM SYMMETRIC SYSID SYSTEM SYSTEM_USER - TABLE TABLE_NAME TEMP TEMPLATE TEMPORARY TERMINATE THAN THEN - TIMEZONE_HOUR TIMEZONE_MINUTE TO TOAST TRAILING - TRANSATION TRANSACTIONS_COMMITTED TRANSACTIONS_ROLLED_BACK - TRANSATION_ACTIVE TRANSFORM TRANSFORMS TRANSLATE TRANSLATION - TREAT TRIGGER TRIGGER_CATALOG TRIGGER_NAME TRIGGER_SCHEMA TRIM - TRUE TRUNCATE TRUSTED TYPE UNCOMMITTED UNDER UNENCRYPTED UNION - UNIQUE UNKNOWN UNLISTEN UNNAMED UNNEST UNTIL UPDATE UPPER - USAGE USER USER_DEFINED_TYPE_CATALOG USER_DEFINED_TYPE_NAME - USER_DEFINED_TYPE_SCHEMA USING VACUUM VALID VALIDATOR VALUES - VARIABLE VERBOSE VERSION VIEW VOLATILE WHEN WHENEVER WHERE - WITH WITHOUT WORK WRITE ZONE - ) - end + id = /@?[_a-z]\w*/i state :whitespace do rule %r/\s+/m, Text @@ -122,17 +46,14 @@ def self.sql_keywords rule %r/'/, Str::Single, :string_s rule %r/\d(\.\d*)?/i, Num rule %r/#{id}(?=\s*[(])/, Name::Function - rule id do |m| - name = m[0].upcase - if self.class.sql_keywords.include? name - token Keyword - else - token Name - end + keywords id do + transform(&:upcase) + + rule SQL_KEYWORDS, Keyword + default Name end end - end end end diff --git a/lib/rouge/lexers/datastudio/keywords.rb b/lib/rouge/lexers/datastudio/keywords.rb new file mode 100644 index 0000000000..fd97d525f7 --- /dev/null +++ b/lib/rouge/lexers/datastudio/keywords.rb @@ -0,0 +1,81 @@ +module Rouge + module Lexers + class Datastudio + SQL_KEYWORDS = Set.new %w( + ABORT ABS ABSOLUTE ACCESS ADA ADD ADMIN AFTER AGGREGATE ALIAS + ALL ALLOCATE ALTER ANALYSE ANALYZE AND ANY ARE AS ASC ASENSITIVE + ASSERTION ASSIGNMENT ASYMMETRIC AT ATOMIC AUTHORIZATION + AVG BACKWARD BEFORE BEGIN BETWEEN BITVAR BIT_LENGTH BOTH + BREADTH BY C CACHE CALL CALLED CARDINALITY CASCADE CASCADED + CASE CAST CATALOG CATALOG_NAME CHAIN CHARACTERISTICS + CHARACTER_LENGTH CHARACTER_SET_CATALOG CHARACTER_SET_NAME + CHARACTER_SET_SCHEMA CHAR_LENGTH CHECK CHECKED CHECKPOINT + CLASS CLASS_ORIGIN CLOB CLOSE CLUSTER COALSECE COBOL COLLATE + COLLATION COLLATION_CATALOG COLLATION_NAME COLLATION_SCHEMA + COLUMN COLUMN_NAME COMMAND_FUNCTION COMMAND_FUNCTION_CODE + COMMENT COMMIT COMMITTED COMPLETION CONDITION_NUMBER + CONNECT CONNECTION CONNECTION_NAME CONSTRAINT CONSTRAINTS + CONSTRAINT_CATALOG CONSTRAINT_NAME CONSTRAINT_SCHEMA + CONSTRUCTOR CONTAINS CONTINUE CONVERSION CONVERT COPY + CORRESPONTING COUNT CREATE CREATEDB CREATEUSER CROSS CUBE + CURRENT CURRENT_DATE CURRENT_PATH CURRENT_ROLE CURRENT_TIME + CURRENT_TIMESTAMP CURRENT_USER CURSOR CURSOR_NAME CYCLE DATA + DATABASE DATETIME_INTERVAL_CODE DATETIME_INTERVAL_PRECISION + DAY DEALLOCATE DECLARE DEFAULT DEFAULTS DEFERRABLE DEFERRED + DEFINED DEFINER DELETE DELIMITER DELIMITERS DEREF DESC DESCRIBE + DESCRIPTOR DESTROY DESTRUCTOR DETERMINISTIC DIAGNOSTICS + DICTIONARY DISCONNECT DISPATCH DISTINCT DO DOMAIN DROP + DYNAMIC DYNAMIC_FUNCTION DYNAMIC_FUNCTION_CODE EACH ELSE + ENCODING ENCRYPTED END END-EXEC EQUALS ESCAPE EVERY EXCEPT + ESCEPTION EXCLUDING EXCLUSIVE EXEC EXECUTE EXISTING EXISTS + EXPLAIN EXTERNAL EXTRACT FALSE FETCH FINAL FIRST FOR FORCE + FOREIGN FORTRAN FORWARD FOUND FREE FREEZE FROM FULL FUNCTION + G GENERAL GENERATED GET GLOBAL GO GOTO GRANT GRANTED GROUP + GROUPING HANDLER HAVING HIERARCHY HOLD HOST IDENTITY IGNORE + ILIKE IMMEDIATE IMMUTABLE IMPLEMENTATION IMPLICIT IN INCLUDING + INCREMENT INDEX INDITCATOR INFIX INHERITS INITIALIZE INITIALLY + INNER INOUT INPUT INSENSITIVE INSERT INSTANTIABLE INSTEAD + INTERSECT INTO INVOKER IS ISNULL ISOLATION ITERATE JOIN KEY + KEY_MEMBER KEY_TYPE LANCOMPILER LANGUAGE LARGE LAST LATERAL + LEADING LEFT LENGTH LESS LEVEL LIKE LIMIT LISTEN LOAD LOCAL + LOCALTIME LOCALTIMESTAMP LOCATION LOCATOR LOCK LOWER MAP MATCH + MAX MAXVALUE MESSAGE_LENGTH MESSAGE_OCTET_LENGTH MESSAGE_TEXT + METHOD MIN MINUTE MINVALUE MOD MODE MODIFIES MODIFY MONTH + MORE MOVE MUMPS NAMES NATURAL NCLOB NEW NEXT + NO NOCREATEDB NOCREATEUSER NONE NOT NOTHING NOTIFY NOTNULL + NULL NULLABLE NULLIF OBJECT OCTET_LENGTH OF OFF OFFSET OIDS + OLD ON ONLY OPEN OPERATION OPERATOR OPTION OPTIONS OR ORDER + ORDINALITY OUT OUTER OUTPUT OVERLAPS OVERLAY OVERRIDING + OWNER PAD PARAMETER PARAMETERS PARAMETER_MODE PARAMATER_NAME + PARAMATER_ORDINAL_POSITION PARAMETER_SPECIFIC_CATALOG + PARAMETER_SPECIFIC_NAME PARAMATER_SPECIFIC_SCHEMA PARTIAL PASCAL + PENDANT PLACING PLI POSITION POSTFIX PREFIX PREORDER + PREPARE PRESERVE PRIMARY PRIOR PRIVILEGES PROCEDURAL PROCEDURE + PUBLIC READ READS RECHECK RECURSIVE REF REFERENCES REFERENCING + REINDEX RELATIVE RENAME REPEATABLE REPLACE RESET RESTART + RESTRICT RESULT RETURN RETURNED_LENGTH RETURNED_OCTET_LENGTH + RETURNED_SQLSTATE RETURNS REVOKE RIGHT ROLE ROLLBACK ROLLUP + ROUTINE ROUTINE_CATALOG ROUTINE_NAME ROUTINE_SCHEMA ROW ROWS + ROW_COUNT RULE SAVE_POINT SCALE SCHEMA SCHEMA_NAME SCOPE SCROLL + SEARCH SECOND SECURITY SELECT SELF SENSITIVE SERIALIZABLE + SERVER_NAME SESSION SESSION_USER SET SETOF SETS SHARE SHOW + SIMILAR SIMPLE SIZE SOME SOURCE SPACE SPECIFIC SPECIFICTYPE + SPECIFIC_NAME SQL SQLCODE SQLERROR SQLEXCEPTION SQLSTATE + SQLWARNINIG STABLE START STATE STATEMENT STATIC STATISTICS + STDIN STDOUT STORAGE STRICT STRUCTURE STYPE SUBCLASS_ORIGIN + SUBLIST SUBSTRING SUM SYMMETRIC SYSID SYSTEM SYSTEM_USER + TABLE TABLE_NAME TEMP TEMPLATE TEMPORARY TERMINATE THAN THEN + TIMEZONE_HOUR TIMEZONE_MINUTE TO TOAST TRAILING + TRANSATION TRANSACTIONS_COMMITTED TRANSACTIONS_ROLLED_BACK + TRANSATION_ACTIVE TRANSFORM TRANSFORMS TRANSLATE TRANSLATION + TREAT TRIGGER TRIGGER_CATALOG TRIGGER_NAME TRIGGER_SCHEMA TRIM + TRUE TRUNCATE TRUSTED TYPE UNCOMMITTED UNDER UNENCRYPTED UNION + UNIQUE UNKNOWN UNLISTEN UNNAMED UNNEST UNTIL UPDATE UPPER + USAGE USER USER_DEFINED_TYPE_CATALOG USER_DEFINED_TYPE_NAME + USER_DEFINED_TYPE_SCHEMA USING VACUUM VALID VALIDATOR VALUES + VARIABLE VERBOSE VERSION VIEW VOLATILE WHEN WHENEVER WHERE + WITH WITHOUT WORK WRITE ZONE + ) + end + end +end From fc603d1f8e262b62df394169aeba351b6ce90a1c Mon Sep 17 00:00:00 2001 From: jneen Date: Sun, 12 Apr 2026 18:32:30 -0400 Subject: [PATCH 042/154] use the keywords api for digdag --- lib/rouge/lexers/digdag.rb | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/rouge/lexers/digdag.rb b/lib/rouge/lexers/digdag.rb index 384145e562..7b1ad7007e 100644 --- a/lib/rouge/lexers/digdag.rb +++ b/lib/rouge/lexers/digdag.rb @@ -14,7 +14,7 @@ class Digdag < YAML # http://docs.digdag.io/operators.html # as of digdag v0.9.10 - KEYWORD_PATTERN = Regexp.union(%w( + KEYWORDS = Set.new(%w( call require loop @@ -59,9 +59,12 @@ class Digdag < YAML )) prepend :block_nodes do - rule %r/(#{KEYWORD_PATTERN})(:)(?=\s|$)/ do |m| - groups Keyword::Reserved, Punctuation::Indicator - set_indent m[0], :implicit => true + keywords %r/(\w+>?)(:)(?=\s|$)/ do + group 1 + rule KEYWORDS do |m| + groups Keyword::Reserved, Punctuation::Indicator + set_indent m[0], :implicit => true + end end end end From afa625d068949e1039fa4ad93d223ebea4ddd620 Mon Sep 17 00:00:00 2001 From: jneen Date: Sun, 12 Apr 2026 20:43:56 -0400 Subject: [PATCH 043/154] use the keywords api for dylan --- lib/rouge/lexers/dylan.rb | 43 ++++++++++++++++----------------------- 1 file changed, 18 insertions(+), 25 deletions(-) diff --git a/lib/rouge/lexers/dylan.rb b/lib/rouge/lexers/dylan.rb index 997d44a9be..4b70bacec2 100644 --- a/lib/rouge/lexers/dylan.rb +++ b/lib/rouge/lexers/dylan.rb @@ -14,14 +14,13 @@ class Dylan < RegexLexer # https://opendylan.org/books/drm/Modules # https://opendylan.org/books/drm/Conditional_Execution # https://opendylan.org/books/drm/Statement_Macros - reserved_words = Set.new %w( + RESERVED_WORDS = Set.new %w( begin block case class constant create define domain else end exception for function generic handler if let library local macro method module otherwise select unless until variable while ) - hash_word = Set.new %w(#t #f #next #rest #key #all-keys #include) - operators = Set.new %w(+ - * / ^ = == ~ ~= ~== < <= > >= & | :=) + OPERATORS = Set.new %w(+ - * / ^ = == ~ ~= ~== < <= > >= & | :=) state :root do rule %r/^[\w.-]+:/, Comment::Preproc, :header @@ -48,16 +47,6 @@ class Dylan < RegexLexer rule %r/\s+/, Text::Whitespace # Keywords - rule %r/\w+/ do |m| - if reserved_words.include?(m[0]) - token Keyword - elsif hash_word.include?(m[0]) - token Keyword::Constant - else - fallthrough! - end - end - rule %r/#(t|f|next|rest|key|all-keys|include)\b/, Keyword::Constant # Numbers @@ -77,18 +66,22 @@ class Dylan < RegexLexer rule %r/\\#{word_re}/, Str::Symbol - rule word_re do |m| - word = m[0] - if operators.include?(word) - token Operator - elsif word.start_with?('<') && word.end_with?('>') - token Name::Class - elsif word.start_with?('*') && word.end_with?('*') - token Name::Variable::Instance - elsif word.start_with?('$') - token Name::Constant - else - token Name + keywords word_re do + rule RESERVED_WORDS, Keyword + rule OPERATORS, Operator + + default do |m| + word = m[0] + + if word.start_with?('<') && word.end_with?('>') + token Name::Class + elsif word.start_with?('*') && word.end_with?('*') + token Name::Variable::Instance + elsif word.start_with?('$') + token Name::Constant + else + token Name + end end end From c4464a6f32df45193bf30594329d689e340b9ab9 Mon Sep 17 00:00:00 2001 From: jneen Date: Sun, 12 Apr 2026 21:04:29 -0400 Subject: [PATCH 044/154] use the keywords api for eiffel --- .rubocop_todo.yml | 6 ------ lib/rouge/lexers/eiffel.rb | 34 +++++++++++++++++++--------------- 2 files changed, 19 insertions(+), 21 deletions(-) diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index d140dbd7ce..f8bc47f69c 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -12,11 +12,6 @@ Lint/EmptyBlock: Exclude: - 'lib/rouge/lexers/nix.rb' -# Offense count: 4 -Naming/ConstantName: - Exclude: - - 'lib/rouge/lexers/eiffel.rb' - # Offense count: 5 # This cop supports unsafe autocorrection (--autocorrect-all). # Configuration parameters: EnforcedStyleForLeadingUnderscores. @@ -50,7 +45,6 @@ Naming/VariableName: Rouge/NoBuildingAlternationPatternInRegexp: Exclude: - 'lib/rouge/lexers/armasm.rb' - - 'lib/rouge/lexers/eiffel.rb' - 'lib/rouge/lexers/elm.rb' - 'lib/rouge/lexers/erlang.rb' - 'lib/rouge/lexers/haskell.rb' diff --git a/lib/rouge/lexers/eiffel.rb b/lib/rouge/lexers/eiffel.rb index 94123c5d10..3519c55d29 100644 --- a/lib/rouge/lexers/eiffel.rb +++ b/lib/rouge/lexers/eiffel.rb @@ -11,7 +11,7 @@ class Eiffel < RegexLexer filenames '*.e' mimetypes 'text/x-eiffel' - LanguageKeywords = %w( + KEYWORDS = Set.new %w( across agent alias all and attached as assign attribute check class convert create debug deferred detachable do else elseif end ensure expanded export external feature from frozen if implies inherit @@ -20,22 +20,30 @@ class convert create debug deferred detachable do else elseif end some then undefine until variant Void when xor ) - BooleanConstants = %w(True False) - - LanguageVariables = %w(Current Result) - - SimpleString = /(?:[^"%\b\f\v]|%[A-DFHLNQR-V%'"()<>]|%\/(?:0[xX][\da-fA-F](?:_*[\da-fA-F])*|0[cC][0-7](?:_*[0-7])*|0[bB][01](?:_*[01])*|\d(?:_*\d)*)\/)+?/ + STRING_RE = /(?:[^"%\b\f\v]|%[A-DFHLNQR-V%'"()<>]|%\/(?:0[xX][\da-fA-F](?:_*[\da-fA-F])*|0[cC][0-7](?:_*[0-7])*|0[bB][01](?:_*[01])*|\d(?:_*\d)*)\/)+?/ state :root do + rule %r/\s+/, Text + rule %r/"\[/, Str::Other, :aligned_verbatim_string rule %r/"\{/, Str::Other, :non_aligned_verbatim_string rule %r/"(?:[^%\b\f\n\r\v]|%[A-DFHLNQR-V%'"()<>]|%\/(?:0[xX][\da-fA-F](?:_*[\da-fA-F])*|0[cC][0-7](?:_*[0-7])*|0[bB][01](?:_*[01])*|\d(?:_*\d)*)\/)*?"/, Str::Double rule %r/--.*/, Comment::Single rule %r/'(?:[^%\b\f\n\r\t\v]|%[A-DFHLNQR-V%'"()<>]|%\/(?:0[xX][\da-fA-F](?:_*[\da-fA-F])*|0[cC][0-7](?:_*[0-7])*|0[bB][01](?:_*[01])*|\d(?:_*\d)*)\/)'/, Str::Char - rule %r/(?:#{LanguageKeywords.join('|')})\b/, Keyword - rule %r/(?:#{LanguageVariables.join('|')})\b/, Keyword::Variable - rule %r/(?:#{BooleanConstants.join('|')})\b/, Keyword::Constant + keywords %r/[a-z]\w*/i do + rule KEYWORDS, Keyword + rule Set['Current', 'Result'], Keyword::Variable + rule Set['True', 'False'], Keyword::Constant + + default do |m| + if ('A'..'Z').cover?(m[0][0]) + token Name::Class + else + token Name + end + end + end rule %r/\b0[xX][\da-fA-F](?:_*[\da-fA-F])*b/, Num::Hex rule %r/\b0[cC][0-7](?:_*[0-7])*\b/, Num::Oct @@ -45,20 +53,16 @@ class convert create debug deferred detachable do else elseif end rule %r/:=|<<|>>|\(\||\|\)|->|\.|[{}\[\];(),:?]/, Punctuation::Indicator rule %r/\\\\|\|\.\.\||\.\.|\/[~\/]?|[><\/]=?|[-+*^=~≤≥−∀∃¦⟳⟲]/, Operator - - rule %r/[A-Z][\dA-Z_]*/, Name::Class - rule %r/[A-Za-z][\dA-Za-z_]*/, Name - rule %r/\s+/, Text end state :aligned_verbatim_string do rule %r/]"/, Str::Other, :pop! - rule SimpleString, Str::Other + rule STRING_RE, Str::Other end state :non_aligned_verbatim_string do rule %r/}"/, Str::Other, :pop! - rule SimpleString, Str::Other + rule STRING_RE, Str::Other end end end From be4e2360b4ce4585d0e30ce74da6bdc7c97f2d82 Mon Sep 17 00:00:00 2001 From: jneen Date: Sun, 12 Apr 2026 23:13:33 -0400 Subject: [PATCH 045/154] use the keywords api for elm --- .rubocop_todo.yml | 1 - lib/rouge/lexers/elm.rb | 13 ++++++------- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index f8bc47f69c..edf936cd90 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -45,7 +45,6 @@ Naming/VariableName: Rouge/NoBuildingAlternationPatternInRegexp: Exclude: - 'lib/rouge/lexers/armasm.rb' - - 'lib/rouge/lexers/elm.rb' - 'lib/rouge/lexers/erlang.rb' - 'lib/rouge/lexers/haskell.rb' - 'lib/rouge/lexers/http.rb' diff --git a/lib/rouge/lexers/elm.rb b/lib/rouge/lexers/elm.rb index 526c33f337..652177c875 100644 --- a/lib/rouge/lexers/elm.rb +++ b/lib/rouge/lexers/elm.rb @@ -12,7 +12,7 @@ class Elm < RegexLexer mimetypes 'text/x-elm' # Keywords are logically grouped by lines - keywords = %w( + KEYWORDS = Set.new %w( module exposing port import as type alias @@ -29,13 +29,12 @@ module exposing port # Multiline comments rule %r/{-/, Comment::Multiline, :multiline_comment - # Keywords - rule %r/\b(#{keywords.join('|')})\b/, Keyword + # Keywords and normal names + keywords %r/[a-z_]\w*/ do + rule KEYWORDS, Keyword + default Name + end - # Variable or a function - rule %r/[a-z]\w*/, Name - # Underscore is a name for a variable, when it won't be used later - rule %r/_/, Name # Type rule %r/[A-Z]\w*/, Keyword::Type From 551acf81460160f77b57121e35edd089bc26437c Mon Sep 17 00:00:00 2001 From: jneen Date: Sun, 12 Apr 2026 23:34:18 -0400 Subject: [PATCH 046/154] use the keywords api for Erlang --- .rubocop_todo.yml | 1 - lib/rouge/lexers/erlang.rb | 22 +++++++++++++--------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index edf936cd90..92cfe27e4e 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -45,7 +45,6 @@ Naming/VariableName: Rouge/NoBuildingAlternationPatternInRegexp: Exclude: - 'lib/rouge/lexers/armasm.rb' - - 'lib/rouge/lexers/erlang.rb' - 'lib/rouge/lexers/haskell.rb' - 'lib/rouge/lexers/http.rb' - 'lib/rouge/lexers/idris.rb' diff --git a/lib/rouge/lexers/erlang.rb b/lib/rouge/lexers/erlang.rb index 031f2bd2bc..95d7f66d3f 100644 --- a/lib/rouge/lexers/erlang.rb +++ b/lib/rouge/lexers/erlang.rb @@ -12,12 +12,12 @@ class Erlang < RegexLexer mimetypes 'text/x-erlang', 'application/x-erlang' - keywords = %w( + KEYWORDS = Set.new %w( after begin case catch cond end fun if let of query receive try when ) - builtins = %w( + BUILTINS = Set.new %w( abs append_element apply atom_to_list binary_to_list bitstring_to_list binary_to_term bit_size bump_reductions byte_size cancel_timer check_process_code delete_module @@ -48,7 +48,7 @@ class Erlang < RegexLexer ) operators = %r{(\+\+?|--?|\*|/|<|>|/=|=:=|=/=|=<|>=|==?|<-|!|\?)} - word_operators = %w( + WORD_OPERATORS = Set.new %w( and andalso band bnot bor bsl bsr bxor div not or orelse rem xor ) @@ -66,9 +66,13 @@ class Erlang < RegexLexer state :root do rule(/\s+/, Text) rule(/%.*\n/, Comment) - rule(%r{(#{keywords.join('|')})\b}, Keyword) - rule(%r{(#{builtins.join('|')})\b}, Name::Builtin) - rule(%r{(#{word_operators.join('|')})\b}, Operator::Word) + + keywords %r/[a-z]\w*/ do + rule KEYWORDS, Keyword + rule BUILTINS, Name::Builtin + rule WORD_OPERATORS, Operator::Word + end + rule(/^-/, Punctuation, :directive) rule(operators, Operator) rule(/"/, Str, :string) @@ -101,14 +105,14 @@ class Erlang < RegexLexer state :directive do rule %r{(define)(\s*)(\()(#{macro_re})} do - groups Name::Entity, Text, Punctuation, Name::Constant + groups Keyword::Type, Text, Punctuation, Name::Constant pop! end rule %r{(record)(\s*)(\()(#{macro_re})} do - groups Name::Entity, Text, Punctuation, Name::Label + groups Keyword::Type, Text, Punctuation, Name::Label pop! end - rule(atom_re, Name::Entity, :pop!) + rule(atom_re, Comment::Preproc, :pop!) end end end From 2c4ec8aaf473cd0545289ae17fea4dd5943657da Mon Sep 17 00:00:00 2001 From: jneen Date: Sun, 12 Apr 2026 23:50:05 -0400 Subject: [PATCH 047/154] use the keywords api for factor --- lib/rouge/lexers/factor.rb | 171 ++-------------------------- lib/rouge/lexers/factor/builtins.rb | 133 ++++++++++++++++++++++ 2 files changed, 141 insertions(+), 163 deletions(-) create mode 100644 lib/rouge/lexers/factor/builtins.rb diff --git a/lib/rouge/lexers/factor.rb b/lib/rouge/lexers/factor.rb index dc3fdf01f7..1a6a065702 100644 --- a/lib/rouge/lexers/factor.rb +++ b/lib/rouge/lexers/factor.rb @@ -10,160 +10,12 @@ class Factor < RegexLexer filenames '*.factor' mimetypes 'text/x-factor' - def self.detect?(text) - return true if text.shebang? 'factor' - end - - def self.builtins - @builtins ||= {}.tap do |builtins| - builtins[:kernel] = Set.new %w( - or 2bi 2tri while wrapper nip 4dip wrapper? bi* - callstack>array both? hashcode die dupd callstack - callstack? 3dup tri@ pick curry build ?execute 3bi prepose - >boolean if clone eq? tri* ? = swapd 2over 2keep 3keep clear - 2dup when not tuple? dup 2bi* 2tri* call tri-curry object bi@ - do unless* if* loop bi-curry* drop when* assert= retainstack - assert? -rot execute 2bi@ 2tri@ boa with either? 3drop bi - curry? datastack until 3dip over 3curry tri-curry* tri-curry@ - swap and 2nip throw bi-curry (clone) hashcode* compose 2dip - 3tri unless compose? tuple keep 2curry equal? assert tri 2drop - most boolean? identity-hashcode identity-tuple? - null new dip bi-curry@ rot xor identity-tuple boolean - ) - - builtins[:assocs] = Set.new %w( - ?at assoc? assoc-clone-like assoc= delete-at* assoc-partition - extract-keys new-assoc value? assoc-size map>assoc push-at - assoc-like key? assoc-intersect assoc-refine update - assoc-union assoc-combine at* assoc-empty? at+ set-at - assoc-all? assoc-subset? assoc-hashcode change-at assoc-each - assoc-diff zip values value-at rename-at inc-at enum? at cache - assoc>map assoc assoc-map enum value-at* assoc-map-as - >alist assoc-filter-as clear-assoc assoc-stack maybe-set-at - substitute assoc-filter 2cache delete-at assoc-find keys - assoc-any? unzip - ) - - builtins[:combinators] = Set.new %w( - case execute-effect no-cond no-case? 3cleave>quot 2cleave - cond>quot wrong-values? no-cond? cleave>quot no-case case>quot - 3cleave wrong-values to-fixed-point alist>quot case-find - cond cleave call-effect 2cleave>quot recursive-hashcode - linear-case-quot spread spread>quot - ) - - builtins[:math] = Set.new %w( - number= if-zero next-power-of-2 each-integer ?1+ - fp-special? imaginary-part unless-zero float>bits number? - fp-infinity? bignum? fp-snan? denominator fp-bitwise= * - + power-of-2? - u>= / >= bitand log2-expects-positive < - log2 > integer? number bits>double 2/ zero? (find-integer) - bits>float float? shift ratio? even? ratio fp-sign bitnot - >fixnum complex? /i /f byte-array>bignum when-zero sgn >bignum - next-float u< u> mod recip rational find-last-integer >float - (all-integers?) 2^ times integer fixnum? neg fixnum sq bignum - (each-integer) bit? fp-qnan? find-integer complex - real double>bits bitor rem fp-nan-payload all-integers? - real-part log2-expects-positive? prev-float align unordered? - float fp-nan? abs bitxor u<= odd? <= /mod rational? >integer - real? numerator - ) - - builtins[:sequences] = Set.new %w( - member-eq? append assert-sequence= find-last-from - trim-head-slice clone-like 3sequence assert-sequence? map-as - last-index-from reversed index-from cut* pad-tail - remove-eq! concat-as but-last snip trim-tail nths - nth 2selector sequence slice? partition - remove-nth tail-slice empty? tail* if-empty - find-from virtual-sequence? member? set-length - drop-prefix unclip unclip-last-slice iota map-sum - bounds-error? sequence-hashcode-step selector-for - accumulate-as map start midpoint@ (accumulate) rest-slice - prepend fourth sift accumulate! new-sequence follow map! like - first4 1sequence reverse slice unless-empty padding virtual@ - repetition? set-last index 4sequence max-length set-second - immutable-sequence first2 first3 replicate-as reduce-index - unclip-slice supremum suffix! insert-nth trim-tail-slice - tail 3append short count suffix concat flip filter sum - immutable? reverse! 2sequence map-integers delete-all start* - indices snip-slice check-slice sequence? head map-find - filter! append-as reduce sequence= halves collapse-slice - interleave 2map filter-as binary-reduce slice-error? product - bounds-check? bounds-check harvest immutable virtual-exemplar - find produce remove pad-head last replicate set-fourth - remove-eq shorten reversed? map-find-last 3map-as - 2unclip-slice shorter? 3map find-last head-slice pop* 2map-as - tail-slice* but-last-slice 2map-reduce iota? collector-for - accumulate each selector append! new-resizable cut-slice - each-index head-slice* 2reverse-each sequence-hashcode - pop set-nth ?nth second join when-empty - collector immutable-sequence? all? 3append-as - virtual-sequence subseq? remove-nth! push-either new-like - length last-index push-if 2all? lengthen assert-sequence - copy map-reduce move third first 3each tail? set-first prefix - bounds-error any? trim-slice exchange surround - 2reduce cut change-nth min-length set-third produce-as - push-all head? delete-slice rest sum-lengths 2each head* - infimum remove! glue slice-error subseq trim replace-slice - push repetition map-index trim-head unclip-last mismatch - ) - - builtins[:namespaces] = Set.new %w( - global +@ change set-namestack change-global init-namespaces - on off set-global namespace set with-scope bind with-variable - inc dec counter initialize namestack get get-global make-assoc - ) - - builtins[:arrays] = Set.new %w( - 2array 3array pair >array 1array 4array pair? - array resize-array array? - ) - - builtins[:io] = Set.new %w( - +character+ bad-seek-type? readln each-morsel - stream-seek read print with-output-stream contents - write1 stream-write1 stream-copy stream-element-type - with-input-stream stream-print stream-read stream-contents - stream-tell tell-output bl seek-output bad-seek-type nl - stream-nl write flush stream-lines +byte+ stream-flush - read1 seek-absolute? stream-read1 lines stream-readln - stream-read-until each-line seek-end with-output-stream* - seek-absolute with-streams seek-input seek-relative? - input-stream stream-write read-partial seek-end? - seek-relative error-stream read-until with-input-stream* - with-streams* tell-input each-block output-stream - stream-read-partial each-stream-block each-stream-line - ) - - builtins[:strings] = Set.new %w( - resize-string >string 1string string string? - ) + KEYWORDS = Set.new %w(deprecated final foldable flushable inline recursive) - builtins[:vectors] = Set.new %w( - with-return restarts return-continuation with-datastack - recover rethrow-restarts ifcc set-catchstack - >continuation< cleanup ignore-errors restart? - compute-restarts attempt-all-error error-thread - continue attempt-all-error? condition? - throw-restarts error catchstack continue-with - thread-error-hook continuation rethrow callcc1 - error-continuation callcc0 attempt-all condition - continuation? restart return - ) + lazy { require_relative 'factor/builtins' } - builtins[:continuations] = Set.new %w( - with-return restarts return-continuation with-datastack - recover rethrow-restarts ifcc set-catchstack - >continuation< cleanup ignore-errors restart? - compute-restarts attempt-all-error error-thread - continue attempt-all-error? condition? - throw-restarts error catchstack continue-with - thread-error-hook continuation rethrow callcc1 - error-continuation callcc0 attempt-all condition - continuation? restart return - ) - end + def self.detect?(text) + return true if text.shebang? 'factor' end state :root do @@ -264,17 +116,10 @@ def self.builtins rule %r([-+/*=<>^](?=\s)), Operator - rule %r/(?:deprecated|final|foldable|flushable|inline|recursive)(?=\s)/, - Keyword - - rule %r/\S+/ do |m| - name = m[0] - - if self.class.builtins.values.any? { |b| b.include? name } - token Name::Builtin - else - token Name - end + keywords %r/\S+/ do + rule KEYWORDS, Keyword + rule BUILTINS, Name::Builtin + default Name end end diff --git a/lib/rouge/lexers/factor/builtins.rb b/lib/rouge/lexers/factor/builtins.rb new file mode 100644 index 0000000000..5780de25ce --- /dev/null +++ b/lib/rouge/lexers/factor/builtins.rb @@ -0,0 +1,133 @@ +module Rouge + module Lexers + class Factor + BUILTINS = Set.new %w( + or 2bi 2tri while wrapper nip 4dip wrapper? bi* + callstack>array both? hashcode die dupd callstack + callstack? 3dup tri@ pick curry build ?execute 3bi prepose + >boolean if clone eq? tri* ? = swapd 2over 2keep 3keep clear + 2dup when not tuple? dup 2bi* 2tri* call tri-curry object bi@ + do unless* if* loop bi-curry* drop when* assert= retainstack + assert? -rot execute 2bi@ 2tri@ boa with either? 3drop bi + curry? datastack until 3dip over 3curry tri-curry* tri-curry@ + swap and 2nip throw bi-curry (clone) hashcode* compose 2dip + 3tri unless compose? tuple keep 2curry equal? assert tri 2drop + most boolean? identity-hashcode identity-tuple? + null new dip bi-curry@ rot xor identity-tuple boolean + + ?at assoc? assoc-clone-like assoc= delete-at* assoc-partition + extract-keys new-assoc value? assoc-size map>assoc push-at + assoc-like key? assoc-intersect assoc-refine update + assoc-union assoc-combine at* assoc-empty? at+ set-at + assoc-all? assoc-subset? assoc-hashcode change-at assoc-each + assoc-diff zip values value-at rename-at inc-at enum? at cache + assoc>map assoc assoc-map enum value-at* assoc-map-as + >alist assoc-filter-as clear-assoc assoc-stack maybe-set-at + substitute assoc-filter 2cache delete-at assoc-find keys + assoc-any? unzip + + case execute-effect no-cond no-case? 3cleave>quot 2cleave + cond>quot wrong-values? no-cond? cleave>quot no-case case>quot + 3cleave wrong-values to-fixed-point alist>quot case-find + cond cleave call-effect 2cleave>quot recursive-hashcode + linear-case-quot spread spread>quot + + number= if-zero next-power-of-2 each-integer ?1+ + fp-special? imaginary-part unless-zero float>bits number? + fp-infinity? bignum? fp-snan? denominator fp-bitwise= * + + power-of-2? - u>= / >= bitand log2-expects-positive < + log2 > integer? number bits>double 2/ zero? (find-integer) + bits>float float? shift ratio? even? ratio fp-sign bitnot + >fixnum complex? /i /f byte-array>bignum when-zero sgn >bignum + next-float u< u> mod recip rational find-last-integer >float + (all-integers?) 2^ times integer fixnum? neg fixnum sq bignum + (each-integer) bit? fp-qnan? find-integer complex + real double>bits bitor rem fp-nan-payload all-integers? + real-part log2-expects-positive? prev-float align unordered? + float fp-nan? abs bitxor u<= odd? <= /mod rational? >integer + real? numerator + + member-eq? append assert-sequence= find-last-from + trim-head-slice clone-like 3sequence assert-sequence? map-as + last-index-from reversed index-from cut* pad-tail + remove-eq! concat-as but-last snip trim-tail nths + nth 2selector sequence slice? partition + remove-nth tail-slice empty? tail* if-empty + find-from virtual-sequence? member? set-length + drop-prefix unclip unclip-last-slice iota map-sum + bounds-error? sequence-hashcode-step selector-for + accumulate-as map start midpoint@ (accumulate) rest-slice + prepend fourth sift accumulate! new-sequence follow map! like + first4 1sequence reverse slice unless-empty padding virtual@ + repetition? set-last index 4sequence max-length set-second + immutable-sequence first2 first3 replicate-as reduce-index + unclip-slice supremum suffix! insert-nth trim-tail-slice + tail 3append short count suffix concat flip filter sum + immutable? reverse! 2sequence map-integers delete-all start* + indices snip-slice check-slice sequence? head map-find + filter! append-as reduce sequence= halves collapse-slice + interleave 2map filter-as binary-reduce slice-error? product + bounds-check? bounds-check harvest immutable virtual-exemplar + find produce remove pad-head last replicate set-fourth + remove-eq shorten reversed? map-find-last 3map-as + 2unclip-slice shorter? 3map find-last head-slice pop* 2map-as + tail-slice* but-last-slice 2map-reduce iota? collector-for + accumulate each selector append! new-resizable cut-slice + each-index head-slice* 2reverse-each sequence-hashcode + pop set-nth ?nth second join when-empty + collector immutable-sequence? all? 3append-as + virtual-sequence subseq? remove-nth! push-either new-like + length last-index push-if 2all? lengthen assert-sequence + copy map-reduce move third first 3each tail? set-first prefix + bounds-error any? trim-slice exchange surround + 2reduce cut change-nth min-length set-third produce-as + push-all head? delete-slice rest sum-lengths 2each head* + infimum remove! glue slice-error subseq trim replace-slice + push repetition map-index trim-head unclip-last mismatch + + global +@ change set-namestack change-global init-namespaces + on off set-global namespace set with-scope bind with-variable + inc dec counter initialize namestack get get-global make-assoc + + 2array 3array pair >array 1array 4array pair? + array resize-array array? + + +character+ bad-seek-type? readln each-morsel + stream-seek read print with-output-stream contents + write1 stream-write1 stream-copy stream-element-type + with-input-stream stream-print stream-read stream-contents + stream-tell tell-output bl seek-output bad-seek-type nl + stream-nl write flush stream-lines +byte+ stream-flush + read1 seek-absolute? stream-read1 lines stream-readln + stream-read-until each-line seek-end with-output-stream* + seek-absolute with-streams seek-input seek-relative? + input-stream stream-write read-partial seek-end? + seek-relative error-stream read-until with-input-stream* + with-streams* tell-input each-block output-stream + stream-read-partial each-stream-block each-stream-line + + resize-string >string 1string string string? + + with-return restarts return-continuation with-datastack + recover rethrow-restarts ifcc set-catchstack + >continuation< cleanup ignore-errors restart? + compute-restarts attempt-all-error error-thread + continue attempt-all-error? condition? + throw-restarts error catchstack continue-with + thread-error-hook continuation rethrow callcc1 + error-continuation callcc0 attempt-all condition + continuation? restart return + + with-return restarts return-continuation with-datastack + recover rethrow-restarts ifcc set-catchstack + >continuation< cleanup ignore-errors restart? + compute-restarts attempt-all-error error-thread + continue attempt-all-error? condition? + throw-restarts error catchstack continue-with + thread-error-hook continuation rethrow callcc1 + error-continuation callcc0 attempt-all condition + continuation? restart return + ) + end + end +end From 5d2b5589deda34850d6de3b4cc0223a2b8e847c6 Mon Sep 17 00:00:00 2001 From: jneen Date: Mon, 13 Apr 2026 12:10:18 -0400 Subject: [PATCH 048/154] use the keywords api for fortran --- lib/rouge/lexers/fortran.rb | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/lib/rouge/lexers/fortran.rb b/lib/rouge/lexers/fortran.rb index ccc8d36b05..aa2e45085f 100644 --- a/lib/rouge/lexers/fortran.rb +++ b/lib/rouge/lexers/fortran.rb @@ -136,19 +136,12 @@ def self.intrinsics rule %r/sync\s+(all|images|memory)\b/i, Keyword rule %r/error\s+stop\b/i, Keyword - rule %r/#{name}/m do |m| - match = m[0].downcase - if self.class.keywords.include? match - token Keyword - elsif self.class.types.include? match - token Keyword::Type - elsif self.class.intrinsics.include? match - token Name::Builtin - else - token Name - end + keywords name do + rule :keywords, Keyword + rule :types, Keyword::Type + rule :intrinsics, Name::Builtin + default Name end - end state :string_single do From 8d86c81b8d833d13bdd43b9d80cfb6a1a2f3f27a Mon Sep 17 00:00:00 2001 From: jneen Date: Tue, 14 Apr 2026 11:40:27 -0400 Subject: [PATCH 049/154] use the keywords api for freefem --- .rubocop_todo.yml | 3 - lib/rouge/lexers/freefem.rb | 197 ++------------------------- lib/rouge/lexers/freefem/keywords.rb | 176 ++++++++++++++++++++++++ 3 files changed, 184 insertions(+), 192 deletions(-) create mode 100644 lib/rouge/lexers/freefem/keywords.rb diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index 92cfe27e4e..32afc73b8b 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -18,7 +18,6 @@ Lint/EmptyBlock: # SupportedStylesForLeadingUnderscores: disallowed, required, optional Naming/MemoizedInstanceVariableName: Exclude: - - 'lib/rouge/lexers/freefem.rb' - 'lib/rouge/lexers/verilog.rb' # Offense count: 16 @@ -36,7 +35,6 @@ Naming/MethodName: Naming/VariableName: Exclude: - 'lib/rouge/lexers/dafny.rb' - - 'lib/rouge/lexers/freefem.rb' - 'lib/rouge/lexers/igorpro.rb' - 'lib/rouge/lexers/kotlin.rb' - 'lib/rouge/lexers/xpath.rb' @@ -73,7 +71,6 @@ Rouge/NoBuildingAlternationPatternInRegexp: # Offense count: 20 Rouge/NoHugeCollections: Exclude: - - 'lib/rouge/lexers/freefem.rb' - 'lib/rouge/lexers/hql.rb' - 'lib/rouge/lexers/idlang.rb' - 'lib/rouge/lexers/igorpro.rb' diff --git a/lib/rouge/lexers/freefem.rb b/lib/rouge/lexers/freefem.rb index 53a784bc59..fbb670bf83 100644 --- a/lib/rouge/lexers/freefem.rb +++ b/lib/rouge/lexers/freefem.rb @@ -14,180 +14,7 @@ class FreeFEM < Cpp filenames '*.edp', '*.idp' mimetypes 'text/x-ffhdr', 'text/x-ffsrc' - # Override C/C++ ones (for example, `do` does not exists) - def self.keywords - @keywords ||= Set.new(%w( - break catch continue else for if return try while - )) - end - - # Override C/C++ ones (for example, `double` does not exists) - def self.keywords_type - @keywords_type ||= Set.new(%w( - bool border complex dmatrix fespace func gslspline ifstream int macro - matrix mesh mesh3 mpiComm mpiGroup mpiRequest NewMacro EndMacro - ofstream Pmmap problem Psemaphore real solve string varf - )) - end - - # Override C/C++ ones (totally different) - def self.reserved - @reserved ||= Set.new(%w( - BDM1 BDM1Ortho Edge03d Edge13d Edge23d FEQF HCT P0 P03d P0Edge P1 - P13d P1b P1b3d P1bl P1bl3d P1dc P1Edge P1nc P2 P23d P2b P2BR P2dc - P2Edge P2h P2Morley P2pnc P3 P3dc P3Edge P4 P4dc P4Edge P5Edge RT0 - RT03d RT0Ortho RT1 RT1Ortho RT2 RT2Ortho - - qf1pE qf1pElump qf1pT qf1pTlump qfV1 qfV1lump qf2pE qf2pT qf2pT4P1 - qfV2 qf3pE qf4pE qf5pE qf5pT qfV5 qf7pT qf9pT qfnbpE - - ARGV append area be binary BoundaryEdge bordermeasure CG Cholesky cin - cout Crout default diag edgeOrientation endl FILE fixed GMRES good - hTriangle im imax imin InternalEdge l1 l2 label lenEdge length LINE - linfty LU m max measure min mpiAnySource mpiBAND mpiBXOR mpiCommWorld - mpiLAND mpiLOR mpiLXOR mpiMAX mpiMIN mpiPROD mpirank mpisize mpiSUM - mpiUndefined n N nbe ndof ndofK noshowbase noshowpos notaregion nt - nTonEdge nuEdge nuTriangle nv P pi precision quantile re region - scientific searchMethod setw showbase showpos sparsesolver sum tellp - true UMFPACK unused whoinElement verbosity version volume x y z - )) - end - - def self.builtins - @builtins ||= Set.new(%w( - abs acos acosh adaptmesh adj AffineCG AffineGMRES arg asin asinh - assert atan atan2 atanh atof atoi BFGS broadcast buildlayers - buildmesh ceil chi complexEigenValue copysign change checkmovemesh - clock cmaes conj convect cos cosh cube d dd dfft diffnp diffpos - dimKrylov dist dumptable dx dxx dxy dxz dy dyx dyy dyz dz dzx dzy dzz - EigenValue emptymesh erf erfc exec exit exp fdim ffind find floor - flush fmax fmin fmod freeyams getARGV getline gmshload gmshload3 - gslcdfugaussianP gslcdfugaussianQ gslcdfugaussianPinv - gslcdfugaussianQinv gslcdfgaussianP gslcdfgaussianQ - gslcdfgaussianPinv gslcdfgaussianQinv gslcdfgammaP gslcdfgammaQ - gslcdfgammaPinv gslcdfgammaQinv gslcdfcauchyP gslcdfcauchyQ - gslcdfcauchyPinv gslcdfcauchyQinv gslcdflaplaceP gslcdflaplaceQ - gslcdflaplacePinv gslcdflaplaceQinv gslcdfrayleighP gslcdfrayleighQ - gslcdfrayleighPinv gslcdfrayleighQinv gslcdfchisqP gslcdfchisqQ - gslcdfchisqPinv gslcdfchisqQinv gslcdfexponentialP gslcdfexponentialQ - gslcdfexponentialPinv gslcdfexponentialQinv gslcdfexppowP - gslcdfexppowQ gslcdftdistP gslcdftdistQ gslcdftdistPinv - gslcdftdistQinv gslcdffdistP gslcdffdistQ gslcdffdistPinv - gslcdffdistQinv gslcdfbetaP gslcdfbetaQ gslcdfbetaPinv gslcdfbetaQinv - gslcdfflatP gslcdfflatQ gslcdfflatPinv gslcdfflatQinv - gslcdflognormalP gslcdflognormalQ gslcdflognormalPinv - gslcdflognormalQinv gslcdfgumbel1P gslcdfgumbel1Q gslcdfgumbel1Pinv - gslcdfgumbel1Qinv gslcdfgumbel2P gslcdfgumbel2Q gslcdfgumbel2Pinv - gslcdfgumbel2Qinv gslcdfweibullP gslcdfweibullQ gslcdfweibullPinv - gslcdfweibullQinv gslcdfparetoP gslcdfparetoQ gslcdfparetoPinv - gslcdfparetoQinv gslcdflogisticP gslcdflogisticQ gslcdflogisticPinv - gslcdflogisticQinv gslcdfbinomialP gslcdfbinomialQ gslcdfpoissonP - gslcdfpoissonQ gslcdfgeometricP gslcdfgeometricQ - gslcdfnegativebinomialP gslcdfnegativebinomialQ gslcdfpascalP - gslcdfpascalQ gslinterpakima gslinterpakimaperiodic - gslinterpcsplineperiodic gslinterpcspline gslinterpsteffen - gslinterplinear gslinterppolynomial gslranbernoullipdf gslranbeta - gslranbetapdf gslranbinomialpdf gslranexponential - gslranexponentialpdf gslranexppow gslranexppowpdf gslrancauchy - gslrancauchypdf gslranchisq gslranchisqpdf gslranerlang - gslranerlangpdf gslranfdist gslranfdistpdf gslranflat gslranflatpdf - gslrangamma gslrangammaint gslrangammapdf gslrangammamt - gslrangammaknuth gslrangaussian gslrangaussianratiomethod - gslrangaussianziggurat gslrangaussianpdf gslranugaussian - gslranugaussianratiomethod gslranugaussianpdf gslrangaussiantail - gslrangaussiantailpdf gslranugaussiantail gslranugaussiantailpdf - gslranlandau gslranlandaupdf gslrangeometricpdf gslrangumbel1 - gslrangumbel1pdf gslrangumbel2 gslrangumbel2pdf gslranlogistic - gslranlogisticpdf gslranlognormal gslranlognormalpdf - gslranlogarithmicpdf gslrannegativebinomialpdf gslranpascalpdf - gslranpareto gslranparetopdf gslranpoissonpdf gslranrayleigh - gslranrayleighpdf gslranrayleightail gslranrayleightailpdf - gslrantdist gslrantdistpdf gslranlaplace gslranlaplacepdf gslranlevy - gslranweibull gslranweibullpdf gslsfairyAi gslsfairyBi - gslsfairyAiscaled gslsfairyBiscaled gslsfairyAideriv gslsfairyBideriv - gslsfairyAiderivscaled gslsfairyBiderivscaled gslsfairyzeroAi - gslsfairyzeroBi gslsfairyzeroAideriv gslsfairyzeroBideriv - gslsfbesselJ0 gslsfbesselJ1 gslsfbesselJn gslsfbesselY0 gslsfbesselY1 - gslsfbesselYn gslsfbesselI0 gslsfbesselI1 gslsfbesselIn - gslsfbesselI0scaled gslsfbesselI1scaled gslsfbesselInscaled - gslsfbesselK0 gslsfbesselK1 gslsfbesselKn gslsfbesselK0scaled - gslsfbesselK1scaled gslsfbesselKnscaled gslsfbesselj0 gslsfbesselj1 - gslsfbesselj2 gslsfbesseljl gslsfbessely0 gslsfbessely1 gslsfbessely2 - gslsfbesselyl gslsfbesseli0scaled gslsfbesseli1scaled - gslsfbesseli2scaled gslsfbesselilscaled gslsfbesselk0scaled - gslsfbesselk1scaled gslsfbesselk2scaled gslsfbesselklscaled - gslsfbesselJnu gslsfbesselYnu gslsfbesselInuscaled gslsfbesselInu - gslsfbesselKnuscaled gslsfbesselKnu gslsfbessellnKnu - gslsfbesselzeroJ0 gslsfbesselzeroJ1 gslsfbesselzeroJnu gslsfclausen - gslsfhydrogenicR1 gslsfdawson gslsfdebye1 gslsfdebye2 gslsfdebye3 - gslsfdebye4 gslsfdebye5 gslsfdebye6 gslsfdilog gslsfmultiply - gslsfellintKcomp gslsfellintEcomp gslsfellintPcomp gslsfellintDcomp - gslsfellintF gslsfellintE gslsfellintRC gslsferfc gslsflogerfc - gslsferf gslsferfZ gslsferfQ gslsfhazard gslsfexp gslsfexpmult - gslsfexpm1 gslsfexprel gslsfexprel2 gslsfexpreln gslsfexpintE1 - gslsfexpintE2 gslsfexpintEn gslsfexpintE1scaled gslsfexpintE2scaled - gslsfexpintEnscaled gslsfexpintEi gslsfexpintEiscaled gslsfShi - gslsfChi gslsfexpint3 gslsfSi gslsfCi gslsfatanint gslsffermidiracm1 - gslsffermidirac0 gslsffermidirac1 gslsffermidirac2 gslsffermidiracint - gslsffermidiracmhalf gslsffermidirachalf gslsffermidirac3half - gslsffermidiracinc0 gslsflngamma gslsfgamma gslsfgammastar - gslsfgammainv gslsftaylorcoeff gslsffact gslsfdoublefact gslsflnfact - gslsflndoublefact gslsflnchoose gslsfchoose gslsflnpoch gslsfpoch - gslsfpochrel gslsfgammaincQ gslsfgammaincP gslsfgammainc gslsflnbeta - gslsfbeta gslsfbetainc gslsfgegenpoly1 gslsfgegenpoly2 - gslsfgegenpoly3 gslsfgegenpolyn gslsfhyperg0F1 gslsfhyperg1F1int - gslsfhyperg1F1 gslsfhypergUint gslsfhypergU gslsfhyperg2F0 - gslsflaguerre1 gslsflaguerre2 gslsflaguerre3 gslsflaguerren - gslsflambertW0 gslsflambertWm1 gslsflegendrePl gslsflegendreP1 - gslsflegendreP2 gslsflegendreP3 gslsflegendreQ0 gslsflegendreQ1 - gslsflegendreQl gslsflegendrePlm gslsflegendresphPlm - gslsflegendrearraysize gslsfconicalPhalf gslsfconicalPmhalf - gslsfconicalP0 gslsfconicalP1 gslsfconicalPsphreg gslsfconicalPcylreg - gslsflegendreH3d0 gslsflegendreH3d1 gslsflegendreH3d gslsflog - gslsflogabs gslsflog1plusx gslsflog1plusxmx gslsfpowint gslsfpsiint - gslsfpsi gslsfpsi1piy gslsfpsi1int gslsfpsi1 gslsfpsin - gslsfsynchrotron1 gslsfsynchrotron2 gslsftransport2 gslsftransport3 - gslsftransport4 gslsftransport5 gslsfsin gslsfcos gslsfhypot - gslsfsinc gslsflnsinh gslsflncosh gslsfanglerestrictsymm - gslsfanglerestrictpos gslsfzetaint gslsfzeta gslsfzetam1 - gslsfzetam1int gslsfhzeta gslsfetaint gslsfeta imag int1d int2d int3d - intalledges intallfaces interpolate invdiff invdiffnp invdiffpos - Isend isInf isNaN isoline Irecv j0 j1 jn jump lgamma LinearCG - LinearGMRES log log10 lrint lround max mean medit min mmg3d movemesh - movemesh23 mpiAlltoall mpiAlltoallv mpiAllgather mpiAllgatherv - mpiAllReduce mpiBarrier mpiGather mpiGatherv mpiRank mpiReduce - mpiScatter mpiScatterv mpiSize mpiWait mpiWaitAny mpiWtick mpiWtime - mshmet NLCG on plot polar Post pow processor processorblock - projection randinit randint31 randint32 random randreal1 randreal2 - randreal3 randres53 Read readmesh readmesh3 Recv rfind rint round - savemesh savesol savevtk seekg Sent set sign signbit sin sinh sort - splitComm splitmesh sqrt square srandom srandomdev Stringification - swap system tan tanh tellg tetg tetgconvexhull tetgreconstruction - tetgtransfo tgamma triangulate trunc Wait Write y0 y1 yn - )) - end - - def self.attributes - @builtinsParameters ||= Set.new(%w( - A A1 abserror absolute aniso aspectratio B B1 bb beginend bin - boundary bw close cmm coef composante cutoff datafilename dataname - dim distmax displacement doptions dparams eps err errg facemerge - facetcl factorize file fill fixedborder flabel flags floatmesh - floatsol fregion gradation grey hmax hmin holelist hsv init inquire - inside IsMetric iso ivalue keepbackvertices label labeldown labelmid - labelup levelset loptions lparams maxit maxsubdiv meditff mem memory - metric mode nbarrow nbiso nbiter nbjacoby nboffacetcl nbofholes - nbofregions nbregul nbsmooth nbvx ncv nev nomeshgeneration - normalization omega op optimize option options order orientation - periodic power precon prev ps ptmerge qfe qforder qft qfV ratio - rawvector reffacelow reffacemid reffaceup refnum reftet reftri region - regionlist renumv rescaling ridgeangle save sigma sizeofvolume - smoothing solver sparams split splitin2 splitpbedge stop strategy - swap switch sym t tgv thetamax tol tolpivot tolpivotsym transfo U2Vc - value varrow vector veps viso wait width withsurfacemesh WindowIndex - which zbound - )) - end + lazy { require_relative 'freefem/keywords' } id = /[a-z_]\w*/i @@ -217,22 +44,14 @@ def self.attributes rule %r/[()\[\],.;]/, Punctuation rule %r/\bcase\b/, Keyword, :case rule %r/(?:true|false|NaN)\b/, Name::Builtin - rule id do |m| - name = m[0] - if self.class.keywords.include? name - token Keyword - elsif self.class.keywords_type.include? name - token Keyword::Type - elsif self.class.reserved.include? name - token Keyword::Reserved - elsif self.class.builtins.include? name - token Name::Builtin - elsif self.class.attributes.include? name - token Name::Attribute - else - token Name - end + keywords id do + rule :keywords, Keyword + rule :keywords_type, Keyword::Type + rule :reserved, Keyword::Reserved + rule :builtins, Name::Builtin + rule :attributes, Name::Attribute + default Name end end end diff --git a/lib/rouge/lexers/freefem/keywords.rb b/lib/rouge/lexers/freefem/keywords.rb new file mode 100644 index 0000000000..9a38339e7c --- /dev/null +++ b/lib/rouge/lexers/freefem/keywords.rb @@ -0,0 +1,176 @@ +module Rouge + module Lexers + class FreeFEM + # Override C/C++ ones (for example, `do` does not exist) + def self.keywords + @keywords ||= Set.new %w(break catch continue else for if return try while) + end + + def self.keywords_type + @keywords_type ||= Set.new %w( + bool border complex dmatrix fespace func gslspline ifstream int macro + matrix mesh mesh3 mpiComm mpiGroup mpiRequest NewMacro EndMacro + ofstream Pmmap problem Psemaphore real solve string varf + ) + end + + def self.reserved + @reserved ||= Set.new %w( + BDM1 BDM1Ortho Edge03d Edge13d Edge23d FEQF HCT P0 P03d P0Edge P1 + P13d P1b P1b3d P1bl P1bl3d P1dc P1Edge P1nc P2 P23d P2b P2BR P2dc + P2Edge P2h P2Morley P2pnc P3 P3dc P3Edge P4 P4dc P4Edge P5Edge RT0 + RT03d RT0Ortho RT1 RT1Ortho RT2 RT2Ortho + + qf1pE qf1pElump qf1pT qf1pTlump qfV1 qfV1lump qf2pE qf2pT qf2pT4P1 + qfV2 qf3pE qf4pE qf5pE qf5pT qfV5 qf7pT qf9pT qfnbpE + + ARGV append area be binary BoundaryEdge bordermeasure CG Cholesky cin + cout Crout default diag edgeOrientation endl FILE fixed GMRES good + hTriangle im imax imin InternalEdge l1 l2 label lenEdge length LINE + linfty LU m max measure min mpiAnySource mpiBAND mpiBXOR mpiCommWorld + mpiLAND mpiLOR mpiLXOR mpiMAX mpiMIN mpiPROD mpirank mpisize mpiSUM + mpiUndefined n N nbe ndof ndofK noshowbase noshowpos notaregion nt + nTonEdge nuEdge nuTriangle nv P pi precision quantile re region + scientific searchMethod setw showbase showpos sparsesolver sum tellp + true UMFPACK unused whoinElement verbosity version volume x y z + ) + end + + def self.builtins + @builtins ||= Set.new %w( + abs acos acosh adaptmesh adj AffineCG AffineGMRES arg asin asinh + assert atan atan2 atanh atof atoi BFGS broadcast buildlayers + buildmesh ceil chi complexEigenValue copysign change checkmovemesh + clock cmaes conj convect cos cosh cube d dd dfft diffnp diffpos + dimKrylov dist dumptable dx dxx dxy dxz dy dyx dyy dyz dz dzx dzy dzz + EigenValue emptymesh erf erfc exec exit exp fdim ffind find floor + flush fmax fmin fmod freeyams getARGV getline gmshload gmshload3 + gslcdfugaussianP gslcdfugaussianQ gslcdfugaussianPinv + gslcdfugaussianQinv gslcdfgaussianP gslcdfgaussianQ + gslcdfgaussianPinv gslcdfgaussianQinv gslcdfgammaP gslcdfgammaQ + gslcdfgammaPinv gslcdfgammaQinv gslcdfcauchyP gslcdfcauchyQ + gslcdfcauchyPinv gslcdfcauchyQinv gslcdflaplaceP gslcdflaplaceQ + gslcdflaplacePinv gslcdflaplaceQinv gslcdfrayleighP gslcdfrayleighQ + gslcdfrayleighPinv gslcdfrayleighQinv gslcdfchisqP gslcdfchisqQ + gslcdfchisqPinv gslcdfchisqQinv gslcdfexponentialP gslcdfexponentialQ + gslcdfexponentialPinv gslcdfexponentialQinv gslcdfexppowP + gslcdfexppowQ gslcdftdistP gslcdftdistQ gslcdftdistPinv + gslcdftdistQinv gslcdffdistP gslcdffdistQ gslcdffdistPinv + gslcdffdistQinv gslcdfbetaP gslcdfbetaQ gslcdfbetaPinv gslcdfbetaQinv + gslcdfflatP gslcdfflatQ gslcdfflatPinv gslcdfflatQinv + gslcdflognormalP gslcdflognormalQ gslcdflognormalPinv + gslcdflognormalQinv gslcdfgumbel1P gslcdfgumbel1Q gslcdfgumbel1Pinv + gslcdfgumbel1Qinv gslcdfgumbel2P gslcdfgumbel2Q gslcdfgumbel2Pinv + gslcdfgumbel2Qinv gslcdfweibullP gslcdfweibullQ gslcdfweibullPinv + gslcdfweibullQinv gslcdfparetoP gslcdfparetoQ gslcdfparetoPinv + gslcdfparetoQinv gslcdflogisticP gslcdflogisticQ gslcdflogisticPinv + gslcdflogisticQinv gslcdfbinomialP gslcdfbinomialQ gslcdfpoissonP + gslcdfpoissonQ gslcdfgeometricP gslcdfgeometricQ + gslcdfnegativebinomialP gslcdfnegativebinomialQ gslcdfpascalP + gslcdfpascalQ gslinterpakima gslinterpakimaperiodic + gslinterpcsplineperiodic gslinterpcspline gslinterpsteffen + gslinterplinear gslinterppolynomial gslranbernoullipdf gslranbeta + gslranbetapdf gslranbinomialpdf gslranexponential + gslranexponentialpdf gslranexppow gslranexppowpdf gslrancauchy + gslrancauchypdf gslranchisq gslranchisqpdf gslranerlang + gslranerlangpdf gslranfdist gslranfdistpdf gslranflat gslranflatpdf + gslrangamma gslrangammaint gslrangammapdf gslrangammamt + gslrangammaknuth gslrangaussian gslrangaussianratiomethod + gslrangaussianziggurat gslrangaussianpdf gslranugaussian + gslranugaussianratiomethod gslranugaussianpdf gslrangaussiantail + gslrangaussiantailpdf gslranugaussiantail gslranugaussiantailpdf + gslranlandau gslranlandaupdf gslrangeometricpdf gslrangumbel1 + gslrangumbel1pdf gslrangumbel2 gslrangumbel2pdf gslranlogistic + gslranlogisticpdf gslranlognormal gslranlognormalpdf + gslranlogarithmicpdf gslrannegativebinomialpdf gslranpascalpdf + gslranpareto gslranparetopdf gslranpoissonpdf gslranrayleigh + gslranrayleighpdf gslranrayleightail gslranrayleightailpdf + gslrantdist gslrantdistpdf gslranlaplace gslranlaplacepdf gslranlevy + gslranweibull gslranweibullpdf gslsfairyAi gslsfairyBi + gslsfairyAiscaled gslsfairyBiscaled gslsfairyAideriv gslsfairyBideriv + gslsfairyAiderivscaled gslsfairyBiderivscaled gslsfairyzeroAi + gslsfairyzeroBi gslsfairyzeroAideriv gslsfairyzeroBideriv + gslsfbesselJ0 gslsfbesselJ1 gslsfbesselJn gslsfbesselY0 gslsfbesselY1 + gslsfbesselYn gslsfbesselI0 gslsfbesselI1 gslsfbesselIn + gslsfbesselI0scaled gslsfbesselI1scaled gslsfbesselInscaled + gslsfbesselK0 gslsfbesselK1 gslsfbesselKn gslsfbesselK0scaled + gslsfbesselK1scaled gslsfbesselKnscaled gslsfbesselj0 gslsfbesselj1 + gslsfbesselj2 gslsfbesseljl gslsfbessely0 gslsfbessely1 gslsfbessely2 + gslsfbesselyl gslsfbesseli0scaled gslsfbesseli1scaled + gslsfbesseli2scaled gslsfbesselilscaled gslsfbesselk0scaled + gslsfbesselk1scaled gslsfbesselk2scaled gslsfbesselklscaled + gslsfbesselJnu gslsfbesselYnu gslsfbesselInuscaled gslsfbesselInu + gslsfbesselKnuscaled gslsfbesselKnu gslsfbessellnKnu + gslsfbesselzeroJ0 gslsfbesselzeroJ1 gslsfbesselzeroJnu gslsfclausen + gslsfhydrogenicR1 gslsfdawson gslsfdebye1 gslsfdebye2 gslsfdebye3 + gslsfdebye4 gslsfdebye5 gslsfdebye6 gslsfdilog gslsfmultiply + gslsfellintKcomp gslsfellintEcomp gslsfellintPcomp gslsfellintDcomp + gslsfellintF gslsfellintE gslsfellintRC gslsferfc gslsflogerfc + gslsferf gslsferfZ gslsferfQ gslsfhazard gslsfexp gslsfexpmult + gslsfexpm1 gslsfexprel gslsfexprel2 gslsfexpreln gslsfexpintE1 + gslsfexpintE2 gslsfexpintEn gslsfexpintE1scaled gslsfexpintE2scaled + gslsfexpintEnscaled gslsfexpintEi gslsfexpintEiscaled gslsfShi + gslsfChi gslsfexpint3 gslsfSi gslsfCi gslsfatanint gslsffermidiracm1 + gslsffermidirac0 gslsffermidirac1 gslsffermidirac2 gslsffermidiracint + gslsffermidiracmhalf gslsffermidirachalf gslsffermidirac3half + gslsffermidiracinc0 gslsflngamma gslsfgamma gslsfgammastar + gslsfgammainv gslsftaylorcoeff gslsffact gslsfdoublefact gslsflnfact + gslsflndoublefact gslsflnchoose gslsfchoose gslsflnpoch gslsfpoch + gslsfpochrel gslsfgammaincQ gslsfgammaincP gslsfgammainc gslsflnbeta + gslsfbeta gslsfbetainc gslsfgegenpoly1 gslsfgegenpoly2 + gslsfgegenpoly3 gslsfgegenpolyn gslsfhyperg0F1 gslsfhyperg1F1int + gslsfhyperg1F1 gslsfhypergUint gslsfhypergU gslsfhyperg2F0 + gslsflaguerre1 gslsflaguerre2 gslsflaguerre3 gslsflaguerren + gslsflambertW0 gslsflambertWm1 gslsflegendrePl gslsflegendreP1 + gslsflegendreP2 gslsflegendreP3 gslsflegendreQ0 gslsflegendreQ1 + gslsflegendreQl gslsflegendrePlm gslsflegendresphPlm + gslsflegendrearraysize gslsfconicalPhalf gslsfconicalPmhalf + gslsfconicalP0 gslsfconicalP1 gslsfconicalPsphreg gslsfconicalPcylreg + gslsflegendreH3d0 gslsflegendreH3d1 gslsflegendreH3d gslsflog + gslsflogabs gslsflog1plusx gslsflog1plusxmx gslsfpowint gslsfpsiint + gslsfpsi gslsfpsi1piy gslsfpsi1int gslsfpsi1 gslsfpsin + gslsfsynchrotron1 gslsfsynchrotron2 gslsftransport2 gslsftransport3 + gslsftransport4 gslsftransport5 gslsfsin gslsfcos gslsfhypot + gslsfsinc gslsflnsinh gslsflncosh gslsfanglerestrictsymm + gslsfanglerestrictpos gslsfzetaint gslsfzeta gslsfzetam1 + gslsfzetam1int gslsfhzeta gslsfetaint gslsfeta imag int1d int2d int3d + intalledges intallfaces interpolate invdiff invdiffnp invdiffpos + Isend isInf isNaN isoline Irecv j0 j1 jn jump lgamma LinearCG + LinearGMRES log log10 lrint lround max mean medit min mmg3d movemesh + movemesh23 mpiAlltoall mpiAlltoallv mpiAllgather mpiAllgatherv + mpiAllReduce mpiBarrier mpiGather mpiGatherv mpiRank mpiReduce + mpiScatter mpiScatterv mpiSize mpiWait mpiWaitAny mpiWtick mpiWtime + mshmet NLCG on plot polar Post pow processor processorblock + projection randinit randint31 randint32 random randreal1 randreal2 + randreal3 randres53 Read readmesh readmesh3 Recv rfind rint round + savemesh savesol savevtk seekg Sent set sign signbit sin sinh sort + splitComm splitmesh sqrt square srandom srandomdev Stringification + swap system tan tanh tellg tetg tetgconvexhull tetgreconstruction + tetgtransfo tgamma triangulate trunc Wait Write y0 y1 yn + ) + end + + def self.attributes + @attributes ||= Set.new %w( + A A1 abserror absolute aniso aspectratio B B1 bb beginend bin + boundary bw close cmm coef composante cutoff datafilename dataname + dim distmax displacement doptions dparams eps err errg facemerge + facetcl factorize file fill fixedborder flabel flags floatmesh + floatsol fregion gradation grey hmax hmin holelist hsv init inquire + inside IsMetric iso ivalue keepbackvertices label labeldown labelmid + labelup levelset loptions lparams maxit maxsubdiv meditff mem memory + metric mode nbarrow nbiso nbiter nbjacoby nboffacetcl nbofholes + nbofregions nbregul nbsmooth nbvx ncv nev nomeshgeneration + normalization omega op optimize option options order orientation + periodic power precon prev ps ptmerge qfe qforder qft qfV ratio + rawvector reffacelow reffacemid reffaceup refnum reftet reftri region + regionlist renumv rescaling ridgeangle save sigma sizeofvolume + smoothing solver sparams split splitin2 splitpbedge stop strategy + swap switch sym t tgv thetamax tol tolpivot tolpivotsym transfo U2Vc + value varrow vector veps viso wait width withsurfacemesh WindowIndex + which zbound + ) + end + end + end +end From 7b18a47682e42f4a32fd16dc27a5bbe5687c4494 Mon Sep 17 00:00:00 2001 From: jneen Date: Tue, 14 Apr 2026 11:42:18 -0400 Subject: [PATCH 050/154] use the keywords api for fsharp --- lib/rouge/lexers/fsharp.rb | 28 ++++++++++------------------ spec/visual/samples/fsharp | 4 ++++ 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/lib/rouge/lexers/fsharp.rb b/lib/rouge/lexers/fsharp.rb index 8640b6c485..3dd5678284 100644 --- a/lib/rouge/lexers/fsharp.rb +++ b/lib/rouge/lexers/fsharp.rb @@ -52,26 +52,18 @@ def self.primitives rule upper_id, Name::Class rule %r/[(][*](?![)])/, Comment, :comment rule %r(//.*?$), Comment::Single - rule id do |m| - match = m[0] - if self.class.keywords.include? match - token Keyword - elsif self.class.word_operators.include? match - token Operator::Word - elsif self.class.primitives.include? match - token Keyword::Type - else - token Name - end + + keywords %r/\w+!?/ do + rule :keywords, Keyword + rule :word_operators, Operator::Word + rule :primitives, Keyword::Type end - rule operator do |m| - match = m[0] - if self.class.keyopts.include? match - token Punctuation - else - token Operator - end + rule id, Name + + keywords operator do + rule :keyopts, Punctuation + default Operator end rule %r/-?\d[\d_]*(.[\d_]*)?(e[+-]?\d[\d_]*)/i, Num::Float diff --git a/spec/visual/samples/fsharp b/spec/visual/samples/fsharp index 2307f9e12f..2cb6df62c6 100644 --- a/spec/visual/samples/fsharp +++ b/spec/visual/samples/fsharp @@ -202,5 +202,9 @@ finally let value = array.[0] let value = dict.Test.["key"] +let! value = 3 +yield! 44 +thing! 5 + // syntax highlighting let number = 123 From f6b95aff6edae2bcc6fa872f67bd7e681fea5d61 Mon Sep 17 00:00:00 2001 From: jneen Date: Tue, 14 Apr 2026 11:59:19 -0400 Subject: [PATCH 051/154] use the keywords api for gdscript --- lib/rouge/lexers/gdscript.rb | 106 ++++++++++++++++------------------- 1 file changed, 47 insertions(+), 59 deletions(-) diff --git a/lib/rouge/lexers/gdscript.rb b/lib/rouge/lexers/gdscript.rb index 123bbc0505..aa43067893 100644 --- a/lib/rouge/lexers/gdscript.rb +++ b/lib/rouge/lexers/gdscript.rb @@ -11,42 +11,42 @@ class GDScript < RegexLexer filenames '*.gd' mimetypes 'text/x-gdscript', 'application/x-gdscript' - def self.keywords - @keywords ||= %w( - and in not or as breakpoint class class_name extends is func setget - signal tool const enum export onready static var break continue - if elif else for pass return match while remote master puppet - remotesync mastersync puppetsync - ).join('|') - end + KEYWORDS = Set.new %w( + and in not or as breakpoint class class_name extends is func setget + signal tool const enum export onready static var break continue + if elif else for pass return match while remote master puppet + remotesync mastersync puppetsync + ) # Reserved for future implementation - def self.keywords_reserved - @keywords_reserved ||= %w( - do switch case - ).join('|') - end - - def self.builtins - @builtins ||= %w( - Color8 ColorN abs acos asin assert atan atan2 bytes2var ceil char - clamp convert cos cosh db2linear decimals dectime deg2rad dict2inst - ease exp floor fmod fposmod funcref hash inst2dict instance_from_id - is_inf is_nan lerp linear2db load log max min nearest_po2 pow - preload print print_stack printerr printraw prints printt rad2deg - rand_range rand_seed randf randi randomize range round seed sign - sin sinh sqrt stepify str str2var tan tan tanh type_exist typeof - var2bytes var2str weakref yield - ).join('|') - end - - def self.builtins_type - @builtins_type ||= %w( - bool int float String Vector2 Rect2 Transform2D Vector3 AABB - Plane Quat Basis Transform Color RID Object NodePath Dictionary - Array PoolByteArray PoolIntArray PoolRealArray PoolStringArray - PoolVector2Array PoolVector3Array PoolColorArray null - ).join('|') + KEYWORDS_RESERVED = Set.new %w(do switch case) + + BUILTINS = Set.new %w( + Color8 ColorN abs acos asin assert atan atan2 bytes2var ceil char + clamp convert cos cosh db2linear decimals dectime deg2rad dict2inst + ease exp floor fmod fposmod funcref hash inst2dict instance_from_id + is_inf is_nan lerp linear2db load log max min nearest_po2 pow + preload print print_stack printerr printraw prints printt rad2deg + rand_range rand_seed randf randi randomize range round seed sign + sin sinh sqrt stepify str str2var tan tan tanh type_exist typeof + var2bytes var2str weakref yield + ) + + CONSTANTS = Set.new %w(self false true PI TAU NAN INF) + + BUILTINS_TYPE = Set.new %w( + bool int float String Vector2 Rect2 Transform2D Vector3 AABB + Plane Quat Basis Transform Color RID Object NodePath Dictionary + Array PoolByteArray PoolIntArray PoolRealArray PoolStringArray + PoolVector2Array PoolVector3Array PoolColorArray null + ) + + state :inline_whitespace do + rule %r/[ \t]+/, Text + rule %r/\\[ \t]*\n/, Str::Escape + rule %r/(\\[ \t]*)(#.*?\n)/ do + groups Str::Escape, Comment + end end state :root do @@ -57,35 +57,25 @@ def self.builtins_type rule %r/\\\n/, Text rule %r/(in|and|or|not)\b/, Operator::Word rule %r/!=|==|<<|>>|&&|\+=|-=|\*=|\/=|%=|&=|\|=|\|\||[-~+\/*%=<>&^.!|$]/, Operator - rule %r/(func)((?:\s|\\)+)/ do - groups Keyword, Text - push :funcname - end - rule %r/(class)((?:\s|\\)+)/ do - groups Keyword, Text - push :classname + rule %r/func\b/, Keyword, :funcname + rule %r/class\b/, Keyword, :classname + + keywords %r/[a-z]\w*/i do + rule KEYWORDS, Keyword + rule KEYWORDS_RESERVED, Keyword::Reserved + rule BUILTINS, Name::Builtin + rule CONSTANTS, Name::Constant + rule BUILTINS_TYPE, Keyword::Type + default Name end - mixin :keywords - mixin :builtins + rule %r/"""/, Str::Double, :escape_tdqs rule %r/'''/, Str::Double, :escape_tsqs rule %r/"/, Str::Double, :escape_dqs rule %r/'/, Str::Double, :escape_sqs - mixin :name mixin :numbers end - state :keywords do - rule %r/\b(#{GDScript.keywords})\b/, Keyword - rule %r/\b(#{GDScript.keywords_reserved})\b/, Keyword::Reserved - end - - state :builtins do - rule %r/\b(#{GDScript.builtins})\b/, Name::Builtin - rule %r/\b((self|false|true)|(PI|TAU|NAN|INF))\b/, Name::Builtin::Pseudo - rule %r/\b(#{GDScript.builtins_type})\b/, Keyword::Type - end - state :numbers do rule %r/(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?j?/, Num::Float rule %r/\d+[eE][+-]?[0-9]+j?/, Num::Float @@ -93,15 +83,13 @@ def self.builtins_type rule %r/\d+j?/, Num::Integer end - state :name do - rule %r/[a-zA-Z_]\w*/, Name - end - state :funcname do + mixin :inline_whitespace rule %r/[a-zA-Z_]\w*/, Name::Function, :pop! end state :classname do + mixin :inline_whitespace rule %r/[a-zA-Z_]\w*/, Name::Class, :pop! end From f66ecaea512a0e17235a157d2c15721e29352679 Mon Sep 17 00:00:00 2001 From: jneen Date: Tue, 14 Apr 2026 22:13:25 -0400 Subject: [PATCH 052/154] use the keywords api for glsl --- lib/rouge/lexers/glsl.rb | 32 ++++++++++---------------------- 1 file changed, 10 insertions(+), 22 deletions(-) diff --git a/lib/rouge/lexers/glsl.rb b/lib/rouge/lexers/glsl.rb index efa3d834b9..21f6efd382 100644 --- a/lib/rouge/lexers/glsl.rb +++ b/lib/rouge/lexers/glsl.rb @@ -33,32 +33,20 @@ class GLSL < RegexLexer rule %r/0/, Num::Integer rule %r([-.+/*%<>\[\](){}^|&~=!:;,?]), Punctuation - rule %r/\w+/ do |m| - if KEYWORDS.include?(m[0]) - token Keyword - elsif TYPES.include?(m[0]) - token Keyword::Type - else - fallthrough! - end + keywords %r/\w+/ do + rule KEYWORDS, Keyword + rule TYPES, Keyword::Type end - rule %r/\w+(?=\s*[(])/ do |m| - if BUILTINS.include?(m[0]) - token Name::Builtin - else - token Name::Function - end + keywords %r/\w+(?=\s*[(])/ do + rule BUILTINS, Name::Builtin + default Name::Function end - rule %r/\w+/ do |m| - if GL_VARS.include?(m[0]) - token Name::Constant - elsif BUILTINS.include?(m[0]) - token Name::Builtin - else - token Name - end + keywords %r/\w+/ do + rule GL_VARS, Name::Constant + rule BUILTINS, Name::Builtin + default Name end end end From f12a6b996e188a8fc140cb90fdcf23e6f5948efb Mon Sep 17 00:00:00 2001 From: jneen Date: Tue, 14 Apr 2026 22:17:55 -0400 Subject: [PATCH 053/154] remove duplicate set element from gdscript --- lib/rouge/lexers/gdscript.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/rouge/lexers/gdscript.rb b/lib/rouge/lexers/gdscript.rb index aa43067893..05174439a4 100644 --- a/lib/rouge/lexers/gdscript.rb +++ b/lib/rouge/lexers/gdscript.rb @@ -28,7 +28,7 @@ class GDScript < RegexLexer is_inf is_nan lerp linear2db load log max min nearest_po2 pow preload print print_stack printerr printraw prints printt rad2deg rand_range rand_seed randf randi randomize range round seed sign - sin sinh sqrt stepify str str2var tan tan tanh type_exist typeof + sin sinh sqrt stepify str str2var tan tanh type_exist typeof var2bytes var2str weakref yield ) From 1ecf01ef34338b446a5aa80f67959da40515c018 Mon Sep 17 00:00:00 2001 From: jneen Date: Tue, 14 Apr 2026 22:18:36 -0400 Subject: [PATCH 054/154] use keywords api for groovy/gradle --- lib/rouge/lexers/groovy.rb | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/lib/rouge/lexers/groovy.rb b/lib/rouge/lexers/groovy.rb index d501f50d92..462b934f09 100644 --- a/lib/rouge/lexers/groovy.rb +++ b/lib/rouge/lexers/groovy.rb @@ -82,18 +82,13 @@ def self.constants end rule %r/[a-zA-Z_][a-zA-Z0-9_]*:/, Name::Label - rule %r/[a-zA-Z_\$][a-zA-Z0-9_]*/ do |m| - if self.class.keywords.include? m[0] - token Keyword - elsif self.class.declarations.include? m[0] - token Keyword::Declaration - elsif self.class.types.include? m[0] - token Keyword::Type - elsif self.class.constants.include? m[0] - token Keyword::Constant - else - token Name - end + + keywords %r/[a-zA-Z_\$][a-zA-Z0-9_]*/ do + rule :keywords, Keyword + rule :declarations, Keyword::Declaration + rule :types, Keyword::Type + rule :constants, Keyword::Constant + default Name end rule %r([~^*!%&\[\](){}<>\|+=:;,./?-]), Operator From 7f65fb8ce2803c129b8e33d72e386c51901e6680 Mon Sep 17 00:00:00 2001 From: jneen Date: Tue, 14 Apr 2026 23:46:14 -0400 Subject: [PATCH 055/154] use the keywords api for haskell --- .rubocop_todo.yml | 1 - lib/rouge/lexers/haskell.rb | 42 +++++++++++++++++-------------------- spec/visual/samples/haskell | 7 +++++-- 3 files changed, 24 insertions(+), 26 deletions(-) diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index 32afc73b8b..89f078a428 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -43,7 +43,6 @@ Naming/VariableName: Rouge/NoBuildingAlternationPatternInRegexp: Exclude: - 'lib/rouge/lexers/armasm.rb' - - 'lib/rouge/lexers/haskell.rb' - 'lib/rouge/lexers/http.rb' - 'lib/rouge/lexers/idris.rb' - 'lib/rouge/lexers/isabelle.rb' diff --git a/lib/rouge/lexers/haskell.rb b/lib/rouge/lexers/haskell.rb index 6891847d15..f9840fbd7b 100644 --- a/lib/rouge/lexers/haskell.rb +++ b/lib/rouge/lexers/haskell.rb @@ -16,15 +16,16 @@ def self.detect?(text) return true if text.shebang?('runhaskell') end - reserved = %w( + RESERVED = Set.new %w( _ case class data default deriving do else if in infix infixl infixr instance let newtype of then type where ) - ascii = %w( - NUL SOH [SE]TX EOT ENQ ACK BEL BS HT LF VT FF CR S[OI] DLE - DC[1-4] NAK SYN ETB CAN EM SUB ESC [FGRU]S SP DEL - ) + ASCII_ESCAPE = %r( + (?: NUL | SOH | [SE]TX | EOT | ENQ | ACK | BEL | BS | HT | LF | VT | FF | CR + | S[OI] | DLE | DC[1-4] | NAK | SYN | ETB | CAN | EM | SUB | ESC | [FGRU]S + | SP | DEL ) + )ox state :basic do rule %r/\s+/m, Text @@ -63,20 +64,17 @@ def self.detect?(text) rule %r/0x[\da-f]+/i, Num::Hex rule %r/\d+/, Num::Integer - rule %r/[\w']+/ do |m| - match = m[0] - if match == "import" - token Keyword::Reserved - push :import - elsif match == "module" - token Keyword::Reserved - push :module - elsif reserved.include?(match) - token Keyword::Reserved - elsif match =~ /\A'?[A-Z]/ - token Keyword::Type - else - token Name + keywords %r/[\w']+/ do + rule Set['import'], Keyword::Reserved, :import + rule Set['module'], Keyword::Reserved, :module + rule RESERVED, Keyword::Reserved + + default do |m| + if m[0].match?(/\A'?[A-Z]/o) + token Keyword::Type + else + token Name + end end end @@ -94,9 +92,7 @@ def self.detect?(text) # Quasiquotations rule %r/(\[)([_a-z][\w']*)(\|)/ do |m| - token Operator, m[1] - token Name, m[2] - token Operator, m[3] + groups Operator, Name, Operator push :quasiquotation end @@ -196,7 +192,7 @@ def self.detect?(text) state :escape do rule %r/[abfnrtv"'&\\]/, Str::Escape, :pop! rule %r/\^[\]\[A-Z@\^_]/, Str::Escape, :pop! - rule %r/#{ascii.join('|')}/, Str::Escape, :pop! + rule ASCII_ESCAPE, Str::Escape, :pop! rule %r/o[0-7]+/i, Str::Escape, :pop! rule %r/x[\da-f]+/i, Str::Escape, :pop! rule %r/\d+/, Str::Escape, :pop! diff --git a/spec/visual/samples/haskell b/spec/visual/samples/haskell index 6d7a9e1f3e..29a965645a 100644 --- a/spec/visual/samples/haskell +++ b/spec/visual/samples/haskell @@ -388,8 +388,11 @@ foo :: String foo = replicate n '\x1f363' where n = 32 -bar :: String -bar = "\x1f389\x1f363\x1f389" +bar :: Int -> String +bar 1 = "\x1f389\x1f363\x1f389" +bar 2 = "\NUL" +bar 3 = "\nul" -- this one should be a newline followed by ul +bar 4 = "\DC1ABC" -- escape should be DC1 quasi1 = [quoter|Some quoted text|] From f5cbc893e4ab5820383521fa0e775a37a088dd40 Mon Sep 17 00:00:00 2001 From: jneen Date: Tue, 14 Apr 2026 23:49:19 -0400 Subject: [PATCH 056/154] use the keywords api for haxe --- lib/rouge/lexers/haxe.rb | 31 +++++++++---------------------- 1 file changed, 9 insertions(+), 22 deletions(-) diff --git a/lib/rouge/lexers/haxe.rb b/lib/rouge/lexers/haxe.rb index 1665ac84c7..e1d07d1d75 100644 --- a/lib/rouge/lexers/haxe.rb +++ b/lib/rouge/lexers/haxe.rb @@ -47,7 +47,7 @@ def self.constants end def self.builtins - @builtins ||= %w( + @builtins ||= Set.new %w( Void Dynamic Math Class Any Float Int UInt String StringTools Sys EReg isNaN parseFloat parseInt this Array Map Date DateTools Bool Lambda Reflect Std File FileSystem @@ -157,27 +157,14 @@ def self.builtins push :expr_start end - rule id do |m| - match = m[0] - - if self.class.imports.include?(match) - token Keyword::Namespace - push :namespace - elsif self.class.keywords.include?(match) - token Keyword - push :expr_start - elsif self.class.declarations.include?(match) - token Keyword::Declaration - push :expr_start - elsif self.class.reserved.include?(match) - token Keyword::Reserved - elsif self.class.constants.include?(match) - token Keyword::Constant - elsif self.class.builtins.include?(match) - token Name::Builtin - else - token Name::Other - end + keywords id do + rule :imports, Keyword::Namespace, :namespace + rule :keywords, Keyword, :expr_start + rule :declarations, Keyword::Declaration, :expr_start + rule :reserved, Keyword::Reserved + rule :constants, Keyword::Constant + rule :builtins, Name::Builtin + default Name::Other end rule %r/\-?\d+\.\d+(?:[eE]\d+)?[fd]?/, Num::Float From 7d04864b553b75b38c44edd9995b2539cfb94f7c Mon Sep 17 00:00:00 2001 From: jneen Date: Thu, 16 Apr 2026 13:05:02 -0400 Subject: [PATCH 057/154] rework the hlsl lexer --- lib/rouge/lexers/hlsl.rb | 215 ++++++++++-------------------- lib/rouge/lexers/hlsl/keywords.rb | 131 ++++++++++++++++++ spec/visual/samples/hlsl | 207 ++++++++++++++++++++++++++++ 3 files changed, 412 insertions(+), 141 deletions(-) create mode 100644 lib/rouge/lexers/hlsl/keywords.rb diff --git a/lib/rouge/lexers/hlsl.rb b/lib/rouge/lexers/hlsl.rb index 171f28e8db..ef2c892f7a 100644 --- a/lib/rouge/lexers/hlsl.rb +++ b/lib/rouge/lexers/hlsl.rb @@ -1,165 +1,98 @@ # -*- coding: utf-8 -*- # # frozen_string_literal: true -require_relative 'c' - module Rouge module Lexers - class HLSL < C + class HLSL < RegexLexer title "HLSL" desc "HLSL, the High Level Shading Language for DirectX (docs.microsoft.com/en-us/windows/win32/direct3dhlsl/dx-graphics-hlsl)" tag 'hlsl' filenames '*.hlsl', '*.hlsli' mimetypes 'text/x-hlsl' - def self.keywords - @keywords ||= Set.new %w( - asm asm_fragment break case cbuffer centroid class column_major - compile compile_fragment const continue default discard do else export - extern for fxgroup globallycoherent groupshared if in inline inout - interface line lineadj linear namespace nointerpolation noperspective - NULL out packoffset pass pixelfragment point precise return register - row_major sample sampler shared stateblock stateblock_state static - struct switch tbuffer technique technique10 technique11 texture - typedef triangle uniform vertexfragment volatile while - ) - end + lazy { require_relative 'hlsl/keywords' } + + id = /[a-zA-Z_][a-zA-Z0-9_]*/ + + start { push :newline } - def self.keywords_type - @keywords_type ||= Set.new %w( - dword matrix snorm string unorm unsigned void vector BlendState Buffer - ByteAddressBuffer ComputeShader DepthStencilState DepthStencilView - DomainShader GeometryShader HullShader InputPatch LineStream - OutputPatch PixelShader PointStream RasterizerState RenderTargetView - RasterizerOrderedBuffer RasterizerOrderedByteAddressBuffer - RasterizerOrderedStructuredBuffer RasterizerOrderedTexture1D - RasterizerOrderedTexture1DArray RasterizerOrderedTexture2D - RasterizerOrderedTexture2DArray RasterizerOrderedTexture3D RWBuffer - RWByteAddressBuffer RWStructuredBuffer RWTexture1D RWTexture1DArray - RWTexture2D RWTexture2DArray RWTexture3D SamplerState - SamplerComparisonState StructuredBuffer Texture1D Texture1DArray - Texture2D Texture2DArray Texture2DMS Texture2DMSArray Texture3D - TextureCube TextureCubeArray TriangleStream VertexShader - - bool1 bool2 bool3 bool4 BOOL1 BOOL2 BOOL3 BOOL4 - int1 int2 int3 int4 - half1 half2 half3 half4 - float1 float2 float3 float4 - double1 double2 double3 double4 - - bool1x1 bool1x2 bool1x3 bool1x4 bool2x1 bool2x2 bool2x3 bool2x4 - bool3x1 bool3x2 bool3x3 bool3x4 bool4x1 bool4x2 bool4x3 bool4x4 - BOOL1x1 BOOL1x2 BOOL1x3 BOOL1x4 BOOL2x1 BOOL2x2 BOOL2x3 BOOL2x4 - BOOL3x1 BOOL3x2 BOOL3x3 BOOL3x4 BOOL4x1 BOOL4x2 BOOL4x3 BOOL4x4 - half1x1 half1x2 half1x3 half1x4 half2x1 half2x2 half2x3 half2x4 - half3x1 half3x2 half3x3 half3x4 half4x1 half4x2 half4x3 half4x4 - int1x1 int1x2 int1x3 int1x4 int2x1 int2x2 int2x3 int2x4 - int3x1 int3x2 int3x3 int3x4 int4x1 int4x2 int4x3 int4x4 - float1x1 float1x2 float1x3 float1x4 float2x1 float2x2 float2x3 float2x4 - float3x1 float3x2 float3x3 float3x4 float4x1 float4x2 float4x3 float4x4 - double1x1 double1x2 double1x3 double1x4 double2x1 double2x2 double2x3 double2x4 - double3x1 double3x2 double3x3 double3x4 double4x1 double4x2 double4x3 double4x4 - ) + state :inline_whitespace do + rule %r/[ \t]+/m, Text + rule %r/\\\n/, Str::Escape end - def self.reserved - @reserved ||= Set.new %w( - auto catch char const_cast delete dynamic_cast enum explicit friend - goto long mutable new operator private protected public - reinterpret_cast short signed sizeof static_cast template this throw - try typename union unsigned using virtual - ) + state :whitespace do + mixin :inline_whitespace + rule %r(//.*\n), Comment, :newline + rule %r(/[*].*?[*]/)m, Comment::Multiline + rule %r/\n\s*/, Text, :newline end - def self.builtins - @builtins ||= Set.new %w( - abort abs acos all AllMemoryBarrier AllMemoryBarrierWithGroupSync any - AppendStructuredBuffer asdouble asfloat asin asint asuint atan - atan2 ceil CheckAccessFullyMapped clamp clip CompileShader - ConsumeStructuredBuffer cos cosh countbits cross D3DCOLORtoUBYTE4 ddx - ddx_coarse ddx_fine ddy ddy_coarse ddy_fine degrees determinant - DeviceMemoryBarrier DeviceMemoryBarrierWithGroupSync distance dot dst - errorf EvaluateAttributeAtCentroid EvaluateAttributeAtSample - EvaluateAttributeSnapped exp exp2 f16tof32 f32tof16 faceforward - firstbithigh firstbitlow floor fma fmod frac frexp fwidth - GetRenderTargetSampleCount GetRenderTargetSamplePosition - GlobalOrderedCountIncrement GroupMemoryBarrier - GroupMemoryBarrierWithGroupSync InterlockedAdd InterlockedAnd - InterlockedCompareExchange InterlockedCompareStore InterlockedExchange - InterlockedMax InterlockedMin InterlockedOr InterlockedXor isfinite - isinf isnan ldexp length lerp lit log log10 log2 mad max min modf - msad4 mul noise normalize pow printf Process2DQuadTessFactorsAvg - Process2DQuadTessFactorsMax Process2DQuadTessFactorsMin - ProcessIsolineTessFactors ProcessQuadTessFactorsAvg - ProcessQuadTessFactorsMax ProcessQuadTessFactorsMin - ProcessTriTessFactorsAvg ProcessTriTessFactorsMax - ProcessTriTessFactorsMin QuadReadLaneAt QuadSwapX QuadSwapY radians - rcp reflect refract reversebits round rsqrt saturate sign sin sincos - sinh smoothstep sqrt step tan tanh tex1D tex1Dbias tex1Dgrad - tex1Dlod tex1Dproj tex2D tex2Dbias tex2Dgrad tex2Dlod tex2Dproj - tex3D tex3Dbias tex3Dgrad tex3Dlod tex3Dproj texCUBE - texCUBEbias texCUBEgrad texCUBElod texCUBEproj transpose trunc - WaveAllBitAnd WaveAllMax WaveAllMin WaveAllBitOr WaveAllBitXor - WaveAllEqual WaveAllProduct WaveAllSum WaveAllTrue WaveAnyTrue - WaveBallot WaveGetLaneCount WaveGetLaneIndex WaveGetOrderedIndex - WaveIsHelperLane WaveOnce WavePrefixProduct WavePrefixSum - WaveReadFirstLane WaveReadLaneAt - - SV_CLIPDISTANCE SV_CLIPDISTANCE0 SV_CLIPDISTANCE1 SV_CULLDISTANCE - SV_CULLDISTANCE0 SV_CULLDISTANCE1 SV_COVERAGE SV_DEPTH - SV_DEPTHGREATEREQUAL SV_DEPTHLESSEQUAL SV_DISPATCHTHREADID - SV_DOMAINLOCATION SV_GROUPID SV_GROUPINDEX SV_GROUPTHREADID - SV_GSINSTANCEID SV_INNERCOVERAGE SV_INSIDETESSFACTOR SV_INSTANCEID - SV_ISFRONTFACE SV_OUTPUTCONTROLPOINTID SV_POSITION SV_PRIMITIVEID - SV_RENDERTARGETARRAYINDEX SV_SAMPLEINDEX SV_STENCILREF SV_TESSFACTOR - SV_VERTEXID SV_VIEWPORTARRAYINDEX - - allow_uav_condition branch call domain earlydepthstencil fastopt - flatten forcecase instance loop maxtessfactor numthreads - outputcontrolpoints outputtopology partitioning patchconstantfunc - unroll - - BINORMAL BINORMAL0 BINORMAL1 BINORMAL2 BINORMAL3 BINORMAL4 - BLENDINDICES0 BLENDINDICES1 BLENDINDICES2 BLENDINDICES3 BLENDINDICES4 - BLENDWEIGHT0 BLENDWEIGHT1 BLENDWEIGHT2 BLENDWEIGHT3 BLENDWEIGHT4 COLOR - COLOR0 COLOR1 COLOR2 COLOR3 COLOR4 NORMAL NORMAL0 NORMAL1 NORMAL2 - NORMAL3 NORMAL4 POSITION POSITION0 POSITION1 POSITION2 POSITION3 - POSITION4 POSITIONT PSIZE0 PSIZE1 PSIZE2 PSIZE3 PSIZE4 TANGENT - TANGENT0 TANGENT1 TANGENT2 TANGENT3 TANGENT4 TESSFACTOR0 TESSFACTOR1 - TESSFACTOR2 TESSFACTOR3 TESSFACTOR4 TEXCOORD0 TEXCOORD1 TEXCOORD2 - TEXCOORD3 TEXCOORD4 - - FOG PSIZE - - VFACE VPOS - - DEPTH0 DEPTH1 DEPTH2 DEPTH3 DEPTH4 - ) + state :newline do + mixin :inline_whitespace + rule %r/#/ do + token Comment::Preproc + goto :preproc + end + + rule(//) { pop! } end - ws = %r((?:\s|//.*?\n|/[*].*?[*]/)+) - id = /[a-zA-Z_][a-zA-Z0-9_]*/ + state :preproc do + mixin :inline_whitespace + rule %r/\n/, Comment::Preproc, :pop! + rule %r/[^\n]/, Comment::Preproc + end + # ref: https://microsoft.github.io/hlsl-specs/specs/hlsl.pdf state :root do - mixin :expr_whitespace - rule %r( - ([\w*\s]+?[\s*]) # return arguments - (#{id}) # function name - (\s*\([^;]*?\)(?:\s*:\s+#{id})?) # signature - (#{ws}?)({|;) # open brace or semicolon - )mx do |m| - # This is copied from the C lexer - recurse m[1] - token Name::Function, m[2] - recurse m[3] - recurse m[4] - token Punctuation, m[5] - if m[5] == ?{ - push :function - end + mixin :whitespace + + exp = %r/e[+-]?\d+/i + + rule %r/[;:(){}\[\]=,]/, Punctuation + rule %r([-+/*]), Operator + rule %r([<>]=?), Operator + + # the language doesn't actually have quoted strings but annotations do + # e.g. [domain("quad")] + rule %r(".*?"), Str::Double + + # handles: 5. + rule %r/\d+[.]\d*#{exp}?[hfl]?/i, Num::Float + + # handles: .5 + rule %r/[.]\d+#{exp}?[hfl]?/i, Num::Float + + # handles: 5f + rule %r/\d+#{exp}?[hfl]/i, Num::Float + + # handles: 5e4 + rule %r/\d+#{exp}[hfl]?/i, Num::Float + + rule %r/0[0-7]+[ul]?/, Num::Oct + rule %r/0x\h+[ul]?/, Num::Hex + rule %r/\d+[ul]?\b/i, Num::Integer + + keywords id do + rule KEYWORDS, Keyword + rule KEYWORDS_TYPE, Keyword::Type + rule RESERVED, Keyword::Reserved + rule BUILTINS, Name::Builtin end - rule %r/\{/, Punctuation, :function - mixin :statements + + rule %r/#{id}(?=[ \t]*[(])/, Name::Function + rule id, Name + + rule %r/[.]/, Punctuation, :post_dot + end + + state :post_dot do + mixin :whitespace + + rule id, Name, :pop! + + rule(//) { pop! } end end end diff --git a/lib/rouge/lexers/hlsl/keywords.rb b/lib/rouge/lexers/hlsl/keywords.rb new file mode 100644 index 0000000000..b6ac8caf2c --- /dev/null +++ b/lib/rouge/lexers/hlsl/keywords.rb @@ -0,0 +1,131 @@ +module Rouge + module Lexers + class HLSL + KEYWORDS = Set.new %w( + asm asm_fragment break case cbuffer centroid class column_major + compile compile_fragment const continue default discard do else export + extern for fxgroup globallycoherent groupshared if in inline inout + interface line lineadj linear namespace nointerpolation noperspective + NULL out packoffset pass pixelfragment point precise return register + row_major sample sampler shared stateblock stateblock_state static + struct switch tbuffer technique technique10 technique11 texture + typedef triangle uniform vertexfragment volatile while + + template + ) + + KEYWORDS_TYPE = Set.new %w( + dword matrix snorm string unorm unsigned void vector BlendState Buffer + ByteAddressBuffer ComputeShader DepthStencilState DepthStencilView + DomainShader GeometryShader HullShader InputPatch LineStream + OutputPatch PixelShader PointStream RasterizerState RenderTargetView + RasterizerOrderedBuffer RasterizerOrderedByteAddressBuffer + RasterizerOrderedStructuredBuffer RasterizerOrderedTexture1D + RasterizerOrderedTexture1DArray RasterizerOrderedTexture2D + RasterizerOrderedTexture2DArray RasterizerOrderedTexture3D RWBuffer + RWByteAddressBuffer RWStructuredBuffer RWTexture1D RWTexture1DArray + RWTexture2D RWTexture2DArray RWTexture3D SamplerState + SamplerComparisonState StructuredBuffer Texture1D Texture1DArray + Texture2D Texture2DArray Texture2DMS Texture2DMSArray Texture3D + TextureCube TextureCubeArray TriangleStream VertexShader + + bool bool1 bool2 bool3 bool4 BOOL1 BOOL2 BOOL3 BOOL4 + int int1 int2 int3 int4 + uint uint1 uint2 uint3 uint4 + half half1 half2 half3 half4 + float float1 float2 float3 float4 + double1 double2 double3 double4 + + bool1x1 bool1x2 bool1x3 bool1x4 bool2x1 bool2x2 bool2x3 bool2x4 + bool3x1 bool3x2 bool3x3 bool3x4 bool4x1 bool4x2 bool4x3 bool4x4 + BOOL1x1 BOOL1x2 BOOL1x3 BOOL1x4 BOOL2x1 BOOL2x2 BOOL2x3 BOOL2x4 + BOOL3x1 BOOL3x2 BOOL3x3 BOOL3x4 BOOL4x1 BOOL4x2 BOOL4x3 BOOL4x4 + half1x1 half1x2 half1x3 half1x4 half2x1 half2x2 half2x3 half2x4 + half3x1 half3x2 half3x3 half3x4 half4x1 half4x2 half4x3 half4x4 + int1x1 int1x2 int1x3 int1x4 int2x1 int2x2 int2x3 int2x4 + int3x1 int3x2 int3x3 int3x4 int4x1 int4x2 int4x3 int4x4 + uint1x1 uint1x2 uint1x3 uint1x4 uint2x1 uint2x2 uint2x3 uint2x4 + uint3x1 uint3x2 uint3x3 uint3x4 uint4x1 uint4x2 uint4x3 uint4x4 + float1x1 float1x2 float1x3 float1x4 float2x1 float2x2 float2x3 float2x4 + float3x1 float3x2 float3x3 float3x4 float4x1 float4x2 float4x3 float4x4 + double1x1 double1x2 double1x3 double1x4 double2x1 double2x2 double2x3 double2x4 + double3x1 double3x2 double3x3 double3x4 double4x1 double4x2 double4x3 double4x4 + + typename + ) + + RESERVED = Set.new %w( + auto catch char const_cast delete dynamic_cast enum explicit friend + goto long mutable new operator private protected public + reinterpret_cast short signed sizeof static_cast this throw + try union unsigned using virtual + ) + + BUILTINS = Set.new %w( + abort abs acos all AllMemoryBarrier AllMemoryBarrierWithGroupSync any + AppendStructuredBuffer asdouble asfloat asin asint asuint atan + atan2 ceil CheckAccessFullyMapped clamp clip CompileShader + ConsumeStructuredBuffer cos cosh countbits cross D3DCOLORtoUBYTE4 ddx + ddx_coarse ddx_fine ddy ddy_coarse ddy_fine degrees determinant + DeviceMemoryBarrier DeviceMemoryBarrierWithGroupSync distance dot dst + errorf EvaluateAttributeAtCentroid EvaluateAttributeAtSample + EvaluateAttributeSnapped exp exp2 f16tof32 f32tof16 faceforward + firstbithigh firstbitlow floor fma fmod frac frexp fwidth + GetRenderTargetSampleCount GetRenderTargetSamplePosition + GlobalOrderedCountIncrement GroupMemoryBarrier + GroupMemoryBarrierWithGroupSync InterlockedAdd InterlockedAnd + InterlockedCompareExchange InterlockedCompareStore InterlockedExchange + InterlockedMax InterlockedMin InterlockedOr InterlockedXor isfinite + isinf isnan ldexp length lerp lit log log10 log2 mad max min modf + msad4 mul noise normalize pow printf Process2DQuadTessFactorsAvg + Process2DQuadTessFactorsMax Process2DQuadTessFactorsMin + ProcessIsolineTessFactors ProcessQuadTessFactorsAvg + ProcessQuadTessFactorsMax ProcessQuadTessFactorsMin + ProcessTriTessFactorsAvg ProcessTriTessFactorsMax + ProcessTriTessFactorsMin QuadReadLaneAt QuadSwapX QuadSwapY radians + rcp reflect refract reversebits round rsqrt saturate sign sin sincos + sinh smoothstep sqrt step tan tanh tex1D tex1Dbias tex1Dgrad + tex1Dlod tex1Dproj tex2D tex2Dbias tex2Dgrad tex2Dlod tex2Dproj + tex3D tex3Dbias tex3Dgrad tex3Dlod tex3Dproj texCUBE + texCUBEbias texCUBEgrad texCUBElod texCUBEproj transpose trunc + WaveAllBitAnd WaveAllMax WaveAllMin WaveAllBitOr WaveAllBitXor + WaveAllEqual WaveAllProduct WaveAllSum WaveAllTrue WaveAnyTrue + WaveBallot WaveGetLaneCount WaveGetLaneIndex WaveGetOrderedIndex + WaveIsHelperLane WaveOnce WavePrefixProduct WavePrefixSum + WaveReadFirstLane WaveReadLaneAt + + SV_CLIPDISTANCE SV_CLIPDISTANCE0 SV_CLIPDISTANCE1 SV_CULLDISTANCE + SV_CULLDISTANCE0 SV_CULLDISTANCE1 SV_COVERAGE SV_DEPTH + SV_DEPTHGREATEREQUAL SV_DEPTHLESSEQUAL SV_DISPATCHTHREADID + SV_DOMAINLOCATION SV_GROUPID SV_GROUPINDEX SV_GROUPTHREADID + SV_GSINSTANCEID SV_INNERCOVERAGE SV_INSIDETESSFACTOR SV_INSTANCEID + SV_ISFRONTFACE SV_OUTPUTCONTROLPOINTID SV_POSITION SV_PRIMITIVEID + SV_RENDERTARGETARRAYINDEX SV_SAMPLEINDEX SV_STENCILREF SV_TESSFACTOR + SV_VERTEXID SV_VIEWPORTARRAYINDEX + + allow_uav_condition branch call domain earlydepthstencil fastopt + flatten forcecase instance loop maxtessfactor numthreads + outputcontrolpoints outputtopology partitioning patchconstantfunc + unroll + + BINORMAL BINORMAL0 BINORMAL1 BINORMAL2 BINORMAL3 BINORMAL4 + BLENDINDICES0 BLENDINDICES1 BLENDINDICES2 BLENDINDICES3 BLENDINDICES4 + BLENDWEIGHT0 BLENDWEIGHT1 BLENDWEIGHT2 BLENDWEIGHT3 BLENDWEIGHT4 COLOR + COLOR0 COLOR1 COLOR2 COLOR3 COLOR4 NORMAL NORMAL0 NORMAL1 NORMAL2 + NORMAL3 NORMAL4 POSITION POSITION0 POSITION1 POSITION2 POSITION3 + POSITION4 POSITIONT PSIZE0 PSIZE1 PSIZE2 PSIZE3 PSIZE4 TANGENT + TANGENT0 TANGENT1 TANGENT2 TANGENT3 TANGENT4 TESSFACTOR0 TESSFACTOR1 + TESSFACTOR2 TESSFACTOR3 TESSFACTOR4 TEXCOORD TEXCOORD0 TEXCOORD1 TEXCOORD2 + TEXCOORD3 TEXCOORD4 + + FOG PSIZE + + VFACE VPOS + + DEPTH0 DEPTH1 DEPTH2 DEPTH3 DEPTH4 + + UV + ) + end + end +end diff --git a/spec/visual/samples/hlsl b/spec/visual/samples/hlsl index 7bbc548ad4..b473d2b007 100644 --- a/spec/visual/samples/hlsl +++ b/spec/visual/samples/hlsl @@ -1,3 +1,6 @@ +#define foo \ + 3 + // Data for each pixel struct PSInput { @@ -36,3 +39,207 @@ float4 PSMain(PSInput input) : SV_TARGET float4 gradient = FourColorGradient(_Color1, _Color2, _Color3, _Color4, input.uv); return input.color * gradient; } + +// from https://devblogs.microsoft.com/directx/announcing-hlsl-2021/ +template +void increment(inout T X) { + X += 1; +} + +template<> +void increment(inout float3 X) { + X.x += 1.0; +} + +template +V cast(T X) { + return (V)X; +} + +[numthreads(1,1,1)] +void main() { + int X = 1; + uint Y = cast(X); +} + +//=========== from pygments ============// + +// Macro inside a single-line comment: #define COMMENT_MACRO 1 + +/* Macro inside a block comment: #define COMMENT_MACRO 2 */ + + # define INDENTED_MACRO 5.0 + +#define SINGLELINE_MACRO 10.0 + +#define MULTILINE_MACRO(a, b) float2( \ + a, \ + b \ +) + +[numthreads(256, 1, 1)] +void cs_main(uint3 threadId : SV_DispatchThreadID) +{ + // Seed the PRNG using the thread ID + rng_state = threadId.x; + + // Generate a few numbers... + uint r0 = rand_xorshift(); + uint r1 = rand_xorshift(); + // Do some stuff with them... + + // Generate a random float in [0, 1)... + float f0 = float(rand_xorshift()) * (1.0 / 4294967296.0); + + // ...etc. +} + +// Constant buffer of parameters +cbuffer IntegratorParams : register(b0) +{ + float2 specPow; // Spec powers in XY directions (equal for isotropic BRDFs) + float3 L; // Unit vector toward light + int2 cThread; // Total threads launched in XY dimensions + int2 xyOutput; // Where in the output buffer to store the result +} + +static const float pi = 3.141592654; + +float AshikhminShirleyNDF(float3 H) +{ + float normFactor = sqrt((specPow.x + 2.0f) * (specPow.y + 2.0)) * (0.5f / pi); + float NdotH = H.z; + float2 Hxy = normalize(H.xy); + return normFactor * pow(NdotH, dot(specPow, Hxy * Hxy)); +} + +float BeckmannNDF(float3 H) +{ + float glossFactor = specPow.x * 0.5f + 1.0f; // This is 1/m^2 in the usual Beckmann formula + float normFactor = glossFactor * (1.0f / pi); + float NdotHSq = H.z * H.z; + return normFactor / (NdotHSq * NdotHSq) * exp(glossFactor * (1.0f - 1.0f / NdotHSq)); +} + +// Output buffer for compute shader (actually float, but must be declared as uint +// for atomic operations to work) +globallycoherent RWTexture2D o_data : register(u0); + +// Sum up the outputs of all threads and store to the output location +static const uint threadGroupSize2D = 16; +static const uint threadGroupSize1D = threadGroupSize2D * threadGroupSize2D; +groupshared float g_partialSums[threadGroupSize1D]; +void SumAcrossThreadsAndStore(float value, uint iThreadInGroup) +{ + // First reduce within the threadgroup: partial sums of 2, 4, 8... elements + // are calculated by 1/2, 1/4, 1/8... of the threads, always keeping the + // active threads at the front of the group to minimize divergence. + + // NOTE: there are faster ways of doing this...but this is simple to code + // and good enough. + + g_partialSums[iThreadInGroup] = value; + GroupMemoryBarrierWithGroupSync(); + + [unroll] for (uint i = threadGroupSize1D / 2; i > 0; i /= 2) + { + if (iThreadInGroup < i) + { + g_partialSums[iThreadInGroup] += g_partialSums[iThreadInGroup + i]; + } + GroupMemoryBarrierWithGroupSync(); + } + + // Then reduce across threadgroups: one thread from each group adds the group + // total to the final output location, using a software transactional memory + // style since D3D11 doesn't support atomic add on floats. + // (Assumes the output value has been cleared to zero beforehand.) + + if (iThreadInGroup == 0) + { + float threadGroupSum = g_partialSums[0]; + uint outputValueRead = o_data[xyOutput]; + while (true) + { + uint newOutputValue = asuint(asfloat(outputValueRead) + threadGroupSum); + uint previousOutputValue; + InterlockedCompareExchange( + o_data[xyOutput], outputValueRead, newOutputValue, previousOutputValue); + if (previousOutputValue == outputValueRead) + break; + outputValueRead = previousOutputValue; + } + } +} + +void main( + in Vertex i_vtx, + out Vertex o_vtx, + out float3 o_vecCamera : CAMERA, + out float4 o_uvzwShadow : UVZW_SHADOW, + out float4 o_posClip : SV_Position) +{ + o_vtx = i_vtx; + o_vecCamera = g_posCamera - i_vtx.m_pos; + o_uvzwShadow = mul(float4(i_vtx.m_pos, 1.0), g_matWorldToUvzwShadow); + o_posClip = mul(float4(i_vtx.m_pos, 1.0), g_matWorldToClip); +} + +#pragma pack_matrix(row_major) + +struct Vertex +{ + float3 m_pos : POSITION; + float3 m_normal : NORMAL; + float2 m_uv : UV; +}; + +cbuffer CBFrame : CB_FRAME // matches struct CBFrame in test.cpp +{ + float4x4 g_matWorldToClip; + float4x4 g_matWorldToUvzwShadow; + float3x3 g_matWorldToUvzShadowNormal; + float3 g_posCamera; + + float3 g_vecDirectionalLight; + float3 g_rgbDirectionalLight; + + float2 g_dimsShadowMap; + float g_normalOffsetShadow; + float g_shadowSharpening; + + float g_exposure; // Exposure multiplier +} + +Texture2D g_texDiffuse : register(t0); +SamplerState g_ss : register(s0); + +void main( + in Vertex i_vtx, + in float3 i_vecCamera : CAMERA, + in float4 i_uvzwShadow : UVZW_SHADOW, + out float3 o_rgb : SV_Target) +{ + float3 normal = normalize(i_vtx.m_normal); + + // Sample shadow map + float shadow = EvaluateShadow(i_uvzwShadow, normal); + + // Evaluate diffuse lighting + float3 diffuseColor = g_texDiffuse.Sample(g_ss, i_vtx.m_uv); + float3 diffuseLight = g_rgbDirectionalLight * (shadow * saturate(dot(normal, g_vecDirectionalLight))); + diffuseLight += SimpleAmbient(normal); + + o_rgb = diffuseColor * diffuseLight; +} + +[domain("quad")] +void ds( + in float edgeFactors[4] : SV_TessFactor, + in float insideFactors[2] : SV_InsideTessFactor, + in OutputPatch inp, + in float2 uv : SV_DomainLocation, + out float4 o_pos : SV_Position) +{ + o_pos = lerp(lerp(inp[0].pos, inp[1].pos, uv.x), lerp(inp[2].pos, inp[3].pos, uv.x), uv.y); +} From 3e6409d4f14f3e3e762d2c6925810307a366430d Mon Sep 17 00:00:00 2001 From: jneen Date: Thu, 16 Apr 2026 13:05:12 -0400 Subject: [PATCH 058/154] add another example to bbcbasic --- spec/visual/samples/bbcbasic | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/spec/visual/samples/bbcbasic b/spec/visual/samples/bbcbasic index 2d8e0c73e6..dda0dafeb2 100644 --- a/spec/visual/samples/bbcbasic +++ b/spec/visual/samples/bbcbasic @@ -42,3 +42,11 @@ REPEATUNTILFALSE END DEFPROCa PRINT"Hello world":ENDPROC DEFFNnumber(x%)=ABS(x%-4) + +MODE1:VDU5 +FORN=8TO247:FORM=0TO319 +X=0:Y=-.1:Z=3:U=(M-159.5)/160:V=(N-127.5)/160:W=1/SQR(U*U+V*V+1):U=U*W:V=V*W:I=SGNU:G=1 +E=X-I:F=Y-I:P=U*E+V*F-W*Z:D=P*P-E*E-F*F-Z*Z+1:IFD>0T=-P-SQRD:IFT>0X=X+T*U:Y=Y+T*V:Z=Z-T*W:E=X-I:F=Y-I:G=Z:P=2*(U*E+V*F-W*G):U=U-P*E:V=V-P*F:W=W+P*G:I=-I:GOTO40 +IFV<0P=(Y+2)/V:V=-V*((INT(X-U*P)+INT(Z-W*P)AND1)/2+.3)+.2 +GCOL0,3-(48*SQRV+16*(PI*(M+17*N)-INT(PI*(M+17*N))))DIV16 +PLOT69,4*M,4*N:NEXT, From 7e86299640c0ab1d50f0d535a19132efc124d077 Mon Sep 17 00:00:00 2001 From: jneen Date: Thu, 16 Apr 2026 13:11:16 -0400 Subject: [PATCH 059/154] rebaseme: eager load superclass first --- lib/rouge/lexer.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/rouge/lexer.rb b/lib/rouge/lexer.rb index 2ba251065a..b9d240c582 100644 --- a/lib/rouge/lexer.rb +++ b/lib/rouge/lexer.rb @@ -237,10 +237,10 @@ def eager_load! return if @_loaded @_loaded = true - lazy_procs.each { |b| instance_eval(&b) } - superclass.eager_load! unless superclass == Lexer + lazy_procs.each { |b| instance_eval(&b) } + self end From 43bf9c2b1b1a401b421e45c09ad6df5ac50fc42d Mon Sep 17 00:00:00 2001 From: jneen Date: Thu, 16 Apr 2026 13:11:40 -0400 Subject: [PATCH 060/154] use keywords api for sql --- .rubocop_todo.yml | 1 - lib/rouge/lexers/sql.rb | 111 +++---------------------------- lib/rouge/lexers/sql/keywords.rb | 97 +++++++++++++++++++++++++++ 3 files changed, 107 insertions(+), 102 deletions(-) create mode 100644 lib/rouge/lexers/sql/keywords.rb diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index 89f078a428..a71a5f1f69 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -78,5 +78,4 @@ Rouge/NoHugeCollections: - 'lib/rouge/lexers/plsql.rb' - 'lib/rouge/lexers/racket.rb' - 'lib/rouge/lexers/sas.rb' - - 'lib/rouge/lexers/sql.rb' - 'lib/rouge/lexers/stata.rb' diff --git a/lib/rouge/lexers/sql.rb b/lib/rouge/lexers/sql.rb index ff5c8a7d41..f372da1050 100644 --- a/lib/rouge/lexers/sql.rb +++ b/lib/rouge/lexers/sql.rb @@ -10,99 +10,7 @@ class SQL < RegexLexer filenames '*.sql' mimetypes 'text/x-sql' - def self.keywords - @keywords ||= Set.new %w( - ABORT ABS ABSOLUTE ACCESS ADA ADD ADMIN AFTER AGGREGATE ALIAS - ALL ALLOCATE ALTER ANALYSE ANALYZE AND ANY ARE AS ASC ASENSITIVE - ASSERTION ASSIGNMENT ASYMMETRIC AT ATOMIC AUTHORIZATION - AVG BACKWARD BEFORE BEGIN BETWEEN BITVAR BIT_LENGTH BOTH - BREADTH BY C CACHE CALL CALLED CARDINALITY CASCADE CASCADED - CASE CAST CATALOG CATALOG_NAME CHAIN CHARACTERISTICS - CHARACTER_LENGTH CHARACTER_SET_CATALOG CHARACTER_SET_NAME - CHARACTER_SET_SCHEMA CHAR_LENGTH CHECK CHECKED CHECKPOINT - CLASS CLASS_ORIGIN CLOB CLOSE CLUSTER COALSECE COBOL COLLATE - COLLATION COLLATION_CATALOG COLLATION_NAME COLLATION_SCHEMA - COLUMN COLUMN_NAME COMMAND_FUNCTION COMMAND_FUNCTION_CODE - COMMENT COMMIT COMMITTED COMPLETION CONDITION_NUMBER - CONNECT CONNECTION CONNECTION_NAME CONSTRAINT CONSTRAINTS - CONSTRAINT_CATALOG CONSTRAINT_NAME CONSTRAINT_SCHEMA - CONSTRUCTOR CONTAINS CONTINUE CONVERSION CONVERT COPY - CORRESPONTING COUNT CREATE CREATEDB CREATEUSER CROSS CUBE - CURRENT CURRENT_DATE CURRENT_PATH CURRENT_ROLE CURRENT_TIME - CURRENT_TIMESTAMP CURRENT_USER CURSOR CURSOR_NAME CYCLE DATA - DATABASE DATETIME_INTERVAL_CODE DATETIME_INTERVAL_PRECISION - DAY DEALLOCATE DECLARE DEFAULT DEFAULTS DEFERRABLE DEFERRED - DEFINED DEFINER DELETE DELIMITER DELIMITERS DEREF DESC DESCRIBE - DESCRIPTOR DESTROY DESTRUCTOR DETERMINISTIC DIAGNOSTICS - DICTIONARY DISCONNECT DISPATCH DISTINCT DO DOMAIN DROP - DYNAMIC DYNAMIC_FUNCTION DYNAMIC_FUNCTION_CODE EACH ELSE - ENCODING ENCRYPTED END END-EXEC EQUALS ESCAPE EVERY EXCEPT - ESCEPTION EXCLUDING EXCLUSIVE EXEC EXECUTE EXISTING EXISTS - EXPLAIN EXTERNAL EXTRACT FALSE FETCH FINAL FIRST FOLLOWING FOR - FORCE FOREIGN FORTRAN FORWARD FOUND FREE FREEZE FROM FULL FUNCTION - G GENERAL GENERATED GET GLOBAL GO GOTO GRANT GRANTED GROUP - GROUPING HANDLER HAVING HIERARCHY HOLD HOST IDENTITY IGNORE - ILIKE IMMEDIATE IMMUTABLE IMPLEMENTATION IMPLICIT IN INCLUDING - INCREMENT INDEX INDITCATOR INFIX INHERITS INITIALIZE INITIALLY - INNER INOUT INPUT INSENSITIVE INSERT INSTANTIABLE INSTEAD - INTERSECT INTO INVOKER IS ISNULL ISOLATION ITERATE JOIN KEY - KEY_MEMBER KEY_TYPE LANCOMPILER LANGUAGE LARGE LAST LATERAL - LEADING LEFT LENGTH LESS LEVEL LIKE LIMIT LISTEN LOAD LOCAL - LOCALTIME LOCALTIMESTAMP LOCATION LOCATOR LOCK LOWER MAP MATCH - MAX MAXVALUE MESSAGE_LENGTH MESSAGE_OCTET_LENGTH MESSAGE_TEXT - METHOD MIN MINUTE MINVALUE MOD MODE MODIFIES MODIFY MONTH - MORE MOVE MUMPS NAMES NATURAL NCLOB NEW NEXT - NO NOCREATEDB NOCREATEUSER NONE NOT NOTHING NOTIFY NOTNULL - NULL NULLABLE NULLIF OBJECT OCTET_LENGTH OF OFF OFFSET OIDS - OLD ON ONLY OPEN OPERATION OPERATOR OPTION OPTIONS OR ORDER - ORDINALITY OUT OUTER OUTPUT OVERLAPS OVERLAY OVERRIDING - OWNER PAD PARAMETER PARAMETERS PARAMETER_MODE PARAMATER_NAME - PARAMATER_ORDINAL_POSITION PARAMETER_SPECIFIC_CATALOG - PARAMETER_SPECIFIC_NAME PARAMATER_SPECIFIC_SCHEMA PARTIAL PARTITION - PASCAL PENDANT PLACING PLI POSITION POSTFIX PREFIX PRECEDING PREORDER - PREPARE PRESERVE PRIMARY PRIOR PRIVILEGES PROCEDURAL PROCEDURE - PUBLIC RANGE READ READS RECHECK RECURSIVE REF REFERENCES REFERENCING - REINDEX RELATIVE RENAME REPEATABLE REPLACE RESET RESTART - RESTRICT RESULT RETURN RETURNED_LENGTH RETURNED_OCTET_LENGTH - RETURNED_SQLSTATE RETURNS REVOKE RIGHT ROLE ROLLBACK ROLLUP - ROUTINE ROUTINE_CATALOG ROUTINE_NAME ROUTINE_SCHEMA ROW ROWS - ROW_COUNT RULE SAVE_POINT SCALE SCHEMA SCHEMA_NAME SCOPE SCROLL - SEARCH SECOND SECURITY SELECT SELF SENSITIVE SERIALIZABLE - SERVER_NAME SESSION SESSION_USER SET SETOF SETS SHARE SHOW - SIMILAR SIMPLE SIZE SOME SOURCE SPACE SPECIFIC SPECIFICTYPE - SPECIFIC_NAME SQL SQLCODE SQLERROR SQLEXCEPTION SQLSTATE - SQLWARNINIG STABLE START STATE STATEMENT STATIC STATISTICS - STDIN STDOUT STORAGE STRICT STRUCTURE STYPE SUBCLASS_ORIGIN - SUBLIST SUBSTRING SUM SYMMETRIC SYSID SYSTEM SYSTEM_USER - TABLE TABLE_NAME TEMP TEMPLATE TEMPORARY TERMINATE THAN THEN - TIMEZONE_HOUR TIMEZONE_MINUTE TO TOAST TRAILING - TRANSATION TRANSACTIONS_COMMITTED TRANSACTIONS_ROLLED_BACK - TRANSATION_ACTIVE TRANSFORM TRANSFORMS TRANSLATE TRANSLATION - TREAT TRIGGER TRIGGER_CATALOG TRIGGER_NAME TRIGGER_SCHEMA TRIM - TRUE TRUNCATE TRUSTED TYPE UNCOMMITTED UNDER UNENCRYPTED UNION - UNIQUE UNKNOWN UNLISTEN UNNAMED UNNEST UNTIL UPDATE UPPER - USAGE USER USER_DEFINED_TYPE_CATALOG USER_DEFINED_TYPE_NAME - USER_DEFINED_TYPE_SCHEMA USING VACUUM VALID VALIDATOR VALUES - VARIABLE VERBOSE VERSION VIEW VOLATILE WHEN WHENEVER WHERE - WINDOW WITH WITHOUT WORK WRITE ZONE - ) - end - - def self.keywords_type - # sources: - # https://dev.mysql.com/doc/refman/5.7/en/numeric-type-overview.html - # https://dev.mysql.com/doc/refman/5.7/en/date-and-time-type-overview.html - # https://dev.mysql.com/doc/refman/5.7/en/string-type-overview.html - @keywords_type ||= Set.new(%w( - ZEROFILL UNSIGNED SIGNED SERIAL BIT TINYINT BOOL BOOLEAN SMALLINT - MEDIUMINT INT INTEGER BIGINT DECIMAL DEC NUMERIC FIXED FLOAT DOUBLE - PRECISION REAL - DATE DATETIME TIMESTAMP TIME YEAR - NATIONAL CHAR CHARACTER NCHAR BYTE - VARCHAR VARYING BINARY VARBINARY TINYBLOB TINYTEXT BLOB TEXT - MEDIUMBLOB MEDIUMTEXT LONGBLOB LONGTEXT ENUM - )) - end + lazy { require_relative 'sql/keywords' } state :root do rule %r/\s+/m, Text @@ -115,14 +23,15 @@ def self.keywords_type rule %r/"/, Name::Variable, :double_string rule %r/`/, Name::Variable, :backtick - rule %r/\w+/ do |m| - if self.class.keywords_type.include? m[0].upcase - token Name::Builtin - elsif self.class.keywords.include? m[0].upcase - token Keyword - else - token Name - end + rule %r/(?:ELS)?IF\s+(?:INSERTING|UPDATING|DELETING)\b/, Keyword + rule %r/END\s+IF\b/, Keyword + + keywords %r/\w+/ do + transform(&:upcase) + + rule KEYWORDS_TYPE, Name::Builtin + rule KEYWORDS, Keyword + default Name end rule %r([+*/<>=~!@#%&|?^-]), Operator diff --git a/lib/rouge/lexers/sql/keywords.rb b/lib/rouge/lexers/sql/keywords.rb new file mode 100644 index 0000000000..f522f029a3 --- /dev/null +++ b/lib/rouge/lexers/sql/keywords.rb @@ -0,0 +1,97 @@ +module Rouge + module Lexers + class SQL + KEYWORDS = Set.new %w( + ABORT ABS ABSOLUTE ACCESS ADA ADD ADMIN AFTER AGGREGATE ALIAS + ALL ALLOCATE ALTER ANALYSE ANALYZE AND ANY ARE AS ASC ASENSITIVE + ASSERTION ASSIGNMENT ASYMMETRIC AT ATOMIC AUTHORIZATION + AVG BACKWARD BEFORE BEGIN BETWEEN BITVAR BIT_LENGTH BOTH + BREADTH BY C CACHE CALL CALLED CARDINALITY CASCADE CASCADED + CASE CAST CATALOG CATALOG_NAME CHAIN CHARACTERISTICS + CHARACTER_LENGTH CHARACTER_SET_CATALOG CHARACTER_SET_NAME + CHARACTER_SET_SCHEMA CHAR_LENGTH CHECK CHECKED CHECKPOINT + CLASS CLASS_ORIGIN CLOB CLOSE CLUSTER COALSECE COBOL COLLATE + COLLATION COLLATION_CATALOG COLLATION_NAME COLLATION_SCHEMA + COLUMN COLUMN_NAME COMMAND_FUNCTION COMMAND_FUNCTION_CODE + COMMENT COMMIT COMMITTED COMPLETION CONDITION_NUMBER + CONNECT CONNECTION CONNECTION_NAME CONSTRAINT CONSTRAINTS + CONSTRAINT_CATALOG CONSTRAINT_NAME CONSTRAINT_SCHEMA + CONSTRUCTOR CONTAINS CONTINUE CONVERSION CONVERT COPY + CORRESPONTING COUNT CREATE CREATEDB CREATEUSER CROSS CUBE + CURRENT CURRENT_DATE CURRENT_PATH CURRENT_ROLE CURRENT_TIME + CURRENT_TIMESTAMP CURRENT_USER CURSOR CURSOR_NAME CYCLE DATA + DATABASE DATETIME_INTERVAL_CODE DATETIME_INTERVAL_PRECISION + DAY DEALLOCATE DECLARE DEFAULT DEFAULTS DEFERRABLE DEFERRED + DEFINED DEFINER DELETE DELIMITER DELIMITERS DEREF DESC DESCRIBE + DESCRIPTOR DESTROY DESTRUCTOR DETERMINISTIC DIAGNOSTICS + DICTIONARY DISCONNECT DISPATCH DISTINCT DO DOMAIN DROP + DYNAMIC DYNAMIC_FUNCTION DYNAMIC_FUNCTION_CODE EACH ELSE + ENCODING ENCRYPTED END END-EXEC EQUALS ESCAPE EVERY EXCEPT + ESCEPTION EXCLUDING EXCLUSIVE EXEC EXECUTE EXISTING EXISTS + EXPLAIN EXTERNAL EXTRACT FALSE FETCH FINAL FIRST FOLLOWING FOR + FORCE FOREIGN FORTRAN FORWARD FOUND FREE FREEZE FROM FULL FUNCTION + G GENERAL GENERATED GET GLOBAL GO GOTO GRANT GRANTED GROUP + GROUPING HANDLER HAVING HIERARCHY HOLD HOST IDENTITY IGNORE + ILIKE IMMEDIATE IMMUTABLE IMPLEMENTATION IMPLICIT IN INCLUDING + INCREMENT INDEX INDITCATOR INFIX INHERITS INITIALIZE INITIALLY + INNER INOUT INPUT INSENSITIVE INSERT INSTANTIABLE INSTEAD + INTERSECT INTO INVOKER IS ISNULL ISOLATION ITERATE JOIN KEY + KEY_MEMBER KEY_TYPE LANCOMPILER LANGUAGE LARGE LAST LATERAL + LEADING LEFT LENGTH LESS LEVEL LIKE LIMIT LISTEN LOAD LOCAL + LOCALTIME LOCALTIMESTAMP LOCATION LOCATOR LOCK LOWER MAP MATCH + MAX MAXVALUE MESSAGE_LENGTH MESSAGE_OCTET_LENGTH MESSAGE_TEXT + METHOD MIN MINUTE MINVALUE MOD MODE MODIFIES MODIFY MONTH + MORE MOVE MUMPS NAMES NATURAL NCLOB NEW NEXT + NO NOCREATEDB NOCREATEUSER NONE NOT NOTHING NOTIFY NOTNULL + NULL NULLABLE NULLIF OBJECT OCTET_LENGTH OF OFF OFFSET OIDS + OLD ON ONLY OPEN OPERATION OPERATOR OPTION OPTIONS OR ORDER + ORDINALITY OUT OUTER OUTPUT OVERLAPS OVERLAY OVERRIDING + OWNER PAD PARAMETER PARAMETERS PARAMETER_MODE PARAMATER_NAME + PARAMATER_ORDINAL_POSITION PARAMETER_SPECIFIC_CATALOG + PARAMETER_SPECIFIC_NAME PARAMATER_SPECIFIC_SCHEMA PARTIAL PARTITION + PASCAL PENDANT PLACING PLI POSITION POSTFIX PREFIX PRECEDING PREORDER + PREPARE PRESERVE PRIMARY PRIOR PRIVILEGES PROCEDURAL PROCEDURE + PUBLIC RANGE READ READS RECHECK RECURSIVE REF REFERENCES REFERENCING + REINDEX RELATIVE RENAME REPEATABLE REPLACE RESET RESTART + RESTRICT RESULT RETURN RETURNED_LENGTH RETURNED_OCTET_LENGTH + RETURNED_SQLSTATE RETURNS REVOKE RIGHT ROLE ROLLBACK ROLLUP + ROUTINE ROUTINE_CATALOG ROUTINE_NAME ROUTINE_SCHEMA ROW ROWS + ROW_COUNT RULE SAVE_POINT SCALE SCHEMA SCHEMA_NAME SCOPE SCROLL + SEARCH SECOND SECURITY SELECT SELF SENSITIVE SERIALIZABLE + SERVER_NAME SESSION SESSION_USER SET SETOF SETS SHARE SHOW + SIMILAR SIMPLE SIZE SOME SOURCE SPACE SPECIFIC SPECIFICTYPE + SPECIFIC_NAME SQL SQLCODE SQLERROR SQLEXCEPTION SQLSTATE + SQLWARNINIG STABLE START STATE STATEMENT STATIC STATISTICS + STDIN STDOUT STORAGE STRICT STRUCTURE STYPE SUBCLASS_ORIGIN + SUBLIST SUBSTRING SUM SYMMETRIC SYSID SYSTEM SYSTEM_USER + TABLE TABLE_NAME TEMP TEMPLATE TEMPORARY TERMINATE THAN THEN + TIMEZONE_HOUR TIMEZONE_MINUTE TO TOAST TRAILING + TRANSATION TRANSACTIONS_COMMITTED TRANSACTIONS_ROLLED_BACK + TRANSATION_ACTIVE TRANSFORM TRANSFORMS TRANSLATE TRANSLATION + TREAT TRIGGER TRIGGER_CATALOG TRIGGER_NAME TRIGGER_SCHEMA TRIM + TRUE TRUNCATE TRUSTED TYPE UNCOMMITTED UNDER UNENCRYPTED UNION + UNIQUE UNKNOWN UNLISTEN UNNAMED UNNEST UNTIL UPDATE UPPER + USAGE USER USER_DEFINED_TYPE_CATALOG USER_DEFINED_TYPE_NAME + USER_DEFINED_TYPE_SCHEMA USING VACUUM VALID VALIDATOR VALUES + VARIABLE VERBOSE VERSION VIEW VOLATILE WHEN WHENEVER WHERE + WINDOW WITH WITHOUT WORK WRITE ZONE + + WHILE + ) + + # sources: + # https://dev.mysql.com/doc/refman/5.7/en/numeric-type-overview.html + # https://dev.mysql.com/doc/refman/5.7/en/date-and-time-type-overview.html + # https://dev.mysql.com/doc/refman/5.7/en/string-type-overview.html + KEYWORDS_TYPE = Set.new %w( + ZEROFILL UNSIGNED SIGNED SERIAL BIT TINYINT BOOL BOOLEAN SMALLINT + MEDIUMINT INT INTEGER BIGINT DECIMAL DEC NUMERIC FIXED FLOAT DOUBLE + PRECISION REAL + DATE DATETIME TIMESTAMP TIME YEAR + NATIONAL CHAR CHARACTER NCHAR BYTE + VARCHAR VARYING BINARY VARBINARY TINYBLOB TINYTEXT BLOB TEXT + MEDIUMBLOB MEDIUMTEXT LONGBLOB LONGTEXT ENUM + ) + end + end +end From 7cb9e8887a072f5ee09df213169b9e658e06bf0b Mon Sep 17 00:00:00 2001 From: jneen Date: Thu, 16 Apr 2026 13:11:58 -0400 Subject: [PATCH 061/154] break out hql keywords --- .rubocop_todo.yml | 1 - lib/rouge/lexers/hql.rb | 95 +---------------------------- lib/rouge/lexers/hql/keywords.rb | 101 +++++++++++++++++++++++++++++++ 3 files changed, 102 insertions(+), 95 deletions(-) create mode 100644 lib/rouge/lexers/hql/keywords.rb diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index a71a5f1f69..b03f3f52ce 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -70,7 +70,6 @@ Rouge/NoBuildingAlternationPatternInRegexp: # Offense count: 20 Rouge/NoHugeCollections: Exclude: - - 'lib/rouge/lexers/hql.rb' - 'lib/rouge/lexers/idlang.rb' - 'lib/rouge/lexers/igorpro.rb' - 'lib/rouge/lexers/janet.rb' diff --git a/lib/rouge/lexers/hql.rb b/lib/rouge/lexers/hql.rb index 7d141d906a..16ab9c51be 100644 --- a/lib/rouge/lexers/hql.rb +++ b/lib/rouge/lexers/hql.rb @@ -10,100 +10,7 @@ class HQL < SQL tag 'hql' filenames '*.hql' - def self.keywords - # sources: - # https://cwiki.apache.org/confluence/display/Hive/LanguageManual+DDL - # https://cwiki.apache.org/confluence/display/Hive/LanguageManual+UDF - @keywords ||= Set.new(%w( - ADD ADMIN AFTER ANALYZE ARCHIVE ASC BEFORE BUCKET BUCKETS CASCADE - CHANGE CLUSTER CLUSTERED CLUSTERSTATUS COLLECTION COLUMNS COMMENT - COMPACT COMPACTIONS COMPUTE CONCATENATE CONTINUE DATA DATABASES - DATETIME DAY DBPROPERTIES DEFERRED DEFINED DELIMITED DEPENDENCY DESC - DIRECTORIES DIRECTORY DISABLE DISTRIBUTE ELEM_TYPE ENABLE ESCAPED - EXCLUSIVE EXPLAIN EXPORT FIELDS FILE FILEFORMAT FIRST FORMAT FORMATTED - FUNCTIONS HOLD_DDLTIME HOUR IDXPROPERTIES IGNORE INDEX INDEXES INPATH - INPUTDRIVER INPUTFORMAT ITEMS JAR KEYS KEY_TYPE LIMIT LINES LOAD - LOCATION LOCK LOCKS LOGICAL LONG MAPJOIN MATERIALIZED METADATA MINUS - MINUTE MONTH MSCK NOSCAN NO_DROP OFFLINE OPTION OUTPUTDRIVER - OUTPUTFORMAT OVERWRITE OWNER PARTITIONED PARTITIONS PLUS PRETTY - PRINCIPALS PROTECTION PURGE READ READONLY REBUILD RECORDREADER - RECORDWRITER REGEXP RELOAD RENAME REPAIR REPLACE REPLICATION RESTRICT - REWRITE RLIKE ROLE ROLES SCHEMA SCHEMAS SECOND SEMI SERDE - SERDEPROPERTIES SERVER SETS SHARED SHOW SHOW_DATABASE SKEWED SORT - SORTED SSL STATISTICS STORED STREAMTABLE STRING STRUCT TABLES - TBLPROPERTIES TEMPORARY TERMINATED TINYINT TOUCH TRANSACTIONS UNARCHIVE - UNDO UNIONTYPE UNLOCK UNSET UNSIGNED URI USE UTC UTCTIMESTAMP - VALUE_TYPE VIEW WHILE YEAR IF - - ALL ALTER AND ARRAY AS AUTHORIZATION BETWEEN BIGINT BINARY BOOLEAN - BOTH BY CASE CAST CHAR COLUMN CONF CREATE CROSS CUBE CURRENT - CURRENT_DATE CURRENT_TIMESTAMP CURSOR DATABASE DATE DECIMAL DELETE - DESCRIBE DISTINCT DOUBLE DROP ELSE END EXCHANGE EXISTS EXTENDED - EXTERNAL FALSE FETCH FLOAT FOLLOWING FOR FROM FULL FUNCTION GRANT - GROUP GROUPING HAVING IMPORT IN INNER INSERT INT INTERSECT - INTERVAL INTO IS JOIN LATERAL LEFT LESS LIKE LOCAL MACRO MAP MORE - NONE NOT NULL OF ON OR ORDER OUT OUTER OVER PARTIALSCAN PARTITION - PERCENT PRECEDING PRESERVE PROCEDURE RANGE READS REDUCE REVOKE RIGHT - ROLLUP ROW ROWS SELECT SET SMALLINT TABLE TABLESAMPLE THEN TIMESTAMP - TO TRANSFORM TRIGGER TRUE TRUNCATE UNBOUNDED UNION UNIQUEJOIN UPDATE - USER USING UTC_TMESTAMP VALUES VARCHAR WHEN WHERE WINDOW WITH - - AUTOCOMMIT ISOLATION LEVEL OFFSET SNAPSHOT TRANSACTION WORK WRITE - - COMMIT ONLY ROLLBACK START - - ABORT KEY LAST NORELY NOVALIDATE NULLS RELY VALIDATE - - CACHE CONSTRAINT FOREIGN PRIMARY REFERENCES - - DETAIL DOW EXPRESSION OPERATOR QUARTER SUMMARY VECTORIZATION WEEK YEARS MONTHS WEEKS DAYS HOURS MINUTES SECONDS - - DAYOFWEEK EXTRACT FLOOR INTEGER PRECISION VIEWS - - TIMESTAMPTZ ZONE - - TIME NUMERIC - - NAMED_STRUCT CREATE_UNION - - ROUND BROUND CEIL CEILING RAND EXP LN LOG10 LOG2 LOG POW POWER SQRT BIN - HEX UNHEX CONV ABS PMOD SIN ASIN COS ACOS TAN ATAN DEGREES RADIANS POSITIVE - NEGATIVE SIGN E PI FACTORIAL CBRT SHIFTLEFT SHIFTRIGHT SHIFTRIGHTUNSIGNED - GREATEST LEAST WIDTH_BUCKET SIZE MAP_KEYS MAP_VALUES ARRAY_CONTAINS - SORT_ARRAY FROM_UNIXTIME UNIX_TIMESTAMP TO_DATE DAYOFMONTH - WEEKOFYEAR DATEDIFF DATE_ADD DATE_SUB FROM_UTC_TIMESTAMP - TO_UTC_TIMESTAMP ADD_MONTHS LAST_DAY NEXT_DAY - TRUNC MONTHS_BETWEEN DATE_FORMAT ISNULL ISNOTNULL NVL COALESCE - then else end NULLIF ASSERT_TRUE ASCII BASE64 CHARACTER_LENGTH CHR CONCAT - CONTEXT_NGRAMS CONCAT_WS DECODE ELT ENCODE FIELD FIND_IN_SET - FORMAT_NUMBER GET_JSON_OBJECT IN_FILE INSTR LENGTH LOCATE LOWER LCASE LPAD LTRIM - NGRAMS OCTET_LENGTH PARSE_URL PRINTF REGEXP_EXTRACT REGEXP_REPLACE REPEAT REVERSE RPAD RTRIM SENTENCES SPACE SPLIT STR_TO_MAP SUBSTR SUBSTRING - SUBSTRING_INDEX TRANSLATE TRIM UNBASE64 UPPER UCASE INITCAP LEVENSHTEIN SOUNDEX - MASK MASK_FIRST_N MASK_LAST_N MASK_SHOW_FIRST_N MASK_SHOW_LAST_N MASK_HASH - JAVA_METHOD REFLECT HASH CURRENT_USER LOGGED_IN_USER CURRENT_DATABASE MD5 SHA1 - SHA CRC32 SHA2 AES_ENCRYPT AES_DECRYPT VERSION COUNT SUM AVG MIN MAX VARIANCE - VAR_POP VAR_SAMP STDDEV_POP STDDEV_SAMP COVAR_POP COVAR_SAMP CORR PERCENTILE - PERCENTILE_APPROX REGR_AVGX REGR_AVGY REGR_COUNT - REGR_INTERCEPT REGR_R2 REGR_SLOPE REGR_SXX REGR_SXY REGR_SYY HISTOGRAM_NUMERIC - COLLECT_SET COLLECT_LIST NTILE EXPLODE POSEXPLODE INLINE STACK - - JSON_TUPLE PARSE_URL_TUPLE - - XPATH XPATH_SHORT XPATH_INT XPATH_LONG XPATH_FLOAT XPATH_DOUBLE - XPATH_NUMBER XPATH_STRING - )) - end - - def self.keywords_type - # source: https://cwiki.apache.org/confluence/display/Hive/LanguageManual+Types - @keywords_type ||= Set.new(%w( - TINYINT SMALLINT INT INTEGER BIGINT FLOAT DOUBLE PRECISION DECIMAL NUMERIC - TIMESTAMP DATE INTERVAL - STRING VARCHAR CHAR - BOOLEAN BINARY - ARRAY MAP STRUCT UNIONTYPE - )) - end + lazy { require_relative 'hql/keywords' } prepend :root do # a double-quoted string is a string literal in Hive QL. diff --git a/lib/rouge/lexers/hql/keywords.rb b/lib/rouge/lexers/hql/keywords.rb new file mode 100644 index 0000000000..dfbc80196f --- /dev/null +++ b/lib/rouge/lexers/hql/keywords.rb @@ -0,0 +1,101 @@ +module Rouge + module Lexers + class HQL + def self.keywords + # sources: + # https://cwiki.apache.org/confluence/display/Hive/LanguageManual+DDL + # https://cwiki.apache.org/confluence/display/Hive/LanguageManual+UDF + @keywords ||= Set.new(%w( + ADD ADMIN AFTER ANALYZE ARCHIVE ASC BEFORE BUCKET BUCKETS CASCADE + CHANGE CLUSTER CLUSTERED CLUSTERSTATUS COLLECTION COLUMNS COMMENT + COMPACT COMPACTIONS COMPUTE CONCATENATE CONTINUE DATA DATABASES + DATETIME DAY DBPROPERTIES DEFERRED DEFINED DELIMITED DEPENDENCY DESC + DIRECTORIES DIRECTORY DISABLE DISTRIBUTE ELEM_TYPE ENABLE ESCAPED + EXCLUSIVE EXPLAIN EXPORT FIELDS FILE FILEFORMAT FIRST FORMAT FORMATTED + FUNCTIONS HOLD_DDLTIME HOUR IDXPROPERTIES IGNORE INDEX INDEXES INPATH + INPUTDRIVER INPUTFORMAT ITEMS JAR KEYS KEY_TYPE LIMIT LINES LOAD + LOCATION LOCK LOCKS LOGICAL LONG MAPJOIN MATERIALIZED METADATA MINUS + MINUTE MONTH MSCK NOSCAN NO_DROP OFFLINE OPTION OUTPUTDRIVER + OUTPUTFORMAT OVERWRITE OWNER PARTITIONED PARTITIONS PLUS PRETTY + PRINCIPALS PROTECTION PURGE READ READONLY REBUILD RECORDREADER + RECORDWRITER REGEXP RELOAD RENAME REPAIR REPLACE REPLICATION RESTRICT + REWRITE RLIKE ROLE ROLES SCHEMA SCHEMAS SECOND SEMI SERDE + SERDEPROPERTIES SERVER SETS SHARED SHOW SHOW_DATABASE SKEWED SORT + SORTED SSL STATISTICS STORED STREAMTABLE STRING STRUCT TABLES + TBLPROPERTIES TEMPORARY TERMINATED TINYINT TOUCH TRANSACTIONS UNARCHIVE + UNDO UNIONTYPE UNLOCK UNSET UNSIGNED URI USE UTC UTCTIMESTAMP + VALUE_TYPE VIEW WHILE YEAR IF + + ALL ALTER AND ARRAY AS AUTHORIZATION BETWEEN BIGINT BINARY BOOLEAN + BOTH BY CASE CAST CHAR COLUMN CONF CREATE CROSS CUBE CURRENT + CURRENT_DATE CURRENT_TIMESTAMP CURSOR DATABASE DATE DECIMAL DELETE + DESCRIBE DISTINCT DOUBLE DROP ELSE END EXCHANGE EXISTS EXTENDED + EXTERNAL FALSE FETCH FLOAT FOLLOWING FOR FROM FULL FUNCTION GRANT + GROUP GROUPING HAVING IMPORT IN INNER INSERT INT INTERSECT + INTERVAL INTO IS JOIN LATERAL LEFT LESS LIKE LOCAL MACRO MAP MORE + NONE NOT NULL OF ON OR ORDER OUT OUTER OVER PARTIALSCAN PARTITION + PERCENT PRECEDING PRESERVE PROCEDURE RANGE READS REDUCE REVOKE RIGHT + ROLLUP ROW ROWS SELECT SET SMALLINT TABLE TABLESAMPLE THEN TIMESTAMP + TO TRANSFORM TRIGGER TRUE TRUNCATE UNBOUNDED UNION UNIQUEJOIN UPDATE + USER USING UTC_TMESTAMP VALUES VARCHAR WHEN WHERE WINDOW WITH + + AUTOCOMMIT ISOLATION LEVEL OFFSET SNAPSHOT TRANSACTION WORK WRITE + + COMMIT ONLY ROLLBACK START + + ABORT KEY LAST NORELY NOVALIDATE NULLS RELY VALIDATE + + CACHE CONSTRAINT FOREIGN PRIMARY REFERENCES + + DETAIL DOW EXPRESSION OPERATOR QUARTER SUMMARY VECTORIZATION WEEK YEARS MONTHS WEEKS DAYS HOURS MINUTES SECONDS + + DAYOFWEEK EXTRACT FLOOR INTEGER PRECISION VIEWS + + TIMESTAMPTZ ZONE + + TIME NUMERIC + + NAMED_STRUCT CREATE_UNION + + ROUND BROUND CEIL CEILING RAND EXP LN LOG10 LOG2 LOG POW POWER SQRT BIN + HEX UNHEX CONV ABS PMOD SIN ASIN COS ACOS TAN ATAN DEGREES RADIANS POSITIVE + NEGATIVE SIGN E PI FACTORIAL CBRT SHIFTLEFT SHIFTRIGHT SHIFTRIGHTUNSIGNED + GREATEST LEAST WIDTH_BUCKET SIZE MAP_KEYS MAP_VALUES ARRAY_CONTAINS + SORT_ARRAY FROM_UNIXTIME UNIX_TIMESTAMP TO_DATE DAYOFMONTH + WEEKOFYEAR DATEDIFF DATE_ADD DATE_SUB FROM_UTC_TIMESTAMP + TO_UTC_TIMESTAMP ADD_MONTHS LAST_DAY NEXT_DAY + TRUNC MONTHS_BETWEEN DATE_FORMAT ISNULL ISNOTNULL NVL COALESCE + then else end NULLIF ASSERT_TRUE ASCII BASE64 CHARACTER_LENGTH CHR CONCAT + CONTEXT_NGRAMS CONCAT_WS DECODE ELT ENCODE FIELD FIND_IN_SET + FORMAT_NUMBER GET_JSON_OBJECT IN_FILE INSTR LENGTH LOCATE LOWER LCASE LPAD LTRIM + NGRAMS OCTET_LENGTH PARSE_URL PRINTF REGEXP_EXTRACT REGEXP_REPLACE REPEAT REVERSE RPAD RTRIM SENTENCES SPACE SPLIT STR_TO_MAP SUBSTR SUBSTRING + SUBSTRING_INDEX TRANSLATE TRIM UNBASE64 UPPER UCASE INITCAP LEVENSHTEIN SOUNDEX + MASK MASK_FIRST_N MASK_LAST_N MASK_SHOW_FIRST_N MASK_SHOW_LAST_N MASK_HASH + JAVA_METHOD REFLECT HASH CURRENT_USER LOGGED_IN_USER CURRENT_DATABASE MD5 SHA1 + SHA CRC32 SHA2 AES_ENCRYPT AES_DECRYPT VERSION COUNT SUM AVG MIN MAX VARIANCE + VAR_POP VAR_SAMP STDDEV_POP STDDEV_SAMP COVAR_POP COVAR_SAMP CORR PERCENTILE + PERCENTILE_APPROX REGR_AVGX REGR_AVGY REGR_COUNT + REGR_INTERCEPT REGR_R2 REGR_SLOPE REGR_SXX REGR_SXY REGR_SYY HISTOGRAM_NUMERIC + COLLECT_SET COLLECT_LIST NTILE EXPLODE POSEXPLODE INLINE STACK + + JSON_TUPLE PARSE_URL_TUPLE + + XPATH XPATH_SHORT XPATH_INT XPATH_LONG XPATH_FLOAT XPATH_DOUBLE + XPATH_NUMBER XPATH_STRING + )) + end + + def self.keywords_type + # source: https://cwiki.apache.org/confluence/display/Hive/LanguageManual+Types + @keywords_type ||= Set.new(%w( + TINYINT SMALLINT INT INTEGER BIGINT FLOAT DOUBLE PRECISION DECIMAL NUMERIC + TIMESTAMP DATE INTERVAL + STRING VARCHAR CHAR + BOOLEAN BINARY + ARRAY MAP STRUCT UNIONTYPE + )) + end + + end + end +end From 00d82aab1f2976b0c5b4809db40f7eafc0a4da7c Mon Sep 17 00:00:00 2001 From: jneen Date: Thu, 16 Apr 2026 13:18:05 -0400 Subject: [PATCH 062/154] use a static regex for http and highlight content-type --- .rubocop_todo.yml | 1 - lib/rouge/lexers/http.rb | 13 ++++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index b03f3f52ce..42cef32c5c 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -43,7 +43,6 @@ Naming/VariableName: Rouge/NoBuildingAlternationPatternInRegexp: Exclude: - 'lib/rouge/lexers/armasm.rb' - - 'lib/rouge/lexers/http.rb' - 'lib/rouge/lexers/idris.rb' - 'lib/rouge/lexers/isabelle.rb' - 'lib/rouge/lexers/java.rb' diff --git a/lib/rouge/lexers/http.rb b/lib/rouge/lexers/http.rb index 681358dc07..0874971d29 100644 --- a/lib/rouge/lexers/http.rb +++ b/lib/rouge/lexers/http.rb @@ -10,9 +10,9 @@ class HTTP < RegexLexer option :content, "the language for the content (default: auto-detect)" - def self.http_methods - @http_methods ||= %w(GET POST PUT DELETE HEAD OPTIONS TRACE PATCH QUERY) - end + HTTP_METHODS = %r( + GET | POST | PUT | DELETE | HEAD | OPTIONS | TRACE | PATCH | QUERY + )mox def content_lexer @content_lexer ||= (lexer_option(:content) || guess_content_lexer) @@ -31,7 +31,7 @@ def guess_content_lexer state :root do # request rule %r( - (#{HTTP.http_methods.join('|')})([ ]+) # method + (#{HTTP_METHODS})([ ]+) # method ([^ ]+)([ ]+) # path (HTTPS?)(/)(\d(?:\.\d)?)(\r?\n|$) # http version )ox do @@ -65,9 +65,12 @@ def guess_content_lexer value = m[5] if key.strip.casecmp('content-type').zero? @content_type = value.split(';')[0].downcase + value_token = Keyword::Type + else + value_token = Str end - groups Name::Attribute, Text, Punctuation, Text, Str, Text + groups Name::Attribute, Text, Punctuation, Text, value_token, Text end rule %r/([^\r\n]+)(\r?\n|$)/ do From 3528c9c180de36b55eb57c56ccdbce16281b5b72 Mon Sep 17 00:00:00 2001 From: jneen Date: Thu, 16 Apr 2026 13:28:27 -0400 Subject: [PATCH 063/154] use the keywords api for hylang --- lib/rouge/lexers/hylang.rb | 60 ++++++++++++----------------- lib/rouge/lexers/hylang/builtins.rb | 31 +++++++++++++++ spec/visual/samples/hylang | 7 ++++ 3 files changed, 63 insertions(+), 35 deletions(-) create mode 100644 lib/rouge/lexers/hylang/builtins.rb diff --git a/lib/rouge/lexers/hylang.rb b/lib/rouge/lexers/hylang.rb index 17de1a2660..b5c6253caa 100644 --- a/lib/rouge/lexers/hylang.rb +++ b/lib/rouge/lexers/hylang.rb @@ -7,6 +7,8 @@ class HyLang < RegexLexer title "HyLang" desc "The HyLang programming language (hylang.org)" + lazy { require_relative 'hylang/builtins' } + tag 'hylang' aliases 'hy' @@ -14,36 +16,6 @@ class HyLang < RegexLexer mimetypes 'text/x-hy', 'application/x-hy' - def self.keywords - @keywords ||= Set.new %w( - False None True and as assert break class continue def - del elif else except finally for from global if import - in is lambda nonlocal not or pass raise return try - ) - end - - def self.builtins - @builtins ||= Set.new %w( - != % %= & &= * ** **= *= *map - + += , - -= -> ->> . / // - //= /= < << <<= <= = > >= >> - >>= @ @= ^ ^= accumulate apply as-> assoc butlast - calling-module-name car cdr chain coll? combinations comp complement compress cond - cons cons? constantly count cut cycle dec defclass defmacro defmacro! - defmacro/g! defmain defn defreader dict-comp disassemble dispatch-reader-macro distinct do doto - drop drop-last drop-while empty? eval eval-and-compile eval-when-compile even? every? filter - first flatten float? fn for* fraction genexpr gensym get group-by - identity if* if-not if-python2 inc input instance? integer integer-char? integer? - interleave interpose islice iterable? iterate iterator? juxt keyword keyword? last - let lif lif-not list* list-comp macro-error macroexpand macroexpand-1 map merge-with - multicombinations name neg? none? not-in not? nth numeric? odd? partition - permutations pos? product quasiquote quote range read read-str reduce remove - repeat repeatedly require rest second set-comp setv some string string? - symbol? take take-nth take-while tee unless unquote unquote-splicing when with* - with-decorator with-gensyms xor yield-from zero? zip zip-longest | |= ~ - ) - end - identifier = %r([\w!$%*+,<=>?/.-]+) keyword = %r([\w!\#$%*+,<=>?/.-]+) @@ -53,9 +25,13 @@ def name_token(name) nil end - state :root do + state :whitespace do rule %r/;.*?$/, Comment::Single rule %r/\s+/m, Text::Whitespace + end + + state :root do + mixin :whitespace rule %r/-?\d+\.\d+/, Num::Float rule %r/-?\d+/, Num::Integer @@ -69,10 +45,12 @@ def name_token(name) rule %r/~@|[`\'#^~&@]/, Operator - rule %r/(\()(\s*)(#{identifier})/m do |m| - token Punctuation, m[1] - token Text::Whitespace, m[2] - token(name_token(m[3]) || Name::Function, m[3]) + rule %r/[(]/, Punctuation, :form_start + + keywords identifier do + rule KEYWORDS, Keyword + rule BUILTINS, Name::Builtin + default Name end rule identifier do |m| @@ -88,6 +66,18 @@ def name_token(name) # parentheses rule %r/[()]/, Punctuation end + + state :form_start do + mixin :whitespace + + keywords identifier do + rule KEYWORDS, Keyword, :pop! + rule BUILTINS, Name::Builtin, :pop! + default Name::Function, :pop! + end + + rule(//) { pop! } + end end end end diff --git a/lib/rouge/lexers/hylang/builtins.rb b/lib/rouge/lexers/hylang/builtins.rb new file mode 100644 index 0000000000..f6b356ca91 --- /dev/null +++ b/lib/rouge/lexers/hylang/builtins.rb @@ -0,0 +1,31 @@ +module Rouge + module Lexers + class HyLang + KEYWORDS = Set.new %w( + False None True and as assert break class continue def + del elif else except finally for from global if import + in is lambda nonlocal not or pass raise return try + ) + + BUILTINS = Set.new %w( + != % %= & &= * ** **= *= *map + + += , - -= -> ->> . / // + //= /= < << <<= <= = > >= >> + >>= @ @= ^ ^= accumulate apply as-> assoc butlast + calling-module-name car cdr chain coll? combinations comp complement compress cond + cons cons? constantly count cut cycle dec defclass defmacro defmacro! + defmacro/g! defmain defn defreader dict-comp disassemble dispatch-reader-macro distinct do doto + drop drop-last drop-while empty? eval eval-and-compile eval-when-compile even? every? filter + first flatten float? fn for* fraction genexpr gensym get group-by + identity if* if-not if-python2 inc input instance? integer integer-char? integer? + interleave interpose islice iterable? iterate iterator? juxt keyword keyword? last + let lif lif-not list* list-comp macro-error macroexpand macroexpand-1 map merge-with + multicombinations name neg? none? not-in not? nth numeric? odd? partition + permutations pos? product quasiquote quote range read read-str reduce remove + repeat repeatedly require rest second set-comp setv some string string? + symbol? take take-nth take-while tee unless unquote unquote-splicing when with* + with-decorator with-gensyms xor yield-from zero? zip zip-longest | |= ~ + ) + end + end +end diff --git a/spec/visual/samples/hylang b/spec/visual/samples/hylang index 780eb96553..170b8a9de2 100644 --- a/spec/visual/samples/hylang +++ b/spec/visual/samples/hylang @@ -70,3 +70,10 @@ [(= highlight-library "rouge") (generate-rouge-file)] [True "Usage: hy hyghlight.hy [highlight.js | rouge]"])))) + +( ; here is a comment ))))) + +my-cool-function ; this should be highlighted Name.Function + +my-arg +) From a125674596014ccdd0bc1873957c6f83c5bb4f67 Mon Sep 17 00:00:00 2001 From: jneen Date: Fri, 17 Apr 2026 17:44:08 -0400 Subject: [PATCH 064/154] use the keywords api for IDLang --- lib/rouge/lexers/idlang.rb | 221 ++-------------------------- lib/rouge/lexers/idlang/builtins.rb | 171 +++++++++++++++++++++ 2 files changed, 184 insertions(+), 208 deletions(-) create mode 100644 lib/rouge/lexers/idlang/builtins.rb diff --git a/lib/rouge/lexers/idlang.rb b/lib/rouge/lexers/idlang.rb index 6431ce4ad2..8874c41ebe 100644 --- a/lib/rouge/lexers/idlang.rb +++ b/lib/rouge/lexers/idlang.rb @@ -16,197 +16,7 @@ class IDLang < RegexLexer kind_param = /(\d+|#{name})/ exponent = /[dDeE][+-]\d+/ - def self.exec_unit - @exec_unit ||= Set.new %w( - PRO FUNCTION - ) - end - - def self.keywords - @keywords ||= Set.new %w( - STRUCT INHERITS - RETURN CONTINUE BEGIN END BREAK GOTO - ) - end - - def self.standalone_statements - # Must not have a comma afterwards - @standalone_statements ||= Set.new %w( - COMMON FORWARD_FUNCTION - ) - end - - def self.decorators - # Must not have a comma afterwards - @decorators ||= Set.new %w( - COMPILE_OPT - ) - end - - def self.operators - @operators ||= Set.new %w( - AND= EQ= GE= GT= LE= LT= MOD= NE= OR= XOR= NOT= - ) - end - - def self.conditionals - @conditionals ||= Set.new %w( - OF DO ENDIF ENDELSE ENDFOR ENDFOREACH ENDWHILE ENDREP ENDCASE ENDSWITCH - IF THEN ELSE FOR FOREACH WHILE REPEAT UNTIL CASE SWITCH - AND EQ GE GT LE LT MOD NE OR XOR NOT - ) - end - - def self.routines - @routines ||= Set.new %w( - A_CORRELATE ABS ACOS ADAPT_HIST_EQUAL ALOG ALOG10 - AMOEBA ANNOTATE ARG_PRESENT ARRAY_EQUAL - ARRAY_INDICES ARROW ASCII_TEMPLATE ASIN ASSOC ATAN - AXIS BAR_PLOT BESELI BESELJ BESELK BESELY BETA - BILINEAR BIN_DATE BINARY_TEMPLATE BINDGEN BINOMIAL - BLAS_AXPY BLK_CON BOX_CURSOR BREAK BREAKPOINT - BROYDEN BYTARR BYTE BYTEORDER BYTSCL C_CORRELATE - CALDAT CALENDAR CALL_EXTERNAL CALL_FUNCTION - CALL_METHOD CALL_PROCEDURE CATCH CD CEIL CHEBYSHEV - CHECK_MATH CHISQR_CVF CHISQR_PDF CHOLDC CHOLSOL - CINDGEN CIR_3PNT CLOSE CLUST_WTS CLUSTER - COLOR_CONVERT COLOR_QUAN COLORMAP_APPLICABLE COMFIT - COMPLEX COMPLEXARR COMPLEXROUND - COMPUTE_MESH_NORMALS COND CONGRID CONJ - CONSTRAINED_MIN CONTOUR CONVERT_COORD CONVOL - COORD2TO3 CORRELATE COS COSH CRAMER CREATE_STRUCT - CREATE_VIEW CROSSP CRVLENGTH CT_LUMINANCE CTI_TEST - CURSOR CURVEFIT CV_COORD CVTTOBM CW_ANIMATE - CW_ANIMATE_GETP CW_ANIMATE_LOAD CW_ANIMATE_RUN - CW_ARCBALL CW_BGROUP CW_CLR_INDEX CW_COLORSEL - CW_DEFROI CW_FIELD CW_FILESEL CW_FORM CW_FSLIDER - CW_LIGHT_EDITOR CW_LIGHT_EDITOR_GET - CW_LIGHT_EDITOR_SET CW_ORIENT CW_PALETTE_EDITOR - CW_PALETTE_EDITOR_GET CW_PALETTE_EDITOR_SET - CW_PDMENU CW_RGBSLIDER CW_TMPL CW_ZOOM DBLARR - DCINDGEN DCOMPLEX DCOMPLEXARR DEFINE_KEY DEFROI - DEFSYSV DELETE_SYMBOL DELLOG DELVAR DERIV DERIVSIG - DETERM DEVICE DFPMIN DIALOG_MESSAGE - DIALOG_PICKFILE DIALOG_PRINTERSETUP - DIALOG_PRINTJOB DIALOG_READ_IMAGE - DIALOG_WRITE_IMAGE DICTIONARY DIGITAL_FILTER DILATE DINDGEN - DISSOLVE DIST DLM_LOAD DLM_REGISTER - DO_APPLE_SCRIPT DOC_LIBRARY DOUBLE DRAW_ROI EFONT - EIGENQL EIGENVEC ELMHES EMPTY ENABLE_SYSRTN EOF - ERASE ERODE ERRORF ERRPLOT EXECUTE EXIT EXP EXPAND - EXPAND_PATH EXPINT EXTRAC EXTRACT_SLICE F_CVF - F_PDF FACTORIAL FFT FILE_CHMOD FILE_DELETE - FILE_EXPAND_PATH FILE_MKDIR FILE_TEST FILE_WHICH - FILE_SEARCH PATH_SEP FILE_DIRNAME FILE_BASENAME - FILE_INFO FILE_MOVE FILE_COPY FILE_LINK FILE_POLL_INPUT - FILEPATH FINDFILE FINDGEN FINITE FIX FLICK FLOAT - FLOOR FLOW3 FLTARR FLUSH FORMAT_AXIS_VALUES - FORWARD_FUNCTION FREE_LUN FSTAT FULSTR FUNCT - FV_TEST FX_ROOT FZ_ROOTS GAMMA GAMMA_CT - GAUSS_CVF GAUSS_PDF GAUSS2DFIT GAUSSFIT GAUSSINT - GET_DRIVE_LIST GET_KBRD GET_LUN GET_SCREEN_SIZE - GET_SYMBOL GETENV GOTO GREG2JUL GRID_TPS GRID3 GS_ITER - H_EQ_CT H_EQ_INT HANNING HASH HEAP_GC HELP HILBERT - HIST_2D HIST_EQUAL HISTOGRAM HLS HOUGH HQR HSV - IBETA IDENTITY IDL_CONTAINER IDLANROI - IDLANROIGROUP IDLFFDICOM IDLFFDXF IDLFFLANGUAGECAT - IDLFFSHAPE IDLGRAXIS IDLGRBUFFER IDLGRCLIPBOARD - IDLGRCOLORBAR IDLGRCONTOUR IDLGRFONT IDLGRIMAGE - IDLGRLEGEND IDLGRLIGHT IDLGRMODEL IDLGRMPEG - IDLGRPALETTE IDLGRPATTERN IDLGRPLOT IDLGRPOLYGON - IDLGRPOLYLINE IDLGRPRINTER IDLGRROI IDLGRROIGROUP - IDLGRSCENE IDLGRSURFACE IDLGRSYMBOL - IDLGRTESSELLATOR IDLGRTEXT IDLGRVIEW - IDLGRVIEWGROUP IDLGRVOLUME IDLGRVRML IDLGRWINDOW - IGAMMA IMAGE_CONT IMAGE_STATISTICS IMAGINARY - INDGEN INT_2D INT_3D INT_TABULATED INTARR INTERPOL - INTERPOLATE INVERT IOCTL ISA ISHFT ISOCONTOUR - ISOSURFACE JOURNAL JUL2GREG JULDAY KEYWORD_SET KRIG2D - KURTOSIS KW_TEST L64INDGEN LABEL_DATE LABEL_REGION - LADFIT LAGUERRE LEEFILT LEGENDRE LINBCG LINDGEN - LINFIT LINKIMAGE LIST LIVE_CONTOUR LIVE_CONTROL - LIVE_DESTROY LIVE_EXPORT LIVE_IMAGE LIVE_INFO - LIVE_LINE LIVE_LOAD LIVE_OPLOT LIVE_PLOT - LIVE_PRINT LIVE_RECT LIVE_STYLE LIVE_SURFACE - LIVE_TEXT LJLCT LL_ARC_DISTANCE LMFIT LMGR LNGAMMA - LNP_TEST LOADCT LOCALE_GET LON64ARR LONARR LONG - LONG64 LSODE LU_COMPLEX LUDC LUMPROVE LUSOL - M_CORRELATE MACHAR MAKE_ARRAY MAKE_DLL MAP_2POINTS - MAP_CONTINENTS MAP_GRID MAP_IMAGE MAP_PATCH - MAP_PROJ_INFO MAP_SET MAX MATRIX_MULTIPLY MD_TEST MEAN - MEANABSDEV MEDIAN MEMORY MESH_CLIP MESH_DECIMATE - MESH_ISSOLID MESH_MERGE MESH_NUMTRIANGLES MESH_OBJ - MESH_SMOOTH MESH_SURFACEAREA MESH_VALIDATE - MESH_VOLUME MESSAGE MIN MIN_CURVE_SURF MK_HTML_HELP - MODIFYCT MOMENT MORPH_CLOSE MORPH_DISTANCE - MORPH_GRADIENT MORPH_HITORMISS MORPH_OPEN - MORPH_THIN MORPH_TOPHAT MPEG_CLOSE MPEG_OPEN - MPEG_PUT MPEG_SAVE MSG_CAT_CLOSE MSG_CAT_COMPILE - MSG_CAT_OPEN MULTI N_ELEMENTS N_PARAMS N_TAGS - NEWTON NORM OBJ_CLASS OBJ_DESTROY OBJ_ISA OBJ_NEW - OBJ_VALID OBJARR ON_ERROR ON_IOERROR ONLINE_HELP - OPEN OPENR OPENW OPENU OPLOT OPLOTERR ORDEREDHASH P_CORRELATE - PARTICLE_TRACE PCOMP PLOT PLOT_3DBOX PLOT_FIELD - PLOTERR PLOTS PNT_LINE POINT_LUN POLAR_CONTOUR - POLAR_SURFACE POLY POLY_2D POLY_AREA POLY_FIT - POLYFILL POLYFILLV POLYSHADE POLYWARP POPD POWELL - PRIMES PRINT PRINTF PRINTD PRODUCT PROFILE PROFILER - PROFILES PROJECT_VOL PS_SHOW_FONTS PSAFM PSEUDO - PTR_FREE PTR_NEW PTR_VALID PTRARR PUSHD QROMB - QROMO QSIMP QUERY_CSV R_CORRELATE R_TEST RADON RANDOMN - RANDOMU RANKS RDPIX READ READF READ_ASCII - READ_BINARY READ_BMP READ_CSV READ_DICOM READ_IMAGE - READ_INTERFILE READ_JPEG READ_PICT READ_PNG - READ_PPM READ_SPR READ_SRF READ_SYLK READ_TIFF - READ_WAV READ_WAVE READ_X11_BITMAP READ_XWD READS - READU REBIN RECALL_COMMANDS RECON3 REDUCE_COLORS - REFORM REGRESS REPLICATE REPLICATE_INPLACE - RESOLVE_ALL RESOLVE_ROUTINE RESTORE RETALL - REVERSE REWIND RK4 ROBERTS ROT ROTATE ROUND - ROUTINE_INFO RS_TEST S_TEST SAVE SAVGOL SCALE3 - SCALE3D SCOPE_LEVEL SCOPE_TRACEBACK SCOPE_VARFETCH - SCOPE_VARNAME SEARCH2D SEARCH3D SET_PLOT SET_SHADING - SET_SYMBOL SETENV SETLOG SETUP_KEYS SFIT - SHADE_SURF SHADE_SURF_IRR SHADE_VOLUME SHIFT SHOW3 - SHOWFONT SIGNUM SIN SINDGEN SINH SIZE SKEWNESS SKIPF - SLICER3 SLIDE_IMAGE SMOOTH SOBEL SOCKET SORT SPAWN - SPH_4PNT SPH_SCAT SPHER_HARM SPL_INIT SPL_INTERP - SPLINE SPLINE_P SPRSAB SPRSAX SPRSIN SPRSTP SQRT - STANDARDIZE STDDEV STOP STRARR STRCMP STRCOMPRESS - STREAMLINE STREGEX STRETCH STRING STRJOIN STRLEN - STRLOWCASE STRMATCH STRMESSAGE STRMID STRPOS - STRPUT STRSPLIT STRTRIM STRUCT_ASSIGN STRUCT_HIDE - STRUPCASE SURFACE SURFR SVDC SVDFIT SVSOL - SWAP_ENDIAN SWITCH SYSTIME T_CVF T_PDF T3D - TAG_NAMES TAN TANH TAPRD TAPWRT TEK_COLOR - TEMPORARY TETRA_CLIP TETRA_SURFACE TETRA_VOLUME - THIN THREED TIME_TEST2 TIMEGEN TM_TEST TOTAL TRACE - TRANSPOSE TRI_SURF TRIANGULATE TRIGRID TRIQL - TRIRED TRISOL TRNLOG TS_COEF TS_DIFF TS_FCAST - TS_SMOOTH TV TVCRS TVLCT TVRD TVSCL TYPENAME UINDGEN UINT - UINTARR UL64INDGEN ULINDGEN ULON64ARR ULONARR - ULONG ULONG64 UNIQ USERSYM VALUE_LOCATE VARIANCE - VAX_FLOAT VECTOR_FIELD VEL VELOVECT VERT_T3D VOIGT - VORONOI VOXEL_PROJ WAIT WARP_TRI WATERSHED WDELETE - WEOF WF_DRAW WHERE WIDGET_BASE WIDGET_BUTTON - WIDGET_CONTROL WIDGET_DRAW WIDGET_DROPLIST - WIDGET_EVENT WIDGET_INFO WIDGET_LABEL WIDGET_LIST - WIDGET_SLIDER WIDGET_TABLE WIDGET_TEXT WINDOW - WRITE_BMP WRITE_CSV WRITE_IMAGE WRITE_JPEG WRITE_NRIF - WRITE_PICT WRITE_PNG WRITE_PPM WRITE_SPR WRITE_SRF - WRITE_SYLK WRITE_TIFF WRITE_WAV WRITE_WAVE WRITEU - WSET WSHOW WTN WV_APPLET WV_CW_WAVELET WV_CWT - WV_DENOISE WV_DWT WV_FN_COIFLET WV_FN_DAUBECHIES - WV_FN_GAUSSIAN WV_FN_HAAR WV_FN_MORLET WV_FN_PAUL - WV_FN_SYMLET WV_IMPORT_DATA WV_IMPORT_WAVELET - WV_PLOT3D_WPS WV_PLOT_MULTIRES WV_PWT - WV_TOOL_DENOISE XBM_EDIT XDISPLAYFILE XDXF XFONT - XINTERANIMATE XLOADCT XMANAGER XMNG_TMPL XMTOOL - XOBJVIEW XPALETTE XPCOLOR XPLOT3D XREGISTERED XROI - XSQ_TEST XSURFACE XVAREDIT XVOLUME XVOLUME_ROTATE - XVOLUME_WRITE_IMAGE XYOUTS ZOOM ZOOM_24 - ) - end + lazy { require_relative 'idlang/builtins' } state :root do rule %r/\s+/, Text::Whitespace @@ -253,23 +63,18 @@ def self.routines push :funcname end - rule %r/#{name}/m do |m| - match = m[0].upcase - if self.class.keywords.include? match - token Keyword - elsif self.class.conditionals.include? match - token Keyword - elsif self.class.decorators.include? match - token Name::Decorator - elsif self.class.standalone_statements.include? match - token Keyword::Reserved - elsif self.class.operators.include? match - token Operator::Word - elsif self.class.routines.include? match - token Name::Builtin - else - token Name - end + rule %r((?:AND|EQ|GE|GT|LE|LT|MOD|NE|OR|XOR|NOT)=), Operator + + keywords %r/#{name}=?/m do + transform(&:upcase) + + rule KEYWORDS, Keyword + rule CONDITIONALS, Keyword + rule DECORATORS, Keyword + rule STANDALONE_STATEMENTS, Keyword::Reserved + rule ROUTINES, Name::Builtin + + default Name end end diff --git a/lib/rouge/lexers/idlang/builtins.rb b/lib/rouge/lexers/idlang/builtins.rb new file mode 100644 index 0000000000..ce62e4e724 --- /dev/null +++ b/lib/rouge/lexers/idlang/builtins.rb @@ -0,0 +1,171 @@ +module Rouge + module Lexers + class IDLang + EXEC_UNIT = Set.new %w(PRO FUNCTION) + + KEYWORDS = Set.new %w( + STRUCT INHERITS + RETURN CONTINUE BEGIN END BREAK GOTO + ) + + STANDALONE_STATEMENTS = Set.new %w(COMMON FORWARD_FUNCTION) + + DECORATORS = Set.new %w(COMPILE_OPT) + + CONDITIONALS = Set.new %w( + OF DO ENDIF ENDELSE ENDFOR ENDFOREACH ENDWHILE ENDREP ENDCASE ENDSWITCH + IF THEN ELSE FOR FOREACH WHILE REPEAT UNTIL CASE SWITCH + AND EQ GE GT LE LT MOD NE OR XOR NOT + ) + + ROUTINES = Set.new %w( + A_CORRELATE ABS ACOS ADAPT_HIST_EQUAL ALOG ALOG10 + AMOEBA ANNOTATE ARG_PRESENT ARRAY_EQUAL + ARRAY_INDICES ARROW ASCII_TEMPLATE ASIN ASSOC ATAN + AXIS BAR_PLOT BESELI BESELJ BESELK BESELY BETA + BILINEAR BIN_DATE BINARY_TEMPLATE BINDGEN BINOMIAL + BLAS_AXPY BLK_CON BOX_CURSOR BREAK BREAKPOINT + BROYDEN BYTARR BYTE BYTEORDER BYTSCL C_CORRELATE + CALDAT CALENDAR CALL_EXTERNAL CALL_FUNCTION + CALL_METHOD CALL_PROCEDURE CATCH CD CEIL CHEBYSHEV + CHECK_MATH CHISQR_CVF CHISQR_PDF CHOLDC CHOLSOL + CINDGEN CIR_3PNT CLOSE CLUST_WTS CLUSTER + COLOR_CONVERT COLOR_QUAN COLORMAP_APPLICABLE COMFIT + COMPLEX COMPLEXARR COMPLEXROUND + COMPUTE_MESH_NORMALS COND CONGRID CONJ + CONSTRAINED_MIN CONTOUR CONVERT_COORD CONVOL + COORD2TO3 CORRELATE COS COSH CRAMER CREATE_STRUCT + CREATE_VIEW CROSSP CRVLENGTH CT_LUMINANCE CTI_TEST + CURSOR CURVEFIT CV_COORD CVTTOBM CW_ANIMATE + CW_ANIMATE_GETP CW_ANIMATE_LOAD CW_ANIMATE_RUN + CW_ARCBALL CW_BGROUP CW_CLR_INDEX CW_COLORSEL + CW_DEFROI CW_FIELD CW_FILESEL CW_FORM CW_FSLIDER + CW_LIGHT_EDITOR CW_LIGHT_EDITOR_GET + CW_LIGHT_EDITOR_SET CW_ORIENT CW_PALETTE_EDITOR + CW_PALETTE_EDITOR_GET CW_PALETTE_EDITOR_SET + CW_PDMENU CW_RGBSLIDER CW_TMPL CW_ZOOM DBLARR + DCINDGEN DCOMPLEX DCOMPLEXARR DEFINE_KEY DEFROI + DEFSYSV DELETE_SYMBOL DELLOG DELVAR DERIV DERIVSIG + DETERM DEVICE DFPMIN DIALOG_MESSAGE + DIALOG_PICKFILE DIALOG_PRINTERSETUP + DIALOG_PRINTJOB DIALOG_READ_IMAGE + DIALOG_WRITE_IMAGE DICTIONARY DIGITAL_FILTER DILATE DINDGEN + DISSOLVE DIST DLM_LOAD DLM_REGISTER + DO_APPLE_SCRIPT DOC_LIBRARY DOUBLE DRAW_ROI EFONT + EIGENQL EIGENVEC ELMHES EMPTY ENABLE_SYSRTN EOF + ERASE ERODE ERRORF ERRPLOT EXECUTE EXIT EXP EXPAND + EXPAND_PATH EXPINT EXTRAC EXTRACT_SLICE F_CVF + F_PDF FACTORIAL FFT FILE_CHMOD FILE_DELETE + FILE_EXPAND_PATH FILE_MKDIR FILE_TEST FILE_WHICH + FILE_SEARCH PATH_SEP FILE_DIRNAME FILE_BASENAME + FILE_INFO FILE_MOVE FILE_COPY FILE_LINK FILE_POLL_INPUT + FILEPATH FINDFILE FINDGEN FINITE FIX FLICK FLOAT + FLOOR FLOW3 FLTARR FLUSH FORMAT_AXIS_VALUES + FORWARD_FUNCTION FREE_LUN FSTAT FULSTR FUNCT + FV_TEST FX_ROOT FZ_ROOTS GAMMA GAMMA_CT + GAUSS_CVF GAUSS_PDF GAUSS2DFIT GAUSSFIT GAUSSINT + GET_DRIVE_LIST GET_KBRD GET_LUN GET_SCREEN_SIZE + GET_SYMBOL GETENV GOTO GREG2JUL GRID_TPS GRID3 GS_ITER + H_EQ_CT H_EQ_INT HANNING HASH HEAP_GC HELP HILBERT + HIST_2D HIST_EQUAL HISTOGRAM HLS HOUGH HQR HSV + IBETA IDENTITY IDL_CONTAINER IDLANROI + IDLANROIGROUP IDLFFDICOM IDLFFDXF IDLFFLANGUAGECAT + IDLFFSHAPE IDLGRAXIS IDLGRBUFFER IDLGRCLIPBOARD + IDLGRCOLORBAR IDLGRCONTOUR IDLGRFONT IDLGRIMAGE + IDLGRLEGEND IDLGRLIGHT IDLGRMODEL IDLGRMPEG + IDLGRPALETTE IDLGRPATTERN IDLGRPLOT IDLGRPOLYGON + IDLGRPOLYLINE IDLGRPRINTER IDLGRROI IDLGRROIGROUP + IDLGRSCENE IDLGRSURFACE IDLGRSYMBOL + IDLGRTESSELLATOR IDLGRTEXT IDLGRVIEW + IDLGRVIEWGROUP IDLGRVOLUME IDLGRVRML IDLGRWINDOW + IGAMMA IMAGE_CONT IMAGE_STATISTICS IMAGINARY + INDGEN INT_2D INT_3D INT_TABULATED INTARR INTERPOL + INTERPOLATE INVERT IOCTL ISA ISHFT ISOCONTOUR + ISOSURFACE JOURNAL JUL2GREG JULDAY KEYWORD_SET KRIG2D + KURTOSIS KW_TEST L64INDGEN LABEL_DATE LABEL_REGION + LADFIT LAGUERRE LEEFILT LEGENDRE LINBCG LINDGEN + LINFIT LINKIMAGE LIST LIVE_CONTOUR LIVE_CONTROL + LIVE_DESTROY LIVE_EXPORT LIVE_IMAGE LIVE_INFO + LIVE_LINE LIVE_LOAD LIVE_OPLOT LIVE_PLOT + LIVE_PRINT LIVE_RECT LIVE_STYLE LIVE_SURFACE + LIVE_TEXT LJLCT LL_ARC_DISTANCE LMFIT LMGR LNGAMMA + LNP_TEST LOADCT LOCALE_GET LON64ARR LONARR LONG + LONG64 LSODE LU_COMPLEX LUDC LUMPROVE LUSOL + M_CORRELATE MACHAR MAKE_ARRAY MAKE_DLL MAP_2POINTS + MAP_CONTINENTS MAP_GRID MAP_IMAGE MAP_PATCH + MAP_PROJ_INFO MAP_SET MAX MATRIX_MULTIPLY MD_TEST MEAN + MEANABSDEV MEDIAN MEMORY MESH_CLIP MESH_DECIMATE + MESH_ISSOLID MESH_MERGE MESH_NUMTRIANGLES MESH_OBJ + MESH_SMOOTH MESH_SURFACEAREA MESH_VALIDATE + MESH_VOLUME MESSAGE MIN MIN_CURVE_SURF MK_HTML_HELP + MODIFYCT MOMENT MORPH_CLOSE MORPH_DISTANCE + MORPH_GRADIENT MORPH_HITORMISS MORPH_OPEN + MORPH_THIN MORPH_TOPHAT MPEG_CLOSE MPEG_OPEN + MPEG_PUT MPEG_SAVE MSG_CAT_CLOSE MSG_CAT_COMPILE + MSG_CAT_OPEN MULTI N_ELEMENTS N_PARAMS N_TAGS + NEWTON NORM OBJ_CLASS OBJ_DESTROY OBJ_ISA OBJ_NEW + OBJ_VALID OBJARR ON_ERROR ON_IOERROR ONLINE_HELP + OPEN OPENR OPENW OPENU OPLOT OPLOTERR ORDEREDHASH P_CORRELATE + PARTICLE_TRACE PCOMP PLOT PLOT_3DBOX PLOT_FIELD + PLOTERR PLOTS PNT_LINE POINT_LUN POLAR_CONTOUR + POLAR_SURFACE POLY POLY_2D POLY_AREA POLY_FIT + POLYFILL POLYFILLV POLYSHADE POLYWARP POPD POWELL + PRIMES PRINT PRINTF PRINTD PRODUCT PROFILE PROFILER + PROFILES PROJECT_VOL PS_SHOW_FONTS PSAFM PSEUDO + PTR_FREE PTR_NEW PTR_VALID PTRARR PUSHD QROMB + QROMO QSIMP QUERY_CSV R_CORRELATE R_TEST RADON RANDOMN + RANDOMU RANKS RDPIX READ READF READ_ASCII + READ_BINARY READ_BMP READ_CSV READ_DICOM READ_IMAGE + READ_INTERFILE READ_JPEG READ_PICT READ_PNG + READ_PPM READ_SPR READ_SRF READ_SYLK READ_TIFF + READ_WAV READ_WAVE READ_X11_BITMAP READ_XWD READS + READU REBIN RECALL_COMMANDS RECON3 REDUCE_COLORS + REFORM REGRESS REPLICATE REPLICATE_INPLACE + RESOLVE_ALL RESOLVE_ROUTINE RESTORE RETALL + REVERSE REWIND RK4 ROBERTS ROT ROTATE ROUND + ROUTINE_INFO RS_TEST S_TEST SAVE SAVGOL SCALE3 + SCALE3D SCOPE_LEVEL SCOPE_TRACEBACK SCOPE_VARFETCH + SCOPE_VARNAME SEARCH2D SEARCH3D SET_PLOT SET_SHADING + SET_SYMBOL SETENV SETLOG SETUP_KEYS SFIT + SHADE_SURF SHADE_SURF_IRR SHADE_VOLUME SHIFT SHOW3 + SHOWFONT SIGNUM SIN SINDGEN SINH SIZE SKEWNESS SKIPF + SLICER3 SLIDE_IMAGE SMOOTH SOBEL SOCKET SORT SPAWN + SPH_4PNT SPH_SCAT SPHER_HARM SPL_INIT SPL_INTERP + SPLINE SPLINE_P SPRSAB SPRSAX SPRSIN SPRSTP SQRT + STANDARDIZE STDDEV STOP STRARR STRCMP STRCOMPRESS + STREAMLINE STREGEX STRETCH STRING STRJOIN STRLEN + STRLOWCASE STRMATCH STRMESSAGE STRMID STRPOS + STRPUT STRSPLIT STRTRIM STRUCT_ASSIGN STRUCT_HIDE + STRUPCASE SURFACE SURFR SVDC SVDFIT SVSOL + SWAP_ENDIAN SWITCH SYSTIME T_CVF T_PDF T3D + TAG_NAMES TAN TANH TAPRD TAPWRT TEK_COLOR + TEMPORARY TETRA_CLIP TETRA_SURFACE TETRA_VOLUME + THIN THREED TIME_TEST2 TIMEGEN TM_TEST TOTAL TRACE + TRANSPOSE TRI_SURF TRIANGULATE TRIGRID TRIQL + TRIRED TRISOL TRNLOG TS_COEF TS_DIFF TS_FCAST + TS_SMOOTH TV TVCRS TVLCT TVRD TVSCL TYPENAME UINDGEN UINT + UINTARR UL64INDGEN ULINDGEN ULON64ARR ULONARR + ULONG ULONG64 UNIQ USERSYM VALUE_LOCATE VARIANCE + VAX_FLOAT VECTOR_FIELD VEL VELOVECT VERT_T3D VOIGT + VORONOI VOXEL_PROJ WAIT WARP_TRI WATERSHED WDELETE + WEOF WF_DRAW WHERE WIDGET_BASE WIDGET_BUTTON + WIDGET_CONTROL WIDGET_DRAW WIDGET_DROPLIST + WIDGET_EVENT WIDGET_INFO WIDGET_LABEL WIDGET_LIST + WIDGET_SLIDER WIDGET_TABLE WIDGET_TEXT WINDOW + WRITE_BMP WRITE_CSV WRITE_IMAGE WRITE_JPEG WRITE_NRIF + WRITE_PICT WRITE_PNG WRITE_PPM WRITE_SPR WRITE_SRF + WRITE_SYLK WRITE_TIFF WRITE_WAV WRITE_WAVE WRITEU + WSET WSHOW WTN WV_APPLET WV_CW_WAVELET WV_CWT + WV_DENOISE WV_DWT WV_FN_COIFLET WV_FN_DAUBECHIES + WV_FN_GAUSSIAN WV_FN_HAAR WV_FN_MORLET WV_FN_PAUL + WV_FN_SYMLET WV_IMPORT_DATA WV_IMPORT_WAVELET + WV_PLOT3D_WPS WV_PLOT_MULTIRES WV_PWT + WV_TOOL_DENOISE XBM_EDIT XDISPLAYFILE XDXF XFONT + XINTERANIMATE XLOADCT XMANAGER XMNG_TMPL XMTOOL + XOBJVIEW XPALETTE XPCOLOR XPLOT3D XREGISTERED XROI + XSQ_TEST XSURFACE XVAREDIT XVOLUME XVOLUME_ROTATE + XVOLUME_WRITE_IMAGE XYOUTS ZOOM ZOOM_24 + ) + end + end +end From 0c81688ff939468cff5955f46a38be3152d52122 Mon Sep 17 00:00:00 2001 From: jneen Date: Fri, 24 Apr 2026 15:14:55 -0400 Subject: [PATCH 065/154] use the keywords api for idris --- .rubocop_todo.yml | 1 - lib/rouge/lexers/idris.rb | 101 +++++++++++++++++++++----------------- 2 files changed, 56 insertions(+), 46 deletions(-) diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index 42cef32c5c..2390568508 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -43,7 +43,6 @@ Naming/VariableName: Rouge/NoBuildingAlternationPatternInRegexp: Exclude: - 'lib/rouge/lexers/armasm.rb' - - 'lib/rouge/lexers/idris.rb' - 'lib/rouge/lexers/isabelle.rb' - 'lib/rouge/lexers/java.rb' - 'lib/rouge/lexers/jsp.rb' diff --git a/lib/rouge/lexers/idris.rb b/lib/rouge/lexers/idris.rb index c1de5d1326..cde390b040 100644 --- a/lib/rouge/lexers/idris.rb +++ b/lib/rouge/lexers/idris.rb @@ -1,6 +1,9 @@ # -*- coding: utf-8 -*- # # frozen_string_literal: true +# For Haskell::ASCII_ESCAPE +require_relative 'haskell' + module Rouge module Lexers class Idris < RegexLexer @@ -12,36 +15,37 @@ class Idris < RegexLexer filenames '*.idr' mimetypes 'text/x-idris' - def self.reserved_keywords - @reserved_keywords ||= %w( - _ data class instance namespace - infix[lr]? let where of with type - do if then else case in - ) - end - - def self.ascii - @ascii ||= %w( - NUL SOH [SE]TX EOT ENQ ACK BEL BS HT LF VT FF CR S[OI] DLE - DC[1-4] NAK SYN ETB CAN EM SUB ESC [FGRU]S SP DEL - ) - end - - def self.prelude_functions - @prelude_functions ||= %w( - abs acos all and any asin atan atan2 break ceiling compare concat - concatMap const cos cosh curry cycle div drop dropWhile elem - encodeFloat enumFrom enumFromThen enumFromThenTo enumFromTo exp - fail filter flip floor foldl foldl1 foldr foldr1 fromInteger fst - gcd getChar getLine head id init iterate last lcm length lines log - lookup map max maxBound maximum maybe min minBound minimum mod - negate not null or pi pred print product putChar putStr putStrLn - readFile recip repeat replicate return reverse scanl scanl1 sequence - sequence_ show sin sinh snd span splitAt sqrt succ sum tail take - takeWhile tan tanh uncurry unlines unwords unzip unzip3 words - writeFile zip zip3 zipWith zipWith3 - ) - end + RESERVED = Set.new %w( + _ data class instance namespace + infixl infixr let where of with type + do if then else case in + ) + + PRELUDE_FUNCTIONS = Set.new %w( + abs acos all and any asin atan atan2 break ceiling compare concat + concatMap const cos cosh curry cycle div drop dropWhile elem + encodeFloat enumFrom enumFromThen enumFromThenTo enumFromTo exp + fail filter flip floor foldl foldl1 foldr foldr1 fromInteger fst + gcd getChar getLine head id init iterate last lcm length lines log + lookup map max maxBound maximum maybe min minBound minimum mod + negate not null or pi pred print product putChar putStr putStrLn + readFile recip repeat replicate return reverse scanl scanl1 sequence + sequence_ show sin sinh snd span splitAt sqrt succ sum tail take + takeWhile tan tanh uncurry unlines unwords unzip unzip3 words + writeFile zip zip3 zipWith zipWith3 + ) + + CONSTANTS = Set.new %w(Just Nothing Left Right True False LT LTE EQ GT GTE) + + KEYWORD_TYPE = Set.new %w( + Type Exists World IO IntTy FTy File Mode Dec Bool Ordering Either IsJust + List Maybe Nat Stream StrM Not Lazy Inf + ) + + NAME_CLASS = Set.new %w( + Eq Ord Num MinBound MaxBound Integral Applicative Alternative Cast Foldable + Functor Monad Traversable Uninhabited Semigroup Monoid + ) state :basic do rule %r/\s+/m, Text @@ -58,6 +62,7 @@ def self.prelude_functions rule %r/[^-{}]+/, Comment::Multiline rule %r/[-{}]/, Comment::Multiline end + state :comment_preproc do rule %r/-}/, Comment::Preproc, :pop! rule %r/{-/, Comment::Preproc, :comment @@ -72,26 +77,32 @@ def self.prelude_functions rule %r/\%(provide)\s+.*\s+(with)\s+/, Keyword # type end - state :prelude do - rule %r/\b(Type|Exists|World|IO|IntTy|FTy|File|Mode|Dec|Bool|Ordering|Either|IsJust|List|Maybe|Nat|Stream|StrM|Not|Lazy|Inf)\s/, Keyword::Type - rule %r/\b(Eq|Ord|Num|MinBound|MaxBound|Integral|Applicative|Alternative|Cast|Foldable|Functor|Monad|Traversable|Uninhabited|Semigroup|Monoid)\s/, Name::Class - rule %r/\b(?:#{Idris.prelude_functions.join('|')})[ ]+(?![=:-])/, Name::Builtin - end - state :root do mixin :basic mixin :directive - rule %r/\bimport\b/, Keyword::Reserved, :import - rule %r/\bmodule\b/, Keyword::Reserved, :module - rule %r/\b(?:#{Idris.reserved_keywords.join('|')})\b/, Keyword::Reserved - rule %r/\b(Just|Nothing|Left|Right|True|False|LT|LTE|EQ|GT|GTE)\b/, Keyword::Constant # function signature + # Special since you can override names defined in Prelude rule %r/^[\w']+\s*:/, Name::Function - # should be below as you can override names defined in Prelude - mixin :prelude - rule %r/[_a-z][\w']*/, Name - rule %r/[A-Z][\w']*/, Keyword::Type + + keywords %r/[_a-z][\w']*/i do + rule Set['import'], Keyword::Reserved, :import + rule Set['module'], Keyword::Reserved, :module + rule Set['public', 'export', 'partial'], Keyword::Namespace + rule RESERVED, Keyword::Reserved + rule CONSTANTS, Keyword::Constant + rule KEYWORD_TYPE, Keyword::Type + rule NAME_CLASS, Name::Class + rule PRELUDE_FUNCTIONS, Name::Builtin + + default do |m| + if m[0].match?(/\A'?[A-Z]/o) + token Keyword::Type + else + token Name + end + end + end # lambda operator rule %r(\\(?![:!#\$\%&*+.\\/<=>?@^\|~-]+)), Name::Function @@ -199,7 +210,7 @@ def self.prelude_functions state :escape do rule %r/[abfnrtv"'&\\]/, Str::Escape, :pop! rule %r/\^[\]\[A-Z@\^_]/, Str::Escape, :pop! - rule %r/#{Idris.ascii.join('|')}/, Str::Escape, :pop! + rule Haskell::ASCII_ESCAPE, Str::Escape, :pop! rule %r/o[0-7]+/i, Str::Escape, :pop! rule %r/x[\da-fA-F]+/i, Str::Escape, :pop! rule %r/\d+/, Str::Escape, :pop! From c1fe5a6dc15916b462879e07e67d9ee24287164d Mon Sep 17 00:00:00 2001 From: jneen Date: Fri, 24 Apr 2026 15:24:44 -0400 Subject: [PATCH 066/154] use the keywords api for iecst --- lib/rouge/lexers/iecst.rb | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/lib/rouge/lexers/iecst.rb b/lib/rouge/lexers/iecst.rb index d1e11969fb..c2af371f7c 100644 --- a/lib/rouge/lexers/iecst.rb +++ b/lib/rouge/lexers/iecst.rb @@ -67,19 +67,14 @@ def self.operators rule %r/(?:\b\d+(?:\.\d*)?|\B\.\d+)(?:e[+-]?\d+)?/i, Literal::Number::Float rule %r/\b[\d.,_]+/, Literal::Number - rule %r/\b[A-Z_]+\b/i do |m| - name = m[0].upcase - if self.class.keywords.include?(name) - token Keyword - elsif self.class.types.include?(name) - token Keyword::Type - elsif self.class.literals.include?(name) - token Literal - elsif self.class.operators.include?(name) - token Operator - else - token Name - end + keywords %r/\b[A-Z_]+\b/i do + transform(&:upcase) + + rule :keywords, Keyword + rule :types, Keyword::Type + rule :literals, Literal + rule :operators, Operator + default Name end rule %r/S?R?:?=>?|&&?|\*\*?|<[=>]?|>=?|[-:^\/+#]/, Operator From 92daecfd47996cc9be1ac4c9c19b9e8790c77a7d Mon Sep 17 00:00:00 2001 From: jneen Date: Fri, 24 Apr 2026 17:43:18 -0400 Subject: [PATCH 067/154] use the keyword api for igor pro --- .rubocop_todo.yml | 3 --- lib/rouge/lexers/igorpro.rb | 45 ++++++++++++++----------------------- 2 files changed, 17 insertions(+), 31 deletions(-) diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index 2390568508..00c1a6b24b 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -26,7 +26,6 @@ Naming/MemoizedInstanceVariableName: # ForbiddenIdentifiers: __id__, __send__ Naming/MethodName: Exclude: - - 'lib/rouge/lexers/igorpro.rb' - 'lib/rouge/lexers/xpath.rb' # Offense count: 40 @@ -35,7 +34,6 @@ Naming/MethodName: Naming/VariableName: Exclude: - 'lib/rouge/lexers/dafny.rb' - - 'lib/rouge/lexers/igorpro.rb' - 'lib/rouge/lexers/kotlin.rb' - 'lib/rouge/lexers/xpath.rb' @@ -69,7 +67,6 @@ Rouge/NoBuildingAlternationPatternInRegexp: Rouge/NoHugeCollections: Exclude: - 'lib/rouge/lexers/idlang.rb' - - 'lib/rouge/lexers/igorpro.rb' - 'lib/rouge/lexers/janet.rb' - 'lib/rouge/lexers/openedge.rb' - 'lib/rouge/lexers/plsql.rb' diff --git a/lib/rouge/lexers/igorpro.rb b/lib/rouge/lexers/igorpro.rb index aaeaa64178..9be171c898 100644 --- a/lib/rouge/lexers/igorpro.rb +++ b/lib/rouge/lexers/igorpro.rb @@ -40,7 +40,7 @@ class IgorPro < RegexLexer lazy { require_relative 'igorpro/builtins' } object = /[a-z][a-z0-9_\.]*/i - noLineBreak = /(?:[ \t]|(?:\\\s*[\r\n]))+/ + no_line_break = /(?:[ \t]|(?:\\\s*[\r\n]))+/ operator = %r([\#$~!%^&*+=\|?:<>/-]) punctuation = /[{}()\[\],.;]/ number_float= /0x[a-f0-9]+/i @@ -50,29 +50,18 @@ class IgorPro < RegexLexer state :root do rule %r(//), Comment, :comments - rule object do |m| - obj = m[0].downcase - - if obj.include?('function') - token Keyword::Declaration - push :parse_function - elsif DECLARATIONS.include?(obj) - token Keyword::Declaration - push :parse_variables - elsif KEYWORDS.include?(obj) - token Keyword - elsif CONSTANTS.include?(obj) - token Keyword::Constant - elsif FUNCTIONS.include?(obj) - token Name::Builtin - elsif OPERATIONS.include?(obj) - token Keyword::Reserved - push :operationFlags - elsif obj.match?(/\A(v|s|w)_[a-z]+[a-z0-9]*/o) - token Name::Constant - else - token Name - end + rule %r/(v|s|w)_[a-z]+[a-z0-9]*/i, Name::Constant + + keywords object do + transform(&:downcase) + + rule Set['function'], Keyword::Declaration, :parse_function + rule DECLARATIONS, Keyword::Declaration, :parse_variables + rule KEYWORDS, Keyword + rule CONSTANTS, Keyword::Constant + rule FUNCTIONS, Name::Builtin + rule OPERATIONS, Keyword::Reserved, :operation_flags + default Name end mixin :preprocessor @@ -124,10 +113,10 @@ class IgorPro < RegexLexer rule(//) { pop! } end - state :operationFlags do - rule noLineBreak, Text + state :operation_flags do + rule no_line_break, Text rule %r/[=]/, Punctuation, :assignment - rule %r([/][a-z]+)i, Keyword::Pseudo, :operationFlags + rule %r([/][a-z]+)i, Keyword::Pseudo, :operation_flags rule %r/(as)(\s*)(#{object})/i do groups Keyword::Type, Text, Name::Label end @@ -164,7 +153,7 @@ class IgorPro < RegexLexer end state :whitespace do - rule noLineBreak, Text + rule no_line_break, Text end state :string1 do From e1b181bad258ddb3a00c819b341088ad092ca566 Mon Sep 17 00:00:00 2001 From: jneen Date: Fri, 24 Apr 2026 17:45:04 -0400 Subject: [PATCH 068/154] use the keywords api for io --- lib/rouge/lexers/io.rb | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/lib/rouge/lexers/io.rb b/lib/rouge/lexers/io.rb index 9e049a9ca3..7cd917859a 100644 --- a/lib/rouge/lexers/io.rb +++ b/lib/rouge/lexers/io.rb @@ -41,16 +41,10 @@ def self.builtins rule %r/[A-Z]\w*/, Name::Class - rule %r/[a-z_]\w*/ do |m| - name = m[0] - - if self.class.constants.include? name - token Keyword::Constant - elsif self.class.builtins.include? name - token Name::Builtin - else - token Name - end + keywords %r/[a-z_]\w*/ do |m| + rule :constants, Keyword::Constant + rule :builtins, Name::Builtin + default Name end rule %r((\d+[.]?\d*|\d*[.]\d+)(e[+-]?[0-9]+)?)i, Num::Float From 08f5fd837ed26a69d2be665b53652b9bfa7a02d3 Mon Sep 17 00:00:00 2001 From: jneen Date: Fri, 24 Apr 2026 18:06:58 -0400 Subject: [PATCH 069/154] use the keywords api for isabelle --- lib/rouge/lexers/isabelle.rb | 189 ++------------------------ lib/rouge/lexers/isabelle/keywords.rb | 116 ++++++++++++++++ 2 files changed, 126 insertions(+), 179 deletions(-) create mode 100644 lib/rouge/lexers/isabelle/keywords.rb diff --git a/lib/rouge/lexers/isabelle.rb b/lib/rouge/lexers/isabelle.rb index 6b97b22a17..68bd2f39ad 100644 --- a/lib/rouge/lexers/isabelle.rb +++ b/lib/rouge/lexers/isabelle.rb @@ -12,152 +12,7 @@ class Isabelle < RegexLexer filenames '*.thy' mimetypes 'text/x-isabelle' - def self.keyword_minor - @keyword_minor ||= Set.new %w( - and assumes attach avoids binder checking - class_instance class_relation code_module congs - constant constrains datatypes defines file fixes - for functions hints identifier if imports in - includes infix infixl infixr is keywords lazy - module_name monos morphisms no_discs_sels notes - obtains open output overloaded parametric permissive - pervasive rep_compat shows structure type_class - type_constructor unchecked unsafe where - ) - end - - def self.keyword_diag - @keyword_diag ||= Set.new %w( - ML_command ML_val class_deps code_deps code_thms - display_drafts find_consts find_theorems find_unused_assms - full_prf help locale_deps nitpick pr prf - print_abbrevs print_antiquotations print_attributes - print_binds print_bnfs print_bundles - print_case_translations print_cases print_claset - print_classes print_codeproc print_codesetup - print_coercions print_commands print_context - print_defn_rules print_dependencies print_facts - print_induct_rules print_inductives print_interps - print_locale print_locales print_methods print_options - print_orders print_quot_maps print_quotconsts - print_quotients print_quotientsQ3 print_quotmapsQ3 - print_rules print_simpset print_state print_statement - print_syntax print_theorems print_theory print_trans_rules - prop pwd quickcheck refute sledgehammer smt_status - solve_direct spark_status term thm thm_deps thy_deps - try try0 typ unused_thms value values welcome - print_ML_antiquotations print_term_bindings values_prolog - ) - end - - def self.keyword_thy - @keyword_thy ||= Set.new %w(theory begin end) - end - - def self.keyword_section - @keyword_section ||= Set.new %w(header chapter) - end - - def self.keyword_subsection - @keyword_subsection ||= Set.new %w(section subsection subsubsection sect subsect subsubsect) - end - - def self.keyword_theory_decl - @keyword_theory_decl ||= Set.new %w( - ML ML_file abbreviation adhoc_overloading arities - atom_decl attribute_setup axiomatization bundle - case_of_simps class classes classrel codatatype - code_abort code_class code_const code_datatype - code_identifier code_include code_instance code_modulename - code_monad code_printing code_reflect code_reserved - code_type coinductive coinductive_set consts context - datatype datatype_new datatype_new_compat declaration - declare default_sort defer_recdef definition defs - domain domain_isomorphism domaindef equivariance - export_code extract extract_type fixrec fun - fun_cases hide_class hide_const hide_fact hide_type - import_const_map import_file import_tptp import_type_map - inductive inductive_set instantiation judgment lemmas - lifting_forget lifting_update local_setup locale - method_setup nitpick_params no_adhoc_overloading - no_notation no_syntax no_translations no_type_notation - nominal_datatype nonterminal notation notepad oracle - overloading parse_ast_translation parse_translation - partial_function primcorec primrec primrec_new - print_ast_translation print_translation quickcheck_generator - quickcheck_params realizability realizers recdef record - refute_params setup setup_lifting simproc_setup - simps_of_case sledgehammer_params spark_end spark_open - spark_open_siv spark_open_vcg spark_proof_functions - spark_types statespace syntax syntax_declaration text - text_raw theorems translations type_notation - type_synonym typed_print_translation typedecl hoarestate - install_C_file install_C_types wpc_setup c_defs c_types - memsafe SML_export SML_file SML_import approximate - bnf_axiomatization cartouche datatype_compat - free_constructors functor nominal_function - nominal_termination permanent_interpretation - binds defining smt2_status term_cartouche - boogie_file text_cartouche - ) - end - - def self.keyword_theory_script - @keyword_theory_script ||= Set.new %w(inductive_cases inductive_simps) - end - - def self.keyword_theory_goal - @keyword_theory_goal ||= Set.new %w( - ax_specification bnf code_pred corollary cpodef - crunch crunch_ignore - enriched_type function instance interpretation lemma - lift_definition nominal_inductive nominal_inductive2 - nominal_primrec pcpodef primcorecursive - quotient_definition quotient_type recdef_tc rep_datatype - schematic_corollary schematic_lemma schematic_theorem - spark_vc specification subclass sublocale termination - theorem typedef wrap_free_constructors - ) - end - - def self.keyword_qed - @keyword_qed ||= Set.new %w(by done qed) - end - - def self.keyword_abandon_proof - @keyword_abandon_proof ||= Set.new %w(sorry oops) - end - - def self.keyword_proof_goal - @keyword_proof_goal ||= Set.new %w(have hence interpret) - end - - def self.keyword_proof_block - @keyword_proof_block ||= Set.new %w(next proof) - end - - def self.keyword_proof_chain - @keyword_proof_chain ||= Set.new %w(finally from then ultimately with) - end - - def self.keyword_proof_decl - @keyword_proof_decl ||= Set.new %w( - ML_prf also include including let moreover note - txt txt_raw unfolding using write - ) - end - - def self.keyword_proof_asm - @keyword_proof_asm ||= Set.new %w(assume case def fix presume) - end - - def self.keyword_proof_asm_goal - @keyword_proof_asm_goal ||= Set.new %w(guess obtain show thus) - end - - def self.keyword_proof_script - @keyword_proof_script ||= Set.new %w(apply apply_end apply_trace back defer prefer) - end + lazy { require_relative 'isabelle/keywords' } state :root do rule %r/\s+/, Text::Whitespace @@ -167,40 +22,16 @@ def self.keyword_proof_script rule %r/::|\[|\]|-|[:()_=,|+!?]/, Operator rule %r/[{}.]|\.\./, Operator::Word - def word(keywords) - return %r/\b(#{keywords.join('|')})\b/ - end - - rule %r/[a-zA-Z]\w*/ do |m| - sym = m[0] + keywords %r/[a-zA-Z]\w*/ do + rule KEYWORD_PSEUDO, Keyword::Pseudo + rule KEYWORD_DIAG, Keyword::Type + rule KEYWORDS, Keyword - if self.class.keyword_minor.include?(sym) || - self.class.keyword_proof_script.include?(sym) - token Keyword::Pseudo - elsif self.class.keyword_diag.include?(sym) - token Keyword::Type - elsif self.class.keyword_thy.include?(sym) || - self.class.keyword_theory_decl.include?(sym) || - self.class.keyword_qed.include?(sym) || - self.class.keyword_proof_goal.include?(sym) || - self.class.keyword_proof_block.include?(sym) || - self.class.keyword_proof_decl.include?(sym) || - self.class.keyword_proof_chain.include?(sym) || - self.class.keyword_proof_asm.include?(sym) || - self.class.keyword_proof_asm_goal.include?(sym) - token Keyword - elsif self.class.keyword_section.include?(sym) - token Generic::Heading - elsif self.class.keyword_subsection.include?(sym) - token Generic::Subheading - elsif self.class.keyword_theory_goal.include?(sym) || - self.class.keyword_theory_script.include?(sym) - token Keyword::Namespace - elsif self.class.keyword_abandon_proof.include?(sym) - token Generic::Error - else - token Name - end + rule KEYWORD_SECTION, Generic::Heading + rule KEYWORD_SUBSECTION, Generic::Subheading + rule KEYWORD_THEORY, Keyword::Namespace + rule KEYWORD_ABANDON_PROOF, Generic::Error + default Name end rule %r/\\<\w*>/, Str::Symbol diff --git a/lib/rouge/lexers/isabelle/keywords.rb b/lib/rouge/lexers/isabelle/keywords.rb new file mode 100644 index 0000000000..cb37886e87 --- /dev/null +++ b/lib/rouge/lexers/isabelle/keywords.rb @@ -0,0 +1,116 @@ +module Rouge + module Lexers + class Isabelle + KEYWORD_PSEUDO = Set.new %w( + and assumes attach avoids binder checking + class_instance class_relation code_module congs + constant constrains datatypes defines file fixes + for functions hints identifier if imports in + includes infix infixl infixr is keywords lazy + module_name monos morphisms no_discs_sels notes + obtains open output overloaded parametric permissive + pervasive rep_compat shows structure type_class + type_constructor unchecked unsafe where + + apply apply_end apply_trace back defer prefer + ) + + KEYWORD_DIAG = Set.new %w( + ML_command ML_val class_deps code_deps code_thms + display_drafts find_consts find_theorems find_unused_assms + full_prf help locale_deps nitpick pr prf + print_abbrevs print_antiquotations print_attributes + print_binds print_bnfs print_bundles + print_case_translations print_cases print_claset + print_classes print_codeproc print_codesetup + print_coercions print_commands print_context + print_defn_rules print_dependencies print_facts + print_induct_rules print_inductives print_interps + print_locale print_locales print_methods print_options + print_orders print_quot_maps print_quotconsts + print_quotients print_quotientsQ3 print_quotmapsQ3 + print_rules print_simpset print_state print_statement + print_syntax print_theorems print_theory print_trans_rules + prop pwd quickcheck refute sledgehammer smt_status + solve_direct spark_status term thm thm_deps thy_deps + try try0 typ unused_thms value values welcome + print_ML_antiquotations print_term_bindings values_prolog + ) + + KEYWORD_SECTION = Set.new %w(header chapter) + + KEYWORD_SUBSECTION = Set.new %w(section subsection subsubsection sect subsect subsubsect) + + KEYWORD_THEORY = Set.new %w( + ax_specification bnf code_pred corollary cpodef + crunch crunch_ignore + enriched_type function instance interpretation lemma + lift_definition nominal_inductive nominal_inductive2 + nominal_primrec pcpodef primcorecursive + quotient_definition quotient_type recdef_tc rep_datatype + schematic_corollary schematic_lemma schematic_theorem + spark_vc specification subclass sublocale termination + theorem typedef wrap_free_constructors + + inductive_cases inductive_simps + ) + + KEYWORD_ABANDON_PROOF = Set.new %w(sorry oops) + + KEYWORDS = Set.new %w( + theory begin end + + ML ML_file abbreviation adhoc_overloading arities + atom_decl attribute_setup axiomatization bundle + case_of_simps class classes classrel codatatype + code_abort code_class code_const code_datatype + code_identifier code_include code_instance code_modulename + code_monad code_printing code_reflect code_reserved + code_type coinductive coinductive_set consts context + datatype datatype_new datatype_new_compat declaration + declare default_sort defer_recdef definition defs + domain domain_isomorphism domaindef equivariance + export_code extract extract_type fixrec fun + fun_cases hide_class hide_const hide_fact hide_type + import_const_map import_file import_tptp import_type_map + inductive inductive_set instantiation judgment lemmas + lifting_forget lifting_update local_setup locale + method_setup nitpick_params no_adhoc_overloading + no_notation no_syntax no_translations no_type_notation + nominal_datatype nonterminal notation notepad oracle + overloading parse_ast_translation parse_translation + partial_function primcorec primrec primrec_new + print_ast_translation print_translation quickcheck_generator + quickcheck_params realizability realizers recdef record + refute_params setup setup_lifting simproc_setup + simps_of_case sledgehammer_params spark_end spark_open + spark_open_siv spark_open_vcg spark_proof_functions + spark_types statespace syntax syntax_declaration text + text_raw theorems translations type_notation + type_synonym typed_print_translation typedecl hoarestate + install_C_file install_C_types wpc_setup c_defs c_types + memsafe SML_export SML_file SML_import approximate + bnf_axiomatization cartouche datatype_compat + free_constructors functor nominal_function + nominal_termination permanent_interpretation + binds defining smt2_status term_cartouche + boogie_file text_cartouche + + by done qed + + have hence interpret + + next proof + + ML_prf also include including let moreover note + txt txt_raw unfolding using write + + finally from then ultimately with + + assume case def fix presume + + guess obtain show thus + ) + end + end +end From 9886befb500192c203c1f83060f00771c8e8e387 Mon Sep 17 00:00:00 2001 From: jneen Date: Fri, 24 Apr 2026 18:08:14 -0400 Subject: [PATCH 070/154] use the keywords api for isbl --- lib/rouge/lexers/isbl.rb | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/lib/rouge/lexers/isbl.rb b/lib/rouge/lexers/isbl.rb index 7544e29382..a5650fb360 100644 --- a/lib/rouge/lexers/isbl.rb +++ b/lib/rouge/lexers/isbl.rb @@ -45,13 +45,14 @@ def self.keywords rule %r/[\[\]();]/, Punctuation rule %r([&*+=<>/-]), Operator rule %r/[\p{Alpha}_!]\p{Word}*(?=[(])/i, Name::Function - rule %r/[\p{Alpha}_!]\p{Word}*/i do |m| - if self.class.keywords.include?(m[0].downcase) - token Keyword - else - token Name::Variable - end + + keywords %r/[\p{Alpha}_!]\p{Word}*/i do + transform(&:downcase) + + rule :keywords, Keyword + default Name::Variable end + rule %r/\b(\d+(\.\d+)?)\b/, Literal::Number rule %r(["].*?["])m, Literal::String::Double rule %r(['].*?['])m, Literal::String::Single From 25ec8530ea7b088f9881b05933d6e8e0146261ff Mon Sep 17 00:00:00 2001 From: jneen Date: Fri, 24 Apr 2026 18:14:27 -0400 Subject: [PATCH 071/154] use the keywords api for Janet --- lib/rouge/lexers/janet.rb | 118 ++--------------------------- lib/rouge/lexers/janet/builtins.rb | 91 ++++++++++++++++++++++ 2 files changed, 98 insertions(+), 111 deletions(-) create mode 100644 lib/rouge/lexers/janet/builtins.rb diff --git a/lib/rouge/lexers/janet.rb b/lib/rouge/lexers/janet.rb index 18b2465b14..bb36de9bed 100644 --- a/lib/rouge/lexers/janet.rb +++ b/lib/rouge/lexers/janet.rb @@ -14,105 +14,7 @@ class Janet < RegexLexer mimetypes 'text/x-janet', 'application/x-janet' - def self.specials - @specials ||= Set.new %w( - break def do fn if quote quasiquote splice set unquote var while - ) - end - - def self.bundled - @bundled ||= Set.new %w( - % %= * *= + ++ += - -- -= -> ->> -?> -?>> / /= < <= = > >= - abstract? accumulate accumulate2 all all-bindings - all-dynamics and apply array array/concat array/ensure - array/fill array/insert array/new array/new-filled - array/peek array/pop array/push array/remove array/slice - array? as-> as?-> asm assert bad-compile bad-parse band - blshift bnot boolean? bor brshift brushift buffer buffer/bit - buffer/bit-clear buffer/bit-set buffer/bit-toggle - buffer/blit buffer/clear buffer/fill buffer/format - buffer/new buffer/new-filled buffer/popn buffer/push-byte - buffer/push-string buffer/push-word buffer/slice buffer? - bxor bytes? case cfunction? chr cli-main comment comp - compare compare= compare< compare<= compare> compare>= - compile complement comptime cond coro count debug - debug/arg-stack debug/break debug/fbreak debug/lineage - debug/stack debug/stacktrace debug/step debug/unbreak - debug/unfbreak debugger-env dec deep-not= deep= default - default-peg-grammar def- defer defmacro defmacro- defn defn- - defglobal describe dictionary? disasm distinct doc doc* - doc-format dofile drop drop-until drop-while dyn each eachk - eachp eachy edefer eflush empty? env-lookup eprin eprinf - eprint eprintf error errorf eval eval-string even? every? - extreme false? fiber/can-resume? fiber/current fiber/getenv - fiber/maxstack fiber/new fiber/root fiber/setenv - fiber/setmaxstack fiber/status fiber? file/close file/flush - file/open file/popen file/read file/seek file/temp - file/write filter find find-index first flatten flatten-into - flush for forv freeze frequencies function? gccollect - gcinterval gcsetinterval generate gensym get get-in getline - hash idempotent? identity import import* if-let if-not - if-with in inc indexed? int/s64 int/u64 int? interleave - interpose invert janet/build janet/config-bits janet/version - juxt juxt* keep keys keyword keyword? kvs label last length - let load-image load-image-dict loop macex macex1 make-env - make-image make-image-dict map mapcat marshal math/-inf - math/abs math/acos math/acosh math/asin math/asinh math/atan - math/atan2 math/atanh math/cbrt math/ceil math/cos math/cosh - math/e math/erf math/erfc math/exp math/exp2 math/expm1 - math/floor math/gamma math/hypot math/inf math/log - math/log10 math/log1p math/log2 math/next math/pi math/pow - math/random math/rng math/rng-buffer math/rng-int - math/rng-uniform math/round math/seedrandom math/sin - math/sinh math/sqrt math/tan math/tanh math/trunc match max - mean merge merge-into min mod module/add-paths module/cache - module/expand-path module/find module/loaders module/loading - module/paths nan? nat? native neg? net/chunk net/close - net/connect net/read net/server net/write next nil? not not= - number? odd? one? or os/arch os/cd os/chmod os/clock - os/cryptorand os/cwd os/date os/dir os/environ os/execute - os/exit os/getenv os/link os/lstat os/mkdir os/mktime - os/perm-int os/perm-string os/readlink os/realpath os/rename - os/rm os/rmdir os/setenv os/shell os/sleep os/stat - os/symlink os/time os/touch os/umask os/which pairs parse - parser/byte parser/clone parser/consume parser/eof - parser/error parser/flush parser/has-more parser/insert - parser/new parser/produce parser/state parser/status - parser/where partial partition peg/compile peg/match pos? - postwalk pp prewalk prin prinf print printf product prompt - propagate protect put put-in quit range reduce reduce2 - repeat repl require resume return reverse reversed root-env - run-context scan-number seq setdyn shortfn signal slice - slurp some sort sort-by sorted sorted-by spit stderr stdin - stdout string string/ascii-lower string/ascii-upper - string/bytes string/check-set string/find string/find-all - string/format string/from-bytes string/has-prefix? - string/has-suffix? string/join string/repeat string/replace - string/replace-all string/reverse string/slice string/split - string/trim string/triml string/trimr string? struct struct? - sum symbol symbol? table table/clone table/getproto - table/new table/rawget table/setproto table/to-struct table? - take take-until take-while tarray/buffer tarray/copy-bytes - tarray/length tarray/new tarray/properties tarray/slice - tarray/swap-bytes thread/close thread/current thread/new - thread/receive thread/send trace tracev true? truthy? try - tuple tuple/brackets tuple/setmap tuple/slice - tuple/sourcemap tuple/type tuple? type unless unmarshal - untrace update update-in use values var- varfn varglobal - walk walk-ind walk-dict when when-let when-with with - with-dyns with-syms with-vars yield zero? zipcoll - ) - end - - def name_token(name) - if self.class.specials.include? name - Keyword - elsif self.class.bundled.include? name - Keyword::Reserved - else - Name::Function - end - end + lazy { require_relative 'janet/builtins' } punctuation = %r/[_!$%^&*+=~<>.?\/-]/o symbol = %r/([[:alpha:]]|#{punctuation})([[:word:]]|#{punctuation}|:)*/o @@ -170,18 +72,12 @@ def name_token(name) state :function do rule %r/[\)]/, Punctuation, :pop! - rule symbol do |m| - case m[0] - when "quote" - token Keyword - goto :quote - when "quasiquote" - token Keyword - goto :quasiquote - else - token name_token(m[0]) - goto :root - end + keywords symbol do + rule(Set['quote']) { token Keyword; goto :quote } + rule(Set['quasiquote']) { token Keyword; goto :quasiquote } + rule(SPECIALS) { token Keyword; goto :root } + rule(BUNDLED) { token Keyword::Reserved; goto :root } + default { token Name::Function; goto :root } end mixin :root diff --git a/lib/rouge/lexers/janet/builtins.rb b/lib/rouge/lexers/janet/builtins.rb new file mode 100644 index 0000000000..4a73a490c0 --- /dev/null +++ b/lib/rouge/lexers/janet/builtins.rb @@ -0,0 +1,91 @@ +module Rouge + module Lexers + class Janet + SPECIALS = Set.new %w( + break def do fn if quote quasiquote splice set unquote var while + ) + + BUNDLED = Set.new %w( + % %= * *= + ++ += - -- -= -> ->> -?> -?>> / /= < <= = > >= + abstract? accumulate accumulate2 all all-bindings + all-dynamics and apply array array/concat array/ensure + array/fill array/insert array/new array/new-filled + array/peek array/pop array/push array/remove array/slice + array? as-> as?-> asm assert bad-compile bad-parse band + blshift bnot boolean? bor brshift brushift buffer buffer/bit + buffer/bit-clear buffer/bit-set buffer/bit-toggle + buffer/blit buffer/clear buffer/fill buffer/format + buffer/new buffer/new-filled buffer/popn buffer/push-byte + buffer/push-string buffer/push-word buffer/slice buffer? + bxor bytes? case cfunction? chr cli-main comment comp + compare compare= compare< compare<= compare> compare>= + compile complement comptime cond coro count debug + debug/arg-stack debug/break debug/fbreak debug/lineage + debug/stack debug/stacktrace debug/step debug/unbreak + debug/unfbreak debugger-env dec deep-not= deep= default + default-peg-grammar def- defer defmacro defmacro- defn defn- + defglobal describe dictionary? disasm distinct doc doc* + doc-format dofile drop drop-until drop-while dyn each eachk + eachp eachy edefer eflush empty? env-lookup eprin eprinf + eprint eprintf error errorf eval eval-string even? every? + extreme false? fiber/can-resume? fiber/current fiber/getenv + fiber/maxstack fiber/new fiber/root fiber/setenv + fiber/setmaxstack fiber/status fiber? file/close file/flush + file/open file/popen file/read file/seek file/temp + file/write filter find find-index first flatten flatten-into + flush for forv freeze frequencies function? gccollect + gcinterval gcsetinterval generate gensym get get-in getline + hash idempotent? identity import import* if-let if-not + if-with in inc indexed? int/s64 int/u64 int? interleave + interpose invert janet/build janet/config-bits janet/version + juxt juxt* keep keys keyword keyword? kvs label last length + let load-image load-image-dict loop macex macex1 make-env + make-image make-image-dict map mapcat marshal math/-inf + math/abs math/acos math/acosh math/asin math/asinh math/atan + math/atan2 math/atanh math/cbrt math/ceil math/cos math/cosh + math/e math/erf math/erfc math/exp math/exp2 math/expm1 + math/floor math/gamma math/hypot math/inf math/log + math/log10 math/log1p math/log2 math/next math/pi math/pow + math/random math/rng math/rng-buffer math/rng-int + math/rng-uniform math/round math/seedrandom math/sin + math/sinh math/sqrt math/tan math/tanh math/trunc match max + mean merge merge-into min mod module/add-paths module/cache + module/expand-path module/find module/loaders module/loading + module/paths nan? nat? native neg? net/chunk net/close + net/connect net/read net/server net/write next nil? not not= + number? odd? one? or os/arch os/cd os/chmod os/clock + os/cryptorand os/cwd os/date os/dir os/environ os/execute + os/exit os/getenv os/link os/lstat os/mkdir os/mktime + os/perm-int os/perm-string os/readlink os/realpath os/rename + os/rm os/rmdir os/setenv os/shell os/sleep os/stat + os/symlink os/time os/touch os/umask os/which pairs parse + parser/byte parser/clone parser/consume parser/eof + parser/error parser/flush parser/has-more parser/insert + parser/new parser/produce parser/state parser/status + parser/where partial partition peg/compile peg/match pos? + postwalk pp prewalk prin prinf print printf product prompt + propagate protect put put-in quit range reduce reduce2 + repeat repl require resume return reverse reversed root-env + run-context scan-number seq setdyn shortfn signal slice + slurp some sort sort-by sorted sorted-by spit stderr stdin + stdout string string/ascii-lower string/ascii-upper + string/bytes string/check-set string/find string/find-all + string/format string/from-bytes string/has-prefix? + string/has-suffix? string/join string/repeat string/replace + string/replace-all string/reverse string/slice string/split + string/trim string/triml string/trimr string? struct struct? + sum symbol symbol? table table/clone table/getproto + table/new table/rawget table/setproto table/to-struct table? + take take-until take-while tarray/buffer tarray/copy-bytes + tarray/length tarray/new tarray/properties tarray/slice + tarray/swap-bytes thread/close thread/current thread/new + thread/receive thread/send trace tracev true? truthy? try + tuple tuple/brackets tuple/setmap tuple/slice + tuple/sourcemap tuple/type tuple? type unless unmarshal + untrace update update-in use values var- varfn varglobal + walk walk-ind walk-dict when when-let when-with with + with-dyns with-syms with-vars yield zero? zipcoll + ) + end + end +end From 1554b60410e451188c5bfcf92a6b8d80b2fcb913 Mon Sep 17 00:00:00 2001 From: jneen Date: Mon, 11 May 2026 11:10:46 -0400 Subject: [PATCH 072/154] raise if the state is closed and keywords is used --- lib/rouge/regex_lexer.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/rouge/regex_lexer.rb b/lib/rouge/regex_lexer.rb index 8adc66de32..d11c56198a 100644 --- a/lib/rouge/regex_lexer.rb +++ b/lib/rouge/regex_lexer.rb @@ -307,6 +307,8 @@ def mixin(state) # Define keyword sets with an overarching regular expression. def keywords(covering_regex, &block) + raise ClosedState.new(self) if @closed + keyword_rule = KeywordRule.new(covering_regex, &block) rule(covering_regex, &keyword_rule.to_proc) From 0a2ef9a97b2cce0b23db5a5bab1e40b9ec8660a0 Mon Sep 17 00:00:00 2001 From: jneen Date: Mon, 11 May 2026 11:11:51 -0400 Subject: [PATCH 073/154] simplify the java lexer --- lib/rouge/lexers/java.rb | 45 ++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/lib/rouge/lexers/java.rb b/lib/rouge/lexers/java.rb index 8bb780cf8a..79b1282e60 100644 --- a/lib/rouge/lexers/java.rb +++ b/lib/rouge/lexers/java.rb @@ -11,18 +11,18 @@ class Java < RegexLexer filenames '*.java' mimetypes 'text/x-java' - keywords = %w( + KEYWORDS = Set.new %w( assert break case catch continue default do else finally for if goto instanceof new return switch this throw try while yield when ) - declarations = %w( + DECLARATIONS = Set.new %w( abstract const extends final implements native permits private protected public sealed static strictfp super synchronized throws transient volatile ) - types = %w(boolean byte char double float int long short var void) + TYPES = Set.new %w(boolean byte char double float int long short var void String) id = /[[:alpha:]_][[:word:]]*/ const_name = /[[:upper:]][[:upper:][:digit:]_]*\b/ @@ -32,30 +32,29 @@ class Java < RegexLexer rule %r/[^\S\n]+/, Text rule %r(//.*?$), Comment::Single rule %r(/\*.*?\*/)m, Comment::Multiline - # keywords: go before method names to avoid lexing "throw new XYZ" - # as a method signature - rule %r/(?:#{keywords.join('|')})\b/, Keyword - - rule %r( - (\s*(?:[a-zA-Z_][a-zA-Z0-9_.\[\]<>]*\s+)+?) # return arguments - ([a-zA-Z_][a-zA-Z0-9_]*) # method name - (\s*)(\() # signature start - )mx do |m| - # TODO: do this better, this shouldn't need a delegation - delegate Java, m[1] - token Name::Function, m[2] - token Text, m[3] - token Operator, m[4] + + keywords %r/\w+/ do + rule KEYWORDS, Keyword end + rule %r/#{id}(?=\s*[(])/, Name::Function + rule %r/#{id}:/, Name::Label + + # special case since this has a - in it, which normally would + # break the identifier rule %r/non-sealed\b/, Keyword::Declaration + + keywords id do + rule Set['class', 'enum', 'interface', 'record'], Keyword::Declaration, :class + rule Set['import', 'package'], Keyword::Namespace, :import + rule DECLARATIONS, Keyword::Declaration + rule TYPES, Keyword::Type + rule Set['true', 'false', 'null'], Keyword::Constant + default Name + end + rule %r/@interface\b/, Keyword::Declaration, :class rule %r/@#{id}/, Name::Decorator - rule %r/(?:#{declarations.join('|')})\b/, Keyword::Declaration - rule %r/(?:#{types.join('|')})\b/, Keyword::Type - rule %r/(?:true|false|null)\b/, Keyword::Constant - rule %r/(?:class|enum|interface|record)\b/, Keyword::Declaration, :class - rule %r/(?:import(?:\s+(?:static|module))?|package)\b/, Keyword::Namespace, :import rule %r/"""\s*\n.*?(? Date: Mon, 11 May 2026 11:18:56 -0400 Subject: [PATCH 074/154] use the keywords api for javscript --- lib/rouge/lexers/javascript.rb | 25 ++++++++----------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/lib/rouge/lexers/javascript.rb b/lib/rouge/lexers/javascript.rb index 45a11844a2..14c7f2802d 100644 --- a/lib/rouge/lexers/javascript.rb +++ b/lib/rouge/lexers/javascript.rb @@ -125,7 +125,7 @@ def self.constants end def self.builtins - @builtins ||= %w( + @builtins ||= Set.new %w( Array Boolean Date Error Function Math netscape Number Object Packages RegExp String sun decodeURI decodeURIComponent encodeURI encodeURIComponent @@ -201,22 +201,13 @@ def self.id_regex rule %r/[{}]/, Punctuation, :statement - rule %r/#?#{id}/ do |m| - if self.class.keywords.include? m[0] - token Keyword - push :expr_start - elsif self.class.declarations.include? m[0] - token Keyword::Declaration - push :expr_start - elsif self.class.reserved.include? m[0] - token Keyword::Reserved - elsif self.class.constants.include? m[0] - token Keyword::Constant - elsif self.class.builtins.include? m[0] - token Name::Builtin - else - token Name::Other - end + keywords %r/#?#{id}/ do + rule :keywords, Keyword, :expr_start + rule :declarations, Keyword::Declaration, :expr_start + rule :reserved, Keyword::Reserved + rule :constants, Keyword::Constant + rule :builtins, Name::Builtin + default Name::Other end rule %r/[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?/, Num::Float From f60851b63c153bda4f4bbc44a788fe43435f296f Mon Sep 17 00:00:00 2001 From: jneen Date: Mon, 11 May 2026 11:21:33 -0400 Subject: [PATCH 075/154] use the keywords api for jinja --- lib/rouge/lexers/jinja.rb | 42 +++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/lib/rouge/lexers/jinja.rb b/lib/rouge/lexers/jinja.rb index 15c28b715b..edbf793588 100644 --- a/lib/rouge/lexers/jinja.rb +++ b/lib/rouge/lexers/jinja.rb @@ -14,25 +14,29 @@ class Jinja < TemplateLexer 'text/html+django', 'text/html+jinja' def self.keywords - @keywords ||= %w(as context do else extends from ignore missing - import include reversed recursive scoped - autoescape endautoescape block endblock call endcall - filter endfilter for endfor if endif macro endmacro - set endset trans endtrans with endwith without) + @keywords ||= Set.new %w( + as context do else extends from ignore missing + import include reversed recursive scoped + autoescape endautoescape block endblock call endcall + filter endfilter for endfor if endif macro endmacro + set endset trans endtrans with endwith without + ) end def self.tests - @tests ||= %w(callable defined divisibleby equalto escaped even iterable - lower mapping none number odd sameas sequence string - undefined upper) + @tests ||= Set.new %w( + callable defined divisibleby equalto escaped even iterable + lower mapping none number odd sameas sequence string + undefined upper + ) end def self.pseudo_keywords - @pseudo_keywords ||= %w(true false none True False None) + @pseudo_keywords ||= Set.new %w(true false none True False None) end def self.word_operators - @word_operators ||= %w(is in and or not) + @word_operators ||= Set.new %w(is in and or not) end state :root do @@ -125,18 +129,12 @@ def self.word_operators end state :statement do - rule %r/(\w+\.?)/ do |m| - if self.class.keywords.include?(m[0]) - groups Keyword - elsif self.class.pseudo_keywords.include?(m[0]) - groups Keyword::Pseudo - elsif self.class.word_operators.include?(m[0]) - groups Operator::Word - elsif self.class.tests.include?(m[0]) - groups Name::Builtin - else - groups Name::Variable - end + keywords %r/(\w+\.?)/ do + rule :keywords, Keyword + rule :pseudo_keywords, Keyword::Pseudo + rule :word_operators, Operator::Word + rule :tests, Name::Builtin + default Name::Variable end mixin :filter From d0f1c3480d1d8d07c71ca769f837d358d31786c3 Mon Sep 17 00:00:00 2001 From: jneen Date: Mon, 11 May 2026 11:23:25 -0400 Subject: [PATCH 076/154] use the keywords api for jsonnet --- lib/rouge/lexers/jsonnet.rb | 82 +++++++------------------------------ 1 file changed, 14 insertions(+), 68 deletions(-) diff --git a/lib/rouge/lexers/jsonnet.rb b/lib/rouge/lexers/jsonnet.rb index 6544366313..9d6883ea6b 100644 --- a/lib/rouge/lexers/jsonnet.rb +++ b/lib/rouge/lexers/jsonnet.rb @@ -31,62 +31,14 @@ def self.constants def self.builtins @builtins ||= Set.new %w( - acos - asin - atan - ceil - char - codepoint - cos - exp - exponent - filter - floor - force - length - log - makeArray - mantissa - objectFields - objectHas - pow - sin - sqrt - tan - thisFile - type - abs - assertEqual - escapeStringBash - escapeStringDollars - escapeStringJson - escapeStringPython - filterMap - flattenArrays - foldl - foldr - format - join - lines - manifestIni - manifestPython - manifestPythonVars - map - max - min - mod - range - set - setDiff - setInter - setMember - setUnion - sort - split - stringChars - substr - toString - uniq + acos asin atan ceil char codepoint cos exp exponent filter + floor force length log makeArray mantissa objectFields + objectHas pow sin sqrt tan thisFile type abs assertEqual + escapeStringBash escapeStringDollars escapeStringJson + escapeStringPython filterMap flattenArrays foldl foldr + format join lines manifestIni manifestPython manifestPythonVars + map max min mod range set setDiff setInter setMember + setUnion sort split stringChars substr toString uniq ) end @@ -109,18 +61,12 @@ def self.builtins rule %r/\$/, Keyword - rule identifier do |m| - if self.class.keywords.include? m[0] - token Keyword - elsif self.class.declarations.include? m[0] - token Keyword::Declaration - elsif self.class.constants.include? m[0] - token Keyword::Constant - elsif self.class.builtins.include? m[0] - token Name::Builtin - else - token Name::Other - end + keywords identifier do + rule :keywords, Keyword + rule :declarations, Keyword::Declaration + rule :constants, Keyword::Constant + rule :builtins, Name::Builtin + default Name::Other end end From 8a845e90ab4f237e9826dc5d3e4aabd8249b3059 Mon Sep 17 00:00:00 2001 From: jneen Date: Thu, 14 May 2026 13:11:39 -0400 Subject: [PATCH 077/154] RegexLexer: thread the lexer class through rules for better error messages --- lib/rouge/regex_lexer.rb | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/lib/rouge/regex_lexer.rb b/lib/rouge/regex_lexer.rb index d11c56198a..ac2cc55d2c 100644 --- a/lib/rouge/regex_lexer.rb +++ b/lib/rouge/regex_lexer.rb @@ -23,8 +23,13 @@ def to_s end class InvalidKeywordSet < StandardError + def initialize(context) + @context = context + super() + end + def to_s - "Invalid keyword set: Keys must be an instance of Set." + "#{@context}: Invalid keyword set: Keys must be an instance of Set." end end @@ -62,7 +67,8 @@ class Fallthrough < Exception # rubocop:disable Lint/InheritException class Rule attr_reader :callback attr_reader :re - def initialize(re, callback) + def initialize(lexer_class, re, callback) + @lexer_class = lexer_class @re = re @callback = callback end @@ -145,7 +151,8 @@ def inspect class StateDSL class KeywordRule - def initialize(covering_regex, &defn) + def initialize(lexer_class, covering_regex, &defn) + @lexer_class = lexer_class @covering_regex = covering_regex @transform = nil @default = nil @@ -164,7 +171,7 @@ def group(g) def rule(keyword_set, *args, &action) unless keyword_set.is_a?(Set) || keyword_set.is_a?(Symbol) - raise InvalidKeywordSet + raise InvalidKeywordSet.new(@lexer_class) end raise InvalidRule if action && args.any? @@ -220,7 +227,8 @@ def to_proc end attr_reader :rules, :name - def initialize(name, &defn) + def initialize(lexer_class, name, &defn) + @lexer_class = lexer_class @name = name @defn = defn @rules = [] @@ -238,7 +246,7 @@ def to_state(lexer_class) def prepended(&defn) parent_defn = @defn - StateDSL.new(@name) do + StateDSL.new(@lexer_class, @name) do instance_eval(&defn) instance_eval(&parent_defn) end @@ -246,7 +254,7 @@ def prepended(&defn) def appended(&defn) parent_defn = @defn - StateDSL.new(@name) do + StateDSL.new(@lexer_class, @name) do instance_eval(&parent_defn) instance_eval(&defn) end @@ -279,7 +287,7 @@ def rule(re, *args, &callback) callback ||= Rule.make_action(re, *args) - rules << Rule.new(re, callback) + rules << Rule.new(@lexer_class, re, callback) close! if matches_empty && !context_sensitive?(re) end @@ -309,7 +317,7 @@ def mixin(state) def keywords(covering_regex, &block) raise ClosedState.new(self) if @closed - keyword_rule = KeywordRule.new(covering_regex, &block) + keyword_rule = KeywordRule.new(@lexer_class, covering_regex, &block) rule(covering_regex, &keyword_rule.to_proc) end @@ -357,7 +365,7 @@ def self.start(&b) # The block will be evaluated in the context of a {StateDSL}. def self.state(name, &b) name = name.to_sym - state_definitions[name] = StateDSL.new(name, &b) + state_definitions[name] = StateDSL.new(self, name, &b) end def self.prepend(name, &b) @@ -405,7 +413,7 @@ def self.get_keyword_set(name) superclass.get_keyword_set(name) end - raise InvalidKeywordSet unless set.is_a?(Set) + raise InvalidKeywordSet.new(self) unless set.is_a?(Set) set end @@ -605,7 +613,7 @@ def push(state_name=nil, &b) push_state = if state_name get_state(state_name) elsif block_given? - StateDSL.new(b.inspect, &b).to_state(self.class) + StateDSL.new(self, b.inspect, &b).to_state(self.class) else # use the top of the stack by default self.state From eede65306d1be70b58b53576237d6ce4a727c861 Mon Sep 17 00:00:00 2001 From: jneen Date: Thu, 14 May 2026 13:12:56 -0400 Subject: [PATCH 078/154] make jinja subclasses use sets --- lib/rouge/lexers/twig.rb | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/lib/rouge/lexers/twig.rb b/lib/rouge/lexers/twig.rb index 0fa7038a4f..1c546833a7 100644 --- a/lib/rouge/lexers/twig.rb +++ b/lib/rouge/lexers/twig.rb @@ -16,24 +16,27 @@ class Twig < Jinja mimetypes 'application/x-twig', 'text/html+twig' def self.keywords - @keywords ||= %w(as do extends flush from import include use else starts - ends with without autoescape endautoescape block - endblock embed endembed filter endfilter for endfor - if endif macro endmacro sandbox endsandbox set endset - spaceless endspaceless) + @keywords ||= Set.new %w( + as do extends flush from import include use else starts + ends with without autoescape endautoescape block + endblock embed endembed filter endfilter for endfor + if endif macro endmacro sandbox endsandbox set endset + spaceless endspaceless + ) end def self.tests - @tests ||= %w(constant defined divisibleby empty even iterable null odd - sameas) + @tests ||= Set.new %w( + constant defined divisibleby empty even iterable null odd sameas + ) end def self.pseudo_keywords - @pseudo_keywords ||= %w(true false none) + @pseudo_keywords ||= Set.new %w(true false none) end def self.word_operators - @word_operators ||= %w(b-and b-or b-xor is in and or not) + @word_operators ||= Set.new %w(b-and b-or b-xor is in and or not) end end end From 2a175f59485b331c31a5548b57fbc93099514696 Mon Sep 17 00:00:00 2001 From: jneen Date: Thu, 14 May 2026 13:13:52 -0400 Subject: [PATCH 079/154] use the keywords api for julia --- lib/rouge/lexers/julia.rb | 161 ++--------------------------- lib/rouge/lexers/julia/builtins.rb | 103 ++++++++++++++++++ 2 files changed, 110 insertions(+), 154 deletions(-) create mode 100644 lib/rouge/lexers/julia/builtins.rb diff --git a/lib/rouge/lexers/julia.rb b/lib/rouge/lexers/julia.rb index 9d5366c7f7..58f1f74039 100644 --- a/lib/rouge/lexers/julia.rb +++ b/lib/rouge/lexers/julia.rb @@ -17,155 +17,8 @@ def self.detect?(text) return true if text.shebang? 'julia' end - BUILTINS = /\b(?: - true | false | missing | nothing - | Inf | Inf16 | Inf32 | Inf64 - | NaN | NaN16 | NaN32 | NaN64 - | stdout | stderr | stdin | devnull - | pi | π | ℯ | im - | ARGS | C_NULL | ENV | ENDIAN_BOM - | VERSION | undef | (LOAD|DEPOT)_PATH - )\b/x + lazy { require_relative 'julia/builtins' } - KEYWORDS = /\b(?: - function | return | module | import | export - | if | else | elseif | end | for - | in | isa | while | try | catch - | const | local | global | using | struct - | mutable struct | abstract type | finally - | begin | do | quote | macro | for outer - | where - )\b/x - - # NOTE: The list of types was generated automatically using the following script: - # using Pkg, InteractiveUtils - # - # allnames = [names(Core); names(Base, imported=true)] - # - # for stdlib in readdir(Pkg.Types.stdlib_dir()) - # mod = Symbol(basename(stdlib)) - # @eval begin - # using $mod - # append!(allnames, names($mod)) - # end - # end - # - # sort!(unique!(allnames)) - # - # i = 1 - # for sym in allnames - # global i # needed at the top level, e.g. in the REPL - # isdefined(Main, sym) || continue - # getfield(which(Main, sym), sym) isa Type || continue - # sym === :(=>) && continue # Actually an alias for Pair - # print("| ", sym) - # i % 3 == 0 ? println() : print(" ") # print 3 to a line - # i += 1 - # end - TYPES = /\b(?: - ARPACKException | AbstractArray | AbstractChannel - | AbstractChar | AbstractDict | AbstractDisplay - | AbstractFloat | AbstractIrrational | AbstractLogger - | AbstractMatrix | AbstractREPL | AbstractRNG - | AbstractRange | AbstractSerializer | AbstractSet - | AbstractSparseArray | AbstractSparseMatrix | AbstractSparseVector - | AbstractString | AbstractUnitRange | AbstractVecOrMat - | AbstractVector | AbstractWorkerPool | Adjoint - | Any | ArgumentError | Array - | AssertionError | Base64DecodePipe | Base64EncodePipe - | BasicREPL | Bidiagonal | BigFloat - | BigInt | BitArray | BitMatrix - | BitSet | BitVector | Bool - | BoundsError | BunchKaufman | CachingPool - | CapturedException | CartesianIndex | CartesianIndices - | Cchar | Cdouble | Cfloat - | Channel | Char | Cholesky - | CholeskyPivoted | Cint | Cintmax_t - | Clong | Clonglong | ClusterManager - | Cmd | Colon | Complex - | ComplexF16 | ComplexF32 | ComplexF64 - | CompositeException | Condition | ConsoleLogger - | Cptrdiff_t | Cshort | Csize_t - | Cssize_t | Cstring | Cuchar - | Cuint | Cuintmax_t | Culong - | Culonglong | Cushort | Cvoid - | Cwchar_t | Cwstring | DataType - | Date | DateFormat | DatePeriod - | DateTime | Day | DenseArray - | DenseMatrix | DenseVecOrMat | DenseVector - | Diagonal | Dict | DimensionMismatch - | Dims | DivideError | DomainError - | EOFError | Eigen | Enum - | ErrorException | Exception | ExponentialBackOff - | Expr | FDWatcher | Factorization - | FileMonitor | Float16 | Float32 - | Float64 | FolderMonitor | Function - | GeneralizedEigen | GeneralizedSVD | GeneralizedSchur - | GenericArray | GenericDict | GenericSet - | GenericString | GitConfig | GitRepo - | GlobalRef | HMAC_CTX | HTML - | Hermitian | Hessenberg | Hour - | IO | IOBuffer | IOContext - | IOStream | IPAddr | IPv4 - | IPv6 | IdDict | IndexCartesian - | IndexLinear | IndexStyle | InexactError - | InitError | Int | Int128 - | Int16 | Int32 | Int64 - | Int8 | Integer | InterruptException - | InvalidStateException | Irrational | KeyError - | LAPACKException | LDLt | LQ - | LU | LinRange | LineEditREPL - | LineNumberNode | LinearIndices | LoadError - | LogLevel | LowerTriangular | MIME - | Matrix | MersenneTwister | Method - | MethodError | Microsecond | Millisecond - | Minute | Missing | MissingException - | Module | Month | NTuple - | NamedTuple | Nanosecond | Nothing - | NullLogger | Number | OrdinalRange - | OutOfMemoryError | OverflowError | PackageMode - | PackageSpec | Pair | PartialQuickSort - | Period | PermutedDimsArray | Pipe - | PollingFileWatcher | PosDefException | ProcessExitedException - | Ptr | QR | QRPivoted - | QuoteNode | RandomDevice | RankDeficientException - | Rational | RawFD | ReadOnlyMemoryError - | Real | ReentrantLock | Ref - | Regex | RegexMatch | RemoteChannel - | RemoteException | RoundingMode | SHA1_CTX - | SHA224_CTX | SHA256_CTX | SHA2_224_CTX - | SHA2_256_CTX | SHA2_384_CTX | SHA2_512_CTX - | SHA384_CTX | SHA3_224_CTX | SHA3_256_CTX - | SHA3_384_CTX | SHA3_512_CTX | SHA512_CTX - | SVD | Schur | Second - | SegmentationFault | Serializer | Set - | SharedArray | SharedMatrix | SharedVector - | Signed | SimpleLogger | SingularException - | Some | SparseMatrixCSC | SparseVector - | StackOverflowError | StepRange | StepRangeLen - | StreamREPL | StridedArray | StridedMatrix - | StridedVecOrMat | StridedVector | String - | StringIndexError | SubArray | SubString - | SubstitutionString | SymTridiagonal | Symbol - | Symmetric | SystemError | TCPSocket - | Task | TestSetException | Text - | TextDisplay | Time | TimePeriod - | TimeType | TimeZone | Timer - | Transpose | Tridiagonal | Tuple - | Type | TypeError | TypeVar - | UDPSocket | UInt | UInt128 - | UInt16 | UInt32 | UInt64 - | UInt8 | UTC | UUID - | UndefInitializer | UndefKeywordError | UndefRefError - | UndefVarError | UniformScaling | Union - | UnionAll | UnitLowerTriangular | UnitRange - | UnitUpperTriangular | Unsigned | UpgradeLevel - | UpperTriangular | Val | Vararg - | VecElement | VecOrMat | Vector - | VersionNumber | WeakKeyDict | WeakRef - | Week | WorkerConfig | WorkerPool - | Year - )\b/x OPERATORS = / \+ | = | - | \* | \/ | \\ | & | \| | \$ | ~ @@ -205,16 +58,16 @@ def self.detect?(text) groups Keyword, Name::Class push :typename end - rule TYPES, Keyword::Type # keywords - rule %r/(local|global|const)\b/, Keyword::Declaration - rule KEYWORDS, Keyword - # TODO: end is a builtin when inside of an indexing expression - rule BUILTINS, Name::Builtin - # TODO: symbols + keywords %r/\w+/ do + rule Set['local', 'global', 'const'], Keyword::Declaration + rule KEYWORDS, Keyword + rule BUILTINS, Name::Builtin + rule TYPES, Keyword::Type + end # backticks rule %r/`.*?`/, Literal::String::Backtick diff --git a/lib/rouge/lexers/julia/builtins.rb b/lib/rouge/lexers/julia/builtins.rb new file mode 100644 index 0000000000..7227072127 --- /dev/null +++ b/lib/rouge/lexers/julia/builtins.rb @@ -0,0 +1,103 @@ +module Rouge + module Lexers + class Julia + BUILTINS = Set.new %w( + true false missing nothing + Inf Inf16 Inf32 Inf64 + NaN NaN16 NaN32 NaN64 + stdout stderr stdin devnull + pi π ℯ im + ARGS C_NULL ENV ENDIAN_BOM + VERSION undef LOAD_PATH DEPOT_PATH + ) + + KEYWORDS = Set.new %w( + function return module import export + if else elseif end for + in isa while try catch + const local global using struct + mutable struct abstract type finally + begin do quote macro for outer + where + ) + + # NOTE: The list of types was generated automatically using the following script: + # using Pkg, InteractiveUtils + # + # allnames = [names(Core); names(Base, imported=true)] + # + # for stdlib in readdir(Pkg.Types.stdlib_dir()) + # mod = Symbol(basename(stdlib)) + # @eval begin + # using $mod + # append!(allnames, names($mod)) + # end + # end + # + # sort!(unique!(allnames)) + # + # i = 1 + # for sym in allnames + # global i # needed at the top level, e.g. in the REPL + # isdefined(Main, sym) || continue + # getfield(which(Main, sym), sym) isa Type || continue + # sym === :(=>) && continue # Actually an alias for Pair + # print(" ", sym) + # i % 3 == 0 ? println() : print(" ") # print 3 to a line + # i += 1 + # end + TYPES = Set.new %w( + ARPACKException AbstractArray AbstractChannel AbstractChar + AbstractDict AbstractDisplay AbstractFloat AbstractIrrational + AbstractLogger AbstractMatrix AbstractREPL AbstractRNG AbstractRange + AbstractSerializer AbstractSet AbstractSparseArray AbstractSparseMatrix + AbstractSparseVector AbstractString AbstractUnitRange AbstractVecOrMat + AbstractVector AbstractWorkerPool Adjoint Any ArgumentError Array + AssertionError Base64DecodePipe Base64EncodePipe BasicREPL Bidiagonal + BigFloat BigInt BitArray BitMatrix BitSet BitVector Bool BoundsError + BunchKaufman CachingPool CapturedException CartesianIndex + CartesianIndices Cchar Cdouble Cfloat Channel Char Cholesky + CholeskyPivoted Cint Cintmax_t Clong Clonglong ClusterManager Cmd + Colon Complex ComplexF16 ComplexF32 ComplexF64 CompositeException + Condition ConsoleLogger Cptrdiff_t Cshort Csize_t Cssize_t Cstring + Cuchar Cuint Cuintmax_t Culong Culonglong Cushort Cvoid Cwchar_t + Cwstring DataType Date DateFormat DatePeriod DateTime Day DenseArray + DenseMatrix DenseVecOrMat DenseVector Diagonal Dict DimensionMismatch + Dims DivideError DomainError EOFError Eigen Enum ErrorException + Exception ExponentialBackOff Expr FDWatcher Factorization FileMonitor + Float16 Float32 Float64 FolderMonitor Function GeneralizedEigen + GeneralizedSVD GeneralizedSchur GenericArray GenericDict GenericSet + GenericString GitConfig GitRepo GlobalRef HMAC_CTX HTML Hermitian + Hessenberg Hour IO IOBuffer IOContext IOStream IPAddr IPv4 IPv6 + IdDict IndexCartesian IndexLinear IndexStyle InexactError InitError + Int Int128 Int16 Int32 Int64 Int8 Integer InterruptException + InvalidStateException Irrational KeyError LAPACKException LDLt LQ + LU LinRange LineEditREPL LineNumberNode LinearIndices LoadError + LogLevel LowerTriangular MIME Matrix MersenneTwister Method + MethodError Microsecond Millisecond Minute Missing MissingException + Module Month NTuple NamedTuple Nanosecond Nothing NullLogger Number + OrdinalRange OutOfMemoryError OverflowError PackageMode PackageSpec + Pair PartialQuickSort Period PermutedDimsArray Pipe PollingFileWatcher + PosDefException ProcessExitedException Ptr QR QRPivoted QuoteNode + RandomDevice RankDeficientException Rational RawFD ReadOnlyMemoryError + Real ReentrantLock Ref Regex RegexMatch RemoteChannel RemoteException + RoundingMode SHA1_CTX SHA224_CTX SHA256_CTX SHA2_224_CTX SHA2_256_CTX + SHA2_384_CTX SHA2_512_CTX SHA384_CTX SHA3_224_CTX SHA3_256_CTX + SHA3_384_CTX SHA3_512_CTX SHA512_CTX SVD Schur Second SegmentationFault + Serializer Set SharedArray SharedMatrix SharedVector Signed + SimpleLogger SingularException Some SparseMatrixCSC SparseVector + StackOverflowError StepRange StepRangeLen StreamREPL StridedArray + StridedMatrix StridedVecOrMat StridedVector String StringIndexError + SubArray SubString SubstitutionString SymTridiagonal Symbol Symmetric + SystemError TCPSocket Task TestSetException Text TextDisplay Time + TimePeriod TimeType TimeZone Timer Transpose Tridiagonal Tuple + Type TypeError TypeVar UDPSocket UInt UInt128 UInt16 UInt32 UInt64 + UInt8 UTC UUID UndefInitializer UndefKeywordError UndefRefError + UndefVarError UniformScaling Union UnionAll UnitLowerTriangular + UnitRange UnitUpperTriangular Unsigned UpgradeLevel UpperTriangular + Val Vararg VecElement VecOrMat Vector VersionNumber WeakKeyDict + WeakRef Week WorkerConfig WorkerPool Year + ) + end + end +end From b21232a9a8aa803b30c3ef2d4a00c7f424bfac78 Mon Sep 17 00:00:00 2001 From: jneen Date: Thu, 14 May 2026 13:19:25 -0400 Subject: [PATCH 080/154] use the keywords api for Kick Assembler --- lib/rouge/lexers/kick_assembler.rb | 54 +++++++++++++----------------- 1 file changed, 24 insertions(+), 30 deletions(-) diff --git a/lib/rouge/lexers/kick_assembler.rb b/lib/rouge/lexers/kick_assembler.rb index 8a5cc6379d..06e8687877 100644 --- a/lib/rouge/lexers/kick_assembler.rb +++ b/lib/rouge/lexers/kick_assembler.rb @@ -29,12 +29,12 @@ def self.mnemonics def self.type_directives @type_directives ||= Set.new %w( - .align .assert .asserterror .break .by .byte .cpu - .disk .dword .dw .encoding .enum .error .errorif .eval - .file .fill .fillword .for .function .import .importonce - .label .lohifill .macro .memblock .modify .namespace - .pc .print .printnow .pseudocommand .pseudopc .return - .segment .segmentdef .struct .text .var .word .while .zp + align assert asserterror break by byte cpu + disk dword dw encoding enum error errorif eval + file fill fillword for function import importonce + label lohifill macro memblock modify namespace + pc print printnow pseudocommand pseudopc return + segment segmentdef struct text var word while zp ) end @@ -58,13 +58,11 @@ def self.preprocessor_directives rule %r/[+\-*\/=<>!]/, Operator - rule %r/#\w+/i do |m| - directive = m[0][1..-1].downcase - if self.class.preprocessor_directives.include?(directive) - token Keyword - else - token Punctuation - end + keywords %r/#(\w+)/ do + group 1 + + rule :preprocessor_directives, Keyword + default Punctuation end rule %r/#/, Punctuation @@ -72,25 +70,21 @@ def self.preprocessor_directives rule %r/[a-z$._?][\w$.?#@~]*:/i, Name::Label - rule %r/\.\w+/i do |m| - if m[0] == '.const' - token Keyword::Constant - elsif m[0] =~ /^\.(for|while|macro|function|pseudopc|pseudocommand)$/io - token Keyword::Reserved - elsif self.class.type_directives.include?(m[0].downcase) - token Keyword::Declaration - else - token Keyword - end + keywords %r/[.](\w+)/i do + transform(&:downcase) + + rule Set['const'], Keyword::Constant + rule Set['for', 'while', 'macro', 'function', 'pseudopc', 'pseudocommand'], + Keyword::Reserved + + rule :type_directives, Keyword::Declaration + default Keyword end - rule %r/\b\w+\b/i do |m| - word = m[0].downcase - if self.class.mnemonics.include?(word) - token Keyword - else - token Name - end + keywords %r/\w+/ do + transform(&:downcase) + rule :mnemonics, Keyword + default Name end rule %r/[ \t\r\n]+/, Text From 0413fedc72cfa12475232d6aba968138d43c0849 Mon Sep 17 00:00:00 2001 From: jneen Date: Fri, 15 May 2026 11:53:00 -0400 Subject: [PATCH 081/154] rubocop nonsense --- .rubocop_todo.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index 00c1a6b24b..300fe895fa 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -41,9 +41,6 @@ Naming/VariableName: Rouge/NoBuildingAlternationPatternInRegexp: Exclude: - 'lib/rouge/lexers/armasm.rb' - - 'lib/rouge/lexers/isabelle.rb' - - 'lib/rouge/lexers/java.rb' - - 'lib/rouge/lexers/jsp.rb' - 'lib/rouge/lexers/kotlin.rb' - 'lib/rouge/lexers/lean.rb' - 'lib/rouge/lexers/magik.rb' From c8ef3f27e8e44961c13d724530b0e793cc8ea4a7 Mon Sep 17 00:00:00 2001 From: jneen Date: Fri, 15 May 2026 11:53:12 -0400 Subject: [PATCH 082/154] javscript: remove duplicate set element --- lib/rouge/lexers/javascript.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/rouge/lexers/javascript.rb b/lib/rouge/lexers/javascript.rb index 14c7f2802d..af7493ce24 100644 --- a/lib/rouge/lexers/javascript.rb +++ b/lib/rouge/lexers/javascript.rb @@ -129,7 +129,7 @@ def self.builtins Array Boolean Date Error Function Math netscape Number Object Packages RegExp String sun decodeURI decodeURIComponent encodeURI encodeURIComponent - Error eval isFinite isNaN parseFloat parseInt + eval isFinite isNaN parseFloat parseInt document window navigator self global Promise Set Map WeakSet WeakMap Symbol Proxy Reflect Int8Array Uint8Array Uint8ClampedArray From 9291ad36624c54a349fc96e11f16b2a084243071 Mon Sep 17 00:00:00 2001 From: jneen Date: Fri, 15 May 2026 11:53:54 -0400 Subject: [PATCH 083/154] stop using interpolated regexes in jsp --- lib/rouge/lexers/jsp.rb | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/lib/rouge/lexers/jsp.rb b/lib/rouge/lexers/jsp.rb index 8c09c413f5..da603c6e40 100644 --- a/lib/rouge/lexers/jsp.rb +++ b/lib/rouge/lexers/jsp.rb @@ -1,6 +1,15 @@ # -*- coding: utf-8 -*- # # frozen_string_literal: true +# [jneen] TODO: this lexer needs an overhaul, and probably needs +# to be split into two delegating lexers, since interpolation is +# allowed even in the values of specialized tags. +# +# Right now something like: +# +# +# +# doesn't highlight the AppConfig part, because is a specialized tag. module Rouge module Lexers class JSP < TemplateLexer @@ -17,15 +26,19 @@ def initialize(*) directives = %w(page include taglib) actions = %w(scriptlet declaration expression) - state :root do - + state :whitespace do + rule %r/\s+/, Text rule %r/<%--/, Comment, :jsp_comment + end + + state :root do + mixin :whitespace - rule %r/<%@\s*(#{directives.join('|')})\s*/, Name::Tag, :jsp_directive + rule %r/<%@\s*\w+/, Name::Tag, :jsp_directive - rule %r//, Name::Tag, :jsp_expression + rule %r//, Name::Tag, :jsp_expression # start of tag, e.g. rule %r/<[a-zA-Z]*:[a-zA-Z]*\s*/, Name::Tag, :jsp_tag @@ -42,36 +55,38 @@ def initialize(*) state :jsp_comment do rule %r/(--%>)/, Comment, :pop! + rule %r/[^-]+/m, Comment rule %r/./m, Comment end state :jsp_directive do rule %r/(%>)/, Name::Tag, :pop! mixin :attributes - rule(/(.+?)(?=%>)/m) { delegate parent } + rule(/.+?(?=%>)/m) { delegate parent } end state :jsp_directive2 do rule %r!(/>)!, Name::Tag, :pop! mixin :attributes - rule(/(.+?)(?=\/>)/m) { delegate parent } + rule(/.+?(?=\/>)/m) { delegate parent } end state :jsp_expression do - rule %r/<\/jsp:(#{actions.join('|')})>/, Name::Tag, :pop! + rule %r(), Name::Tag, :pop! mixin :attributes - rule(/[^<\/]+/) { delegate @java } + rule(%r([^<\/]+)) { delegate @java } end state :jsp_expression2 do rule %r/%>/, Name::Tag, :pop! - rule(/[^%>]+/) { delegate @java } + rule(/.+?(?=%>|\z)/m) { delegate @java } end state :jsp_tag do rule %r/\/?>/, Name::Tag, :pop! mixin :attributes rule(/(.+?)(?=\/?>)/m) { delegate parent } + # rule(%r{.+?(?=/?>)}m) { delegate parent } end state :attributes do From 64bfe57ecf8bb0cc085e44884616efe014fb159c Mon Sep 17 00:00:00 2001 From: jneen Date: Fri, 15 May 2026 11:54:09 -0400 Subject: [PATCH 084/154] fix kick assembler dot keywords --- lib/rouge/lexers/kick_assembler.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/rouge/lexers/kick_assembler.rb b/lib/rouge/lexers/kick_assembler.rb index 06e8687877..391cecc29d 100644 --- a/lib/rouge/lexers/kick_assembler.rb +++ b/lib/rouge/lexers/kick_assembler.rb @@ -71,6 +71,7 @@ def self.preprocessor_directives rule %r/[a-z$._?][\w$.?#@~]*:/i, Name::Label keywords %r/[.](\w+)/i do + group 1 transform(&:downcase) rule Set['const'], Keyword::Constant From 877cbf7606a77ce7950a963de896bf6ea7dd5935 Mon Sep 17 00:00:00 2001 From: jneen Date: Fri, 15 May 2026 11:55:07 -0400 Subject: [PATCH 085/154] remove armasm from rubocop todo --- .rubocop_todo.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index 300fe895fa..a30d5b2cc4 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -40,7 +40,6 @@ Naming/VariableName: # Offense count: 113 Rouge/NoBuildingAlternationPatternInRegexp: Exclude: - - 'lib/rouge/lexers/armasm.rb' - 'lib/rouge/lexers/kotlin.rb' - 'lib/rouge/lexers/lean.rb' - 'lib/rouge/lexers/magik.rb' From b50f8ca97ac386d819654a04470bd591fce861ac Mon Sep 17 00:00:00 2001 From: jneen Date: Fri, 15 May 2026 11:58:48 -0400 Subject: [PATCH 086/154] use the keywords api for kotlin --- .rubocop_todo.yml | 2 -- lib/rouge/lexers/kotlin.rb | 23 ++++++++++++++++------- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index a30d5b2cc4..22f6003948 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -34,13 +34,11 @@ Naming/MethodName: Naming/VariableName: Exclude: - 'lib/rouge/lexers/dafny.rb' - - 'lib/rouge/lexers/kotlin.rb' - 'lib/rouge/lexers/xpath.rb' # Offense count: 113 Rouge/NoBuildingAlternationPatternInRegexp: Exclude: - - 'lib/rouge/lexers/kotlin.rb' - 'lib/rouge/lexers/lean.rb' - 'lib/rouge/lexers/magik.rb' - 'lib/rouge/lexers/make.rb' diff --git a/lib/rouge/lexers/kotlin.rb b/lib/rouge/lexers/kotlin.rb index e534fdb5f0..ed134840eb 100644 --- a/lib/rouge/lexers/kotlin.rb +++ b/lib/rouge/lexers/kotlin.rb @@ -13,7 +13,7 @@ class Kotlin < RegexLexer filenames '*.kt', '*.kts' mimetypes 'text/x-kotlin' - keywords = %w( + KEYWORDS = Set.new %w( abstract annotation as break by catch class companion const constructor continue crossinline do dynamic else enum external false final finally for fun get if import in infix @@ -35,31 +35,40 @@ class Kotlin < RegexLexer rule %r'\b(companion)(\s+)(object)\b' do groups Keyword, Text, Keyword end + rule %r'\b(class|data\s+class|interface|object)(\s+)' do groups Keyword::Declaration, Text push :class end + rule %r'\b(fun)(\s+)' do groups Keyword, Text push :function end + rule %r'\b(package|import)(\s+)' do groups Keyword, Text push :package end + rule %r'\b(val|var)(\s+)(\()' do groups Keyword::Declaration, Text, Punctuation push :destructure end + rule %r'\b(val|var)(\s+)' do groups Keyword::Declaration, Text push :property end + rule %r'(return|continue|break|this|super)(@#{name})?\b' do groups Keyword, Name::Decorator end - rule %r'\bfun\b', Keyword - rule %r'\b(?:#{keywords.join('|')})\b', Keyword + + keywords %r/\w+/ do + rule KEYWORDS, Keyword + end + rule %r'^\s*\[.*?\]', Name::Attribute rule %r'[^\S\n]+', Text rule %r'\\\n', Text # line continuation @@ -73,10 +82,10 @@ class Kotlin < RegexLexer rule %r'::|!!|\?[:.]', Operator rule %r"(\.\.)", Operator # Number literals - decDigits = %r"([0-9][0-9_]*[0-9])|[0-9]" - exponent = %r"[eE][+-]?(#{decDigits})" - double = %r"((#{decDigits})?\.#{decDigits}(#{exponent})?)|(#{decDigits}#{exponent})" - rule %r"(#{double}[fF]?)|(#{decDigits}[fF])", Num::Float + dec_digits = %r"([0-9][0-9_]*[0-9])|[0-9]" + exponent = %r"[eE][+-]?(#{dec_digits})" + double = %r"((#{dec_digits})?\.#{dec_digits}(#{exponent})?)|(#{dec_digits}#{exponent})" + rule %r"(#{double}[fF]?)|(#{dec_digits}[fF])", Num::Float rule %r"0[bB]([01][01_]*[01]|[01])[uU]?L?", Num::Bin rule %r"0[xX]([0-9a-fA-F][0-9a-fA-F_]*[0-9a-fA-F]|[0-9a-fA-F])[uU]?L?", Num::Hex rule %r"(([1-9][0-9_]*[0-9])|[0-9])[uU]?L?", Num::Integer From 9eb4009c1ebf5b5349043f946d053440c8679525 Mon Sep 17 00:00:00 2001 From: jneen Date: Fri, 15 May 2026 12:55:15 -0400 Subject: [PATCH 087/154] change support for annotation use-site targets to be more generic --- lib/rouge/lexers/kotlin.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/rouge/lexers/kotlin.rb b/lib/rouge/lexers/kotlin.rb index ed134840eb..06212687cd 100644 --- a/lib/rouge/lexers/kotlin.rb +++ b/lib/rouge/lexers/kotlin.rb @@ -96,9 +96,11 @@ class Kotlin < RegexLexer rule %r'"(\\\\|\\"|[^"\n])*["\n]'m, Str rule %r"'\\.'|'[^\\]'", Str::Char rule %r'(@#{class_name})', Name::Decorator - rule %r'(@)(#{use_site_targets.join('|')})(:)' do - groups Name::Decorator, Keyword, Punctuation + + rule %r'(@\w+)(:)' do + groups Name::Decorator, Punctuation end + rule %r'(#{class_name})(<)' do groups Name::Class, Punctuation push :generic_parameters From cd243ec96bc75893ba232cf27f962c506ee95631 Mon Sep 17 00:00:00 2001 From: jneen Date: Fri, 15 May 2026 13:03:48 -0400 Subject: [PATCH 088/154] fix rubocop for dafny --- .rubocop_todo.yml | 1 - lib/rouge/lexers/dafny.rb | 8 ++++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index 22f6003948..5f6bc2e00b 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -33,7 +33,6 @@ Naming/MethodName: # SupportedStyles: snake_case, camelCase Naming/VariableName: Exclude: - - 'lib/rouge/lexers/dafny.rb' - 'lib/rouge/lexers/xpath.rb' # Offense count: 113 diff --git a/lib/rouge/lexers/dafny.rb b/lib/rouge/lexers/dafny.rb index 4e7cbbc023..c6b72bc8de 100644 --- a/lib/rouge/lexers/dafny.rb +++ b/lib/rouge/lexers/dafny.rb @@ -47,8 +47,8 @@ class Dafny < RegexLexer idchar = /[0-9a-zA-Z_'?]/ id = /#{idstart}#{idchar}*/ - arrayType = /array(?:1[0-9]+|[2-9][0-9]*)\??(?!#{idchar})/ - bvType = /bv(?:0|[1-9][0-9]*)(?!#{idchar})/ + array_type = /array(?:1[0-9]+|[2-9][0-9]*)\??(?!#{idchar})/ + bv_type = /bv(?:0|[1-9][0-9]*)(?!#{idchar})/ digit = /\d/ digits = /#{digit}+(?:_#{digit}+)*/ @@ -92,8 +92,8 @@ class Dafny < RegexLexer rule %r/[0-9_]+_(?!#{idchar})/, Error rule %r/[0-9]#{idchar}+/, Error - rule %r/#{arrayType}/, Keyword::Type - rule %r/#{bvType}/, Keyword::Type + rule %r/#{array_type}/, Keyword::Type + rule %r/#{bv_type}/, Keyword::Type keywords id do rule TYPES, Keyword::Type From 28c5ab7c0df8d78e988cc09ce9c84618079a61b0 Mon Sep 17 00:00:00 2001 From: jneen Date: Fri, 15 May 2026 13:04:09 -0400 Subject: [PATCH 089/154] use the keywords api for lean --- .rubocop_todo.yml | 1 - lib/rouge/lexers/lean.rb | 51 +++++++++++++++++++++++----------------- 2 files changed, 30 insertions(+), 22 deletions(-) diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index 5f6bc2e00b..0289356d64 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -38,7 +38,6 @@ Naming/VariableName: # Offense count: 113 Rouge/NoBuildingAlternationPatternInRegexp: Exclude: - - 'lib/rouge/lexers/lean.rb' - 'lib/rouge/lexers/magik.rb' - 'lib/rouge/lexers/make.rb' - 'lib/rouge/lexers/mason.rb' diff --git a/lib/rouge/lexers/lean.rb b/lib/rouge/lexers/lean.rb index d19d7f8bbc..4ff85c5ffb 100644 --- a/lib/rouge/lexers/lean.rb +++ b/lib/rouge/lexers/lean.rb @@ -93,22 +93,32 @@ def self.keywords end def self.types - @types ||= %w( + @types ||= Set.new %w( Sort Prop Type ) end - def self.operators - @operators ||= %w( - != # & && \* \+ - / @ ! ` -\. -> - \. \.\. \.\.\. :: :> ; ;; < - <- = == > _ \| \|\| ~ => <= >= - /\ \/ ∀ Π λ ↔ ∧ ∨ ≠ ≤ ≥ ⊎ - ¬ ⁻¹ ⬝ ▸ → ∃ ℕ ℤ ≈ × ⌞ ⌟ ≡ ⟨ ⟩ - ) - end + OPERATORS = %r( + != + | -[.] + | -> + | [.]{1,3} + | :: + | :> + | ;;? + | <[-=]? + | =[=>]? + | >=? + | _ + | [|][|]? + | ~ + | /\\ + | \\/ + | ⁻¹ + | [-+*#&/@!∀Πλ↔∧∨≠≤≥⊎¬⬝▸→∃ℕℤ≈×⌞⌟≡⟨⟩]+ + )x state :root do # comments starting after some space @@ -117,18 +127,17 @@ def self.operators rule %r/"/, Str, :string rule %r/\d+/, Num::Integer + rule %r/@\[.*?\]/, Comment::Preproc + # special commands or keywords - rule(/#?\w+/) do |m| - match = m[0] - if self.class.keywords.include?(match) - token Keyword - elsif self.class.types.include?(match) - token Keyword::Type - else - token Name - end + keywords %r/#?\w+/ do + rule :keywords, Keyword + rule :types, Keyword::Type + default Name end + rule %r/`\S+?`/, Operator::Word + # special unicode keywords rule %r/[λ]/, Keyword @@ -136,10 +145,10 @@ def self.operators # operators rules # ---------------- - rule %r/\:=?/, Text + rule %r/\:=?/, Punctuation rule %r/\.[0-9]*/, Operator - rule %r(#{Lean.operators.join('|')}), Operator + rule OPERATORS, Operator # unmatched symbols rule %r/[\s\(\),\[\]αβ‹›]+/, Text From 9e099de814123b4fa36ea3b092328af455723cd1 Mon Sep 17 00:00:00 2001 From: jneen Date: Fri, 15 May 2026 13:04:18 -0400 Subject: [PATCH 090/154] jsp: remove unused vars --- lib/rouge/lexers/jsp.rb | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/rouge/lexers/jsp.rb b/lib/rouge/lexers/jsp.rb index da603c6e40..054c9df0fb 100644 --- a/lib/rouge/lexers/jsp.rb +++ b/lib/rouge/lexers/jsp.rb @@ -23,9 +23,6 @@ def initialize(*) @java = Java.new end - directives = %w(page include taglib) - actions = %w(scriptlet declaration expression) - state :whitespace do rule %r/\s+/, Text rule %r/<%--/, Comment, :jsp_comment From cf118a1b3e5e4aa1e4746378a050f82f8061edfd Mon Sep 17 00:00:00 2001 From: jneen Date: Fri, 15 May 2026 13:04:28 -0400 Subject: [PATCH 091/154] kotlin: remove unused vars --- lib/rouge/lexers/kotlin.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/rouge/lexers/kotlin.rb b/lib/rouge/lexers/kotlin.rb index 06212687cd..dc911e50d6 100644 --- a/lib/rouge/lexers/kotlin.rb +++ b/lib/rouge/lexers/kotlin.rb @@ -24,8 +24,6 @@ class Kotlin < RegexLexer while yield ) - use_site_targets = %w(file property field get set receiver param setparam delegate) - name_chars = %r'[-\p{Lu}\p{Ll}\p{Lt}\p{Lm}\p{Nl}\p{Nd}\p{Pc}\p{Cf}\p{Mn}\p{Mc}]*' class_name = %r'`?[\p{Lu}]#{name_chars}`?' From 14d93bff55e17fcba0fcd4ff7eb80d0ba849307c Mon Sep 17 00:00:00 2001 From: jneen Date: Fri, 15 May 2026 13:04:35 -0400 Subject: [PATCH 092/154] use the keywords api for lasso --- lib/rouge/lexers/lasso.rb | 34 ++++++++++++---------------------- 1 file changed, 12 insertions(+), 22 deletions(-) diff --git a/lib/rouge/lexers/lasso.rb b/lib/rouge/lexers/lasso.rb index 33fb5914e7..ba0ddfab36 100644 --- a/lib/rouge/lexers/lasso.rb +++ b/lib/rouge/lexers/lasso.rb @@ -156,28 +156,18 @@ def start_inline? groups Name, Name::Label, Operator end - rule %r((/?)([\w.]+)) do |m| - name = m[2].downcase - - if m[1] != '' - token Punctuation, m[1] - end - - if name == 'namespace_using' - token Keyword::Namespace, m[2] - elsif KEYWORDS[:exceptions].include? name - token Name::Exception, m[2] - elsif KEYWORDS[:types].include? name - token Keyword::Type, m[2] - elsif KEYWORDS[:traits].include? name - token Name::Decorator, m[2] - elsif KEYWORDS[:keywords].include? name - token Keyword, m[2] - elsif KEYWORDS[:builtins].include? name - token Name::Builtin, m[2] - else - token Name::Other, m[2] - end + rule %r(/(?=[\w.])), Punctuation + + keywords %r/\w+/ do + transform(&:downcase) + + rule Set['namespace_using'], Keyword::Namespace + rule KEYWORDS[:exceptions], Name::Exception + rule KEYWORDS[:types], Keyword::Type + rule KEYWORDS[:traits], Name::Decorator + rule KEYWORDS[:keywords], Keyword + rule KEYWORDS[:builtins], Name::Builtin + default Name::Other end rule %r/(=)(n?bw|n?ew|n?cn|lte?|gte?|n?eq|n?rx|ft)\b/i do From 1f0276b874762b56ca37449556a70e142360f63a Mon Sep 17 00:00:00 2001 From: jneen Date: Fri, 15 May 2026 13:23:25 -0400 Subject: [PATCH 093/154] remove idlang and janet from rubocop todo --- .rubocop_todo.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index 0289356d64..3d8f2e2366 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -58,8 +58,6 @@ Rouge/NoBuildingAlternationPatternInRegexp: # Offense count: 20 Rouge/NoHugeCollections: Exclude: - - 'lib/rouge/lexers/idlang.rb' - - 'lib/rouge/lexers/janet.rb' - 'lib/rouge/lexers/openedge.rb' - 'lib/rouge/lexers/plsql.rb' - 'lib/rouge/lexers/racket.rb' From 979e5b511bbcde5f7c9a5e536bbda13acd77644f Mon Sep 17 00:00:00 2001 From: jneen Date: Fri, 15 May 2026 13:49:05 -0400 Subject: [PATCH 094/154] use the keywords api for verilog --- .rubocop_todo.yml | 8 -------- lib/rouge/lexers/verilog.rb | 19 ++++++------------- 2 files changed, 6 insertions(+), 21 deletions(-) diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index 3d8f2e2366..f16c8844f1 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -12,14 +12,6 @@ Lint/EmptyBlock: Exclude: - 'lib/rouge/lexers/nix.rb' -# Offense count: 5 -# This cop supports unsafe autocorrection (--autocorrect-all). -# Configuration parameters: EnforcedStyleForLeadingUnderscores. -# SupportedStylesForLeadingUnderscores: disallowed, required, optional -Naming/MemoizedInstanceVariableName: - Exclude: - - 'lib/rouge/lexers/verilog.rb' - # Offense count: 16 # Configuration parameters: EnforcedStyle, AllowedPatterns, ForbiddenIdentifiers, ForbiddenPatterns. # SupportedStyles: snake_case, camelCase diff --git a/lib/rouge/lexers/verilog.rb b/lib/rouge/lexers/verilog.rb index 820c4350f1..acd8bb5a7c 100644 --- a/lib/rouge/lexers/verilog.rb +++ b/lib/rouge/lexers/verilog.rb @@ -50,7 +50,7 @@ def self.keywords_type end def self.keywords_system_task - @keyword_system_task ||= Set.new %w( + @keywords_system_task ||= Set.new %w( acos acosh asin asinh assertfailoff assertfailon assertkill assertnonvacuouson assertoff asserton assertpassoff assertpasson assertvacuousoff atan atan2 atanh bits bitstoreal bitstoshortreal @@ -130,18 +130,11 @@ def self.keywords_system_task rule %r/[()\[\],.$\#;]/, Punctuation rule %r/`(\w+)/, Comment::Preproc - rule id do |m| - name = m[0] - - if self.class.keywords.include? name - token Keyword - elsif self.class.keywords_type.include? name - token Keyword::Type - elsif self.class.keywords_system_task.include? name - token Name::Builtin - else - token Name - end + keywords id do + rule :keywords, Keyword + rule :keywords_type, Keyword::Type + rule :keywords_system_task, Name::Builtin + default Name end end From fd42094846f4c5180d75fc18788a42518d0e8413 Mon Sep 17 00:00:00 2001 From: jneen Date: Fri, 29 May 2026 11:26:37 -0400 Subject: [PATCH 095/154] use the keywords api for addmusick --- lib/rouge/lexers/addmusick.rb | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/lib/rouge/lexers/addmusick.rb b/lib/rouge/lexers/addmusick.rb index c943981ab0..203e56fc00 100644 --- a/lib/rouge/lexers/addmusick.rb +++ b/lib/rouge/lexers/addmusick.rb @@ -151,12 +151,10 @@ def step(state, stream) @last = nil end - rule %r/#(\w+)/ do |m| - if KEYWORDS.include?(m[1]) - token Keyword - else - token Name::Class - end + keywords %r/#(\w+)/ do + group 1 + rule KEYWORDS, Keyword + default Name::Class end rule %r/(")([^"=]+)(=)/ do |m| From 753603782c99cf3f030893c0266fd346160917f4 Mon Sep 17 00:00:00 2001 From: jneen Date: Fri, 29 May 2026 11:33:43 -0400 Subject: [PATCH 096/154] use the keywords api for javascript, but more --- lib/rouge/lexers/javascript.rb | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/lib/rouge/lexers/javascript.rb b/lib/rouge/lexers/javascript.rb index af7493ce24..c94f802bcb 100644 --- a/lib/rouge/lexers/javascript.rb +++ b/lib/rouge/lexers/javascript.rb @@ -190,13 +190,9 @@ def self.id_regex rule %r/function(?=(\(.*\)))/, Keyword::Declaration # For anonymous functions - rule %r/(#?#{id})[ \t]*(?=(\(.*\)))/m do |m| - if self.class.keywords.include?(m[1]) - # "if" in "if (...)" or "switch" in "switch (...)" are recognized as keywords. - token Keyword - else - token Name::Function - end + keywords %r/#?#{id}(?=[ \t]*[(])/m do + rule :keywords, Keyword + default Name::Function end rule %r/[{}]/, Punctuation, :statement From be6ed8b8b9dca0e739ad998f2687d08b9357436d Mon Sep 17 00:00:00 2001 From: jneen Date: Fri, 29 May 2026 11:44:49 -0400 Subject: [PATCH 097/154] use the keywords api for livescript --- lib/rouge/lexers/livescript.rb | 32 ++++++++++++++------------------ spec/visual/samples/livescript | 12 ++++++++++++ 2 files changed, 26 insertions(+), 18 deletions(-) diff --git a/lib/rouge/lexers/livescript.rb b/lib/rouge/lexers/livescript.rb index 0825e6386d..02890dfc0b 100644 --- a/lib/rouge/lexers/livescript.rb +++ b/lib/rouge/lexers/livescript.rb @@ -107,25 +107,21 @@ def self.loop_control_keywords rule %r/:/, Punctuation # keywords - rule %r/#{id}/ do |m| - if self.class.loop_control_keywords.include? m[0] - token Keyword - push :loop_control - next - elsif self.class.keywords.include? m[0] - token Keyword - elsif self.class.constants.include? m[0] - token Name::Constant - elsif self.class.builtins.include? m[0] - token Name::Builtin - elsif self.class.declarations.include? m[0] - token Keyword::Declaration - elsif /^[A-Z]/.match(m[0]) && /[^-][a-z]/.match(m[0]) - token Name::Class - else - token Name::Variable + keywords id do + rule :loop_control_keywords, Keyword, :loop_control + rule :keywords, Keyword, :id + rule :constants, Name::Constant, :id + rule :builtins, Name::Builtin, :id + rule :declarations, Keyword::Declaration, :id + default do |m| + if ('A'..'Z').include?(m[0][0]) + token Name::Class + else + token Name + end + + push :id end - push :id end # punctuation and brackets diff --git a/spec/visual/samples/livescript b/spec/visual/samples/livescript index b3996f4a2f..dcf89d533a 100644 --- a/spec/visual/samples/livescript +++ b/spec/visual/samples/livescript @@ -302,3 +302,15 @@ do-line = (code, index) -> try Function "'use strict'; var #id" catch @carp "invalid variable interpolation '#id'" + +class A + (num) -> + @x = num + property: 1 + method: (y) -> + @x + @property + y + +a = new A 3 +a.x #=> 3 +a.property #=> 1 +a.method 6 #=> 10 From 8490f5385f20bd7cddcc71077e8f8fdcfe0f9d39 Mon Sep 17 00:00:00 2001 From: jneen Date: Fri, 29 May 2026 11:45:29 -0400 Subject: [PATCH 098/154] rubocop fix --- lib/rouge/lexers/livescript.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/rouge/lexers/livescript.rb b/lib/rouge/lexers/livescript.rb index 02890dfc0b..25e781767e 100644 --- a/lib/rouge/lexers/livescript.rb +++ b/lib/rouge/lexers/livescript.rb @@ -114,7 +114,7 @@ def self.loop_control_keywords rule :builtins, Name::Builtin, :id rule :declarations, Keyword::Declaration, :id default do |m| - if ('A'..'Z').include?(m[0][0]) + if ('A'..'Z').cover?(m[0][0]) token Name::Class else token Name From b9335539c4dbc374b1005a229fd019705d3c7293 Mon Sep 17 00:00:00 2001 From: jneen Date: Fri, 29 May 2026 11:48:22 -0400 Subject: [PATCH 099/154] use the keywords api for lustre --- lib/rouge/lexers/lustre.rb | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/lib/rouge/lexers/lustre.rb b/lib/rouge/lexers/lustre.rb index 65cced8998..8b82976476 100644 --- a/lib/rouge/lexers/lustre.rb +++ b/lib/rouge/lexers/lustre.rb @@ -37,17 +37,12 @@ def self.primitives rule %r(\-\-.*), Comment::Single rule %r(/\*.*?\*/)m, Comment::Multiline rule %r(\(\*.*?\*\))m, Comment::Multiline - rule id do |m| - match = m[0] - if self.class.keywords.include? match - token Keyword - elsif self.class.word_operators.include? match - token Operator::Word - elsif self.class.primitives.include? match - token Keyword::Type - else - token Name - end + + keywords id do + rule :keywords, Keyword + rule :word_operators, Operator::Word + rule :primitives, Keyword::Type + default Name end rule %r/[(){}\[\];]+/, Punctuation From 765b136b0ccd8d3007599322e5c786fe6301f970 Mon Sep 17 00:00:00 2001 From: jneen Date: Fri, 29 May 2026 11:56:19 -0400 Subject: [PATCH 100/154] use the keywords api and remove overhighlighting on m68k --- lib/rouge/lexers/m68k.rb | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/lib/rouge/lexers/m68k.rb b/lib/rouge/lexers/m68k.rb index e7c52be210..a8d30442c1 100644 --- a/lib/rouge/lexers/m68k.rb +++ b/lib/rouge/lexers/m68k.rb @@ -112,22 +112,13 @@ def self.builtins rule %r/[(),.]/, Punctuation rule %r/\[[a-zA-Z0-9]*\]/, Punctuation - rule id do |m| - name = m[0] - - if self.class.keywords.include? name.downcase - token Keyword - elsif self.class.keywords_type.include? name.downcase - token Keyword::Type - elsif self.class.reserved.include? name.downcase - token Keyword::Reserved - elsif self.class.builtins.include? name.downcase - token Name::Builtin - elsif name =~ /[a-zA-Z0-9]+/ - token Name::Variable - else - token Name - end + keywords id do + transform(&:downcase) + rule :keywords, Keyword + rule :keywords_type, Keyword::Type + rule :reserved, Keyword::Reserved + rule :builtins, Name::Builtin + default Name end end From 6d25242039e5be43a5c4be4f442f9a44f03c829d Mon Sep 17 00:00:00 2001 From: jneen Date: Fri, 29 May 2026 12:38:28 -0400 Subject: [PATCH 101/154] use the keywords api for magik, simplify the lexer, and fix multichar number radix --- lib/rouge/lexers/magik.rb | 128 ++++++++++++++------------------------ 1 file changed, 45 insertions(+), 83 deletions(-) diff --git a/lib/rouge/lexers/magik.rb b/lib/rouge/lexers/magik.rb index 0a3f47aa1b..b799623e52 100644 --- a/lib/rouge/lexers/magik.rb +++ b/lib/rouge/lexers/magik.rb @@ -10,81 +10,41 @@ class Magik < RegexLexer filenames '*.magik' mimetypes 'text/x-magik', 'application/x-magik' - def self.keywords - @keywords ||= %w( - _package - _pragma - _block _endblock - _handling _default - _protect _protection _endprotect - _try _with _when _endtry - _catch _endcatch - _throw - _lock _endlock - _if _then _elif _else _endif - _for _over _while _loop _finally _endloop _loopbody _continue _leave - _return - _class - _local _constant _recursive _global _dynamic _import - _private _iter _abstract _method _endmethod - _proc _endproc - _gather _scatter _allresults _optional - _thisthread _self _clone _super - _primitive - _unset _true _false _maybe - _is _isnt _not _and _or _xor _cf _andif _orif - _div _mod - ) - end + KEYWORDS = Set.new %w( + _package + _pragma + _block _endblock + _handling _default + _protect _protection _endprotect + _try _with _when _endtry + _catch _endcatch + _throw + _lock _endlock + _if _then _elif _else _endif + _for _over _while _loop _finally _endloop _loopbody _continue _leave + _return + _class + _local _constant _recursive _global _dynamic _import + _private _iter _abstract _method _endmethod + _proc _endproc + _gather _scatter _allresults _optional + _thisthread _self _clone _super + _primitive + _unset _true _false _maybe + _is _isnt _not _and _or _xor _cf _andif _orif + _div _mod + ) - def self.string_double - @string_double ||= /"[^"\n]*?"/ - end - def self.string_single - @string_single ||= /'[^'\n]*?'/ - end - - def self.digits - @digits ||= /[0-9]+/ - end - def self.radix - @radix ||= /r[0-9a-z]/i - end - def self.exponent - @exponent ||= /(e|&)[+-]?#{Magik.digits}/i - end - def self.decimal - @decimal ||= /\.#{Magik.digits}/ - end - def self.number - @number = /#{Magik.digits}(#{Magik.radix}|#{Magik.exponent}|#{Magik.decimal})*/ - end - def self.character - @character ||= /%u[0-9a-z]{4}|%[^\s]+/i - end + digits = /[0-9]+/ + radix = /r[0-9a-z]+/i + exponent = /(e|&)[+-]?#{digits}/i + decimal = /\.#{digits}/ + number = /#{digits}(#{radix}|#{exponent}|#{decimal})*/ - def self.simple_identifier - @simple_identifier ||= /(?>(?:[a-z0-9_!?]|\\.)+)/i - end - def self.piped_identifier - @piped_identifier ||= /\|[^\|\n]*\|/ - end - def self.identifier - @identifier ||= /(?:#{Magik.simple_identifier}|#{Magik.piped_identifier})+/i - end - def self.package_identifier - @package_identifier ||= /#{Magik.identifier}:#{Magik.identifier}/ - end - def self.symbol - @symbol ||= /:#{Magik.identifier}/i - end - def self.global_ref - @global_ref ||= /@[\s]*#{Magik.identifier}:#{Magik.identifier}/ - end - def self.label - @label = /@[\s]*#{Magik.identifier}/ - end + simple_identifier = /(?>(?:[a-z0-9_!?]|\\.)+)/i + piped_identifier = /\|[^\|\n]*\|/ + identifier = /(?:#{simple_identifier}|#{piped_identifier})+/i state :root do rule %r/##(.*)?/, Comment::Doc @@ -95,17 +55,19 @@ def self.label push :method_name end - rule %r/(?:#{Magik.keywords.join('|')})\b/, Keyword + keywords %r/_\w+/ do + rule KEYWORDS, Keyword + end - rule Magik.string_double, Literal::String - rule Magik.string_single, Literal::String - rule Magik.symbol, Str::Symbol - rule Magik.global_ref, Name::Label - rule Magik.label, Name::Label - rule Magik.character, Literal::String::Char - rule Magik.number, Literal::Number - rule Magik.package_identifier, Name - rule Magik.identifier, Name + rule /"[^"\n]*?"/, Literal::String + rule /'[^'\n]*?'/, Literal::String + rule /:#{identifier}/, Str::Symbol + rule /@[\s]*#{identifier}:#{identifier}/, Name::Label + rule /@[\s]*#{identifier}/, Name::Label + rule /%u[0-9a-z]{4}|%[^\s]+/i, Literal::String::Char + rule number, Literal::Number + rule /#{identifier}:#{identifier}/, Name + rule identifier, Name rule %r/[\[\]{}()\.,;]/, Punctuation rule %r/\$/, Punctuation @@ -117,7 +79,7 @@ def self.label end state :method_name do - rule %r/(#{Magik.identifier})(\.)(#{Magik.identifier})/ do + rule %r/(#{identifier})(\.)(#{identifier})/ do groups Name::Class, Punctuation, Name::Function pop! end From bee5f7c0e7ace17448f7152c2645c071222d9e6d Mon Sep 17 00:00:00 2001 From: jneen Date: Fri, 29 May 2026 12:39:46 -0400 Subject: [PATCH 102/154] fix warnings on magik and delete rubocop exception --- .rubocop_todo.yml | 1 - lib/rouge/lexers/magik.rb | 14 +++++++------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index f16c8844f1..1e7625fdf2 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -30,7 +30,6 @@ Naming/VariableName: # Offense count: 113 Rouge/NoBuildingAlternationPatternInRegexp: Exclude: - - 'lib/rouge/lexers/magik.rb' - 'lib/rouge/lexers/make.rb' - 'lib/rouge/lexers/mason.rb' - 'lib/rouge/lexers/mosel.rb' diff --git a/lib/rouge/lexers/magik.rb b/lib/rouge/lexers/magik.rb index b799623e52..a6bfc550d9 100644 --- a/lib/rouge/lexers/magik.rb +++ b/lib/rouge/lexers/magik.rb @@ -59,14 +59,14 @@ class Magik < RegexLexer rule KEYWORDS, Keyword end - rule /"[^"\n]*?"/, Literal::String - rule /'[^'\n]*?'/, Literal::String - rule /:#{identifier}/, Str::Symbol - rule /@[\s]*#{identifier}:#{identifier}/, Name::Label - rule /@[\s]*#{identifier}/, Name::Label - rule /%u[0-9a-z]{4}|%[^\s]+/i, Literal::String::Char + rule %r/"[^"\n]*?"/, Literal::String + rule %r/'[^'\n]*?'/, Literal::String + rule %r/:#{identifier}/, Str::Symbol + rule %r/@[\s]*#{identifier}:#{identifier}/, Name::Label + rule %r/@[\s]*#{identifier}/, Name::Label + rule %r/%u[0-9a-z]{4}|%[^\s]+/i, Literal::String::Char rule number, Literal::Number - rule /#{identifier}:#{identifier}/, Name + rule %r/#{identifier}:#{identifier}/, Name rule identifier, Name rule %r/[\[\]{}()\.,;]/, Punctuation From 8c7f00c02d6d0cba0bede6abf9e90c113c3100bf Mon Sep 17 00:00:00 2001 From: jneen Date: Tue, 9 Jun 2026 17:34:46 -0400 Subject: [PATCH 103/154] use the keywords api for make (and refactor a bit) --- .rubocop_todo.yml | 1 - lib/rouge/lexers/make.rb | 53 ++++++++++++++++++++++++---------------- spec/visual/samples/make | 2 ++ 3 files changed, 34 insertions(+), 22 deletions(-) diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index 1e7625fdf2..d55b8be759 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -30,7 +30,6 @@ Naming/VariableName: # Offense count: 113 Rouge/NoBuildingAlternationPatternInRegexp: Exclude: - - 'lib/rouge/lexers/make.rb' - 'lib/rouge/lexers/mason.rb' - 'lib/rouge/lexers/mosel.rb' - 'lib/rouge/lexers/nix.rb' diff --git a/lib/rouge/lexers/make.rb b/lib/rouge/lexers/make.rb index f7a2421295..e3485a1ddc 100644 --- a/lib/rouge/lexers/make.rb +++ b/lib/rouge/lexers/make.rb @@ -11,14 +11,12 @@ class Make < RegexLexer filenames '*.make', '*.mak', '*.mk', 'Makefile', 'makefile', 'Makefile.*', 'GNUmakefile', '*,fe1' mimetypes 'text/x-makefile' - def self.functions - @functions ||= %w( - abspath addprefix addsuffix and basename call dir error eval file - filter filter-out findstring firstword flavor foreach if join lastword - notdir or origin patsubst realpath shell sort strip subst suffix value - warning wildcard word wordlist words - ) - end + FUNCTIONS = Set.new %w( + abspath addprefix addsuffix and basename call dir error eval file + filter filter-out findstring firstword flavor foreach if join lastword + notdir or origin patsubst realpath shell sort strip subst suffix value + warning wildcard word wordlist words + ) def initialize(opts={}) super @@ -27,10 +25,17 @@ def initialize(opts={}) start { @shell.reset! } - state :root do + state :whitespace do rule %r/\s+/, Text - rule %r/#.*?\n/, Comment + end + + state :dollar_paren_interp do + rule %r/[$]+[({]/, Punctuation, :dollar_paren + end + + state :root do + mixin :whitespace rule %r/([-s]?include)((?:[\t ]+[^\t\n #]+)+)/ do groups Keyword, Literal::String::Other @@ -61,19 +66,28 @@ def initialize(opts={}) rule %r/"(\\\\|\\.|[^"\\])*"/, Str::Double rule %r/'(\\\\|\\.|[^'\\])*'/, Str::Single + + rule %r/(override\b)*([\t ])*(define)([\t ]+)([^#\n]+)/ do + groups Keyword::Declaration, Text, Keyword, Text, Name::Variable + end + rule %r/([^\n:]+)(:+)([ \t]*)/ do groups Name::Label, Operator, Text push :block_header end - rule %r/(override\b)*([\t ])*(define)([\t ]+)([^#\n]+)/ do - groups Name::Builtin, Text, Keyword, Text, Name::Variable - end + mixin :dollar_paren_interp + end + + state :dollar_paren do + mixin :whitespace - rule %r/(\$[({])([\t ]*)(#{Make.functions.join('|')})([\t ]+)/m do - groups Name::Function, Text, Name::Builtin, Text - push :shell_expr + keywords %r/[\w-]+/ do + rule(FUNCTIONS) { token Name::Builtin; goto :shell_expr } + default { token Name::Function; goto :shell_expr } end + + rule(//) { goto :shell_expr } end state :export do @@ -115,13 +129,10 @@ def initialize(opts={}) state :shell do # macro interpolation - rule %r/[\$]{1,2}[({]/, Punctuation, :macro_expr + # rule %r/[\$]{1,2}[({]/, Punctuation, :macro_expr + mixin :dollar_paren_interp # function invocation - rule %r/(\$[({])([\t ]*)(#{Make.functions.join('|')})([\t ]+)/m do - groups Punctuation, Text, Name::Builtin, Text - push :shell_expr - end rule(/\\./m) { delegate @shell } stop = /[\$]{1,2}\(|[\$]{1,2}\{|\(|\)|\}|\\|$/ diff --git a/spec/visual/samples/make b/spec/visual/samples/make index deffd5b81a..7c0447cd2a 100644 --- a/spec/visual/samples/make +++ b/spec/visual/samples/make @@ -2,6 +2,8 @@ VERSION := 0.1.$(shell git log --oneline | wc -l | sed -e "s/ //g") +thing += other-thing + # Simple assignments LIST= Parser Grammar Objects Python Modules Mac STRING= '--with-pydebug' From 3eb061cd3c76673ccafee96635f544d0aeb634d7 Mon Sep 17 00:00:00 2001 From: jneen Date: Tue, 9 Jun 2026 17:35:35 -0400 Subject: [PATCH 104/154] use the keywords api for shell and highlight | as punctuation in root, not sure why we didn't before --- lib/rouge/lexers/shell.rb | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/lib/rouge/lexers/shell.rb b/lib/rouge/lexers/shell.rb index 7ee471aaaf..f89c5cf3b6 100644 --- a/lib/rouge/lexers/shell.rb +++ b/lib/rouge/lexers/shell.rb @@ -24,12 +24,12 @@ def self.detect?(text) return true if text.start_with?('#compdef', '#autoload') end - KEYWORDS = %w( + KEYWORDS = Set.new %w( if fi else while do done for then return function select continue until esac elif in - ).join('|') + ) - BUILTINS = %w( + BUILTINS = Set.new %w( alias bg bind break builtin caller cd command compgen complete declare dirs disown enable eval exec exit export false fc fg getopts hash help history jobs let @@ -43,19 +43,21 @@ def self.detect?(text) unexpand ls dir vdir dircolors cp dd install mv rm shred link ln mkdir mkfifo mknod readlink rmdir unlink chown chgrp chmod touch df du stat sync truncate echo printf yes expr tee basename dirname - pathchk mktemp realpath pwd stty printenv tty id logname whoami + pathchk mktemp realpath stty printenv tty id logname whoami groups users who date arch nproc uname hostname hostid uptime chcon runcon chroot env nice nohup stdbuf timeout kill sleep factor numfmt seq tar grep sudo awk sed gzip gunzip - ).join('|') + ) state :basic do rule %r/#.*$/, Comment - rule %r/(#{KEYWORDS})\s*\b/, Keyword - rule %r/case\b/, Keyword, :case + keywords %r/\w+(?![.-])/ do + rule Set['case'], Keyword, :case + rule KEYWORDS, Keyword + rule BUILTINS, Name::Builtin + end - rule %r/(#{BUILTINS})\s*\b(?!(\.|-))/, Name::Builtin rule %r/[.](?=\s)/, Name::Builtin rule %r/(\w+)(=)/ do @@ -64,6 +66,7 @@ def self.detect?(text) rule %r/[\[\]{}()!=>]/, Operator rule %r/&&|\|\|/, Operator + rule %r/[|]/, Punctuation # here-string rule %r/<< Date: Tue, 9 Jun 2026 17:49:51 -0400 Subject: [PATCH 105/154] fix brittle specs --- spec/lexers/console_spec.rb | 4 ++-- spec/lexers/make_spec.rb | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/spec/lexers/console_spec.rb b/spec/lexers/console_spec.rb index 68fe9fd298..225b0f4ac4 100644 --- a/spec/lexers/console_spec.rb +++ b/spec/lexers/console_spec.rb @@ -26,8 +26,8 @@ assert_tokens_equal '$ cd application\\bin\\', ['Generic.Prompt', '$'], ['Text.Whitespace', ' '], - ['Name.Builtin', 'cd '], - ['Text', 'application'], + ['Name.Builtin', 'cd'], + ['Text', ' application'], ['Literal.String.Escape', '\\b'], ['Keyword', 'in'], ['Literal.String.Escape', '\\'] diff --git a/spec/lexers/make_spec.rb b/spec/lexers/make_spec.rb index dc7fbe3e2d..c7c3390eef 100644 --- a/spec/lexers/make_spec.rb +++ b/spec/lexers/make_spec.rb @@ -28,7 +28,7 @@ it 'recognizes declarations not terminated by a new line (#694)' do assert_tokens_equal "hello: \n\techo hello", - ["Name.Label", "hello"], ["Operator", ":"], ["Text", " \n\t"], ["Name.Builtin", "echo "], ["Text", "hello"] + ["Name.Label", "hello"], ["Operator", ":"], ["Text", " \n\t"], ["Name.Builtin", "echo"], ["Text", " hello"] end end end From 61866c3da80ce5a52e0e2d6d389d4fcc68ac7aec Mon Sep 17 00:00:00 2001 From: jneen Date: Tue, 9 Jun 2026 20:48:12 -0400 Subject: [PATCH 106/154] clean up the mason lexer somewhat and handle the quoting problem slightly more elegantly --- .rubocop_todo.yml | 1 - lib/rouge/lexers/mason.rb | 40 ++++++++++++++++++++------------------- spec/visual/samples/mason | 2 ++ 3 files changed, 23 insertions(+), 20 deletions(-) diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index d55b8be759..7c038364bd 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -30,7 +30,6 @@ Naming/VariableName: # Offense count: 113 Rouge/NoBuildingAlternationPatternInRegexp: Exclude: - - 'lib/rouge/lexers/mason.rb' - 'lib/rouge/lexers/mosel.rb' - 'lib/rouge/lexers/nix.rb' - 'lib/rouge/lexers/p4.rb' diff --git a/lib/rouge/lexers/mason.rb b/lib/rouge/lexers/mason.rb index c174436733..5c32592b63 100644 --- a/lib/rouge/lexers/mason.rb +++ b/lib/rouge/lexers/mason.rb @@ -10,15 +10,12 @@ class Mason < TemplateLexer filenames '*.mi', '*.mc', '*.mas', '*.m', '*.mhtml', '*.mcomp', 'autohandler', 'dhandler' mimetypes 'text/x-mason', 'application/x-mason' - def initialize(*) - super - @perl = Perl.new - end - # Note: If you add a tag in the lines below, you also need to modify "disambiguate '*.m'" in file disambiguation.rb - TEXT_BLOCKS = %w(text doc) - PERL_BLOCKS = %w(args flags attr init once shared perl cleanup filter) - COMPONENTS = %w(def method) + PERL_BLOCKS = Set.new %w(args flags attr init once shared perl cleanup filter) + + start do + @perl = Perl.new(@options) + end state :root do mixin :mason_tags @@ -27,14 +24,17 @@ def initialize(*) state :mason_tags do rule %r/\s+/, Text::Whitespace - rule %r/<%(#{TEXT_BLOCKS.join('|')})>/oi, Comment::Preproc, :text_block + keywords %r/<%(\w+)>/ do + transform(&:downcase) + group 1 + rule Set['text', 'doc'], Comment::Preproc, :text_block + rule PERL_BLOCKS, Comment::Preproc, :perl_block + end - rule %r/<%(#{PERL_BLOCKS.join('|')})>/oi, Comment::Preproc, :perl_block + # rule %r/<%(?:def|method)/ - rule %r/(<%(#{COMPONENTS.join('|')}))([^>]*)(>)/oi do |m| - token Comment::Preproc, m[1] - token Name, m[3] - token Comment::Preproc, m[4] + rule %r/(<%(?:def|method))([^>]*)(>)/oi do |m| + groups Comment::Preproc, Name::Function, Comment::Preproc push :component_block end @@ -65,21 +65,23 @@ def initialize(*) end state :perl_block do - rule %r/<\/%(#{PERL_BLOCKS.join('|')})>/oi, Comment::Preproc, :pop! + rule %r/<\/%\w+>/, Comment::Preproc, :pop! rule %r/\s+/, Text::Whitespace rule %r/^(#.*)$/, Comment - rule(/(.*?[^"])(?=<\/%)/m) { delegate @perl } + rule(/(['"`])(?:\\.|[^\\])*?\1/) { delegate @perl } + rule(/[<]/) { delegate @perl } + rule(/[^'"`<]+/m) { delegate @perl } end state :text_block do - rule %r/<\/%(#{TEXT_BLOCKS.join('|')})>/oi, Comment::Preproc, :pop! + rule %r/<\/%\w+>/, Comment::Preproc, :pop! rule %r/\s+/, Text::Whitespace rule %r/^(#.*)$/, Comment - rule %r/(.*?[^"])(?=<\/%)/m, Comment + rule %r/(.+?[^"])(?=<\/%)/m, Comment end state :component_block do - rule %r/<\/%(#{COMPONENTS.join('|')})>/oi, Comment::Preproc, :pop! + rule %r/<\/%\w+>/, Comment::Preproc, :pop! rule %r/\s+/, Text::Whitespace rule %r/^(#.*)$/, Comment mixin :mason_tags diff --git a/spec/visual/samples/mason b/spec/visual/samples/mason index b51d20585e..7f2c606938 100644 --- a/spec/visual/samples/mason +++ b/spec/visual/samples/mason @@ -15,6 +15,8 @@ <%flags> inherit => "page-framework.mi" inherit => "" + inherit => '' + thing => (a < b) <%attr> From 66a18117747551fbe89f7f0ec77c119dfea67617 Mon Sep 17 00:00:00 2001 From: jneen Date: Tue, 9 Jun 2026 20:53:50 -0400 Subject: [PATCH 107/154] use the keywords api for Mathematica --- lib/rouge/lexers/mathematica.rb | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/lib/rouge/lexers/mathematica.rb b/lib/rouge/lexers/mathematica.rb index 00404b7456..c2535114ff 100644 --- a/lib/rouge/lexers/mathematica.rb +++ b/lib/rouge/lexers/mathematica.rb @@ -53,11 +53,7 @@ class Mathematica < RegexLexer # Although Module, With and Block are normal built-in symbols, we give them a special treatment as they are # the most important expressions for defining local variables - def self.keywords - @keywords = Set.new %w( - Module With Block - ) - end + KEYWORDS = Set.new %w(Module With Block) state :root do rule %r/\s+/, Text::Whitespace @@ -66,16 +62,13 @@ def self.keywords rule %r/(?:#{number}#{precision}?(?:\*\^[+-]?\d+)?)/, Num # all other numbers rule message, Name::Tag rule in_out, Generic::Prompt - rule %r/#{context_symbol}/m do |m| - match = m[0] - if self.class.keywords.include? match - token Name::Builtin::Pseudo - elsif BUILTINS.include? match - token Name::Builtin - else - token Name::Variable - end + + keywords context_symbol do + rule KEYWORDS, Keyword::Pseudo + rule BUILTINS, Name::Builtin + default Name end + rule slot, Name::Function rule operators, Operator rule braces, Punctuation From aab9ed45cb05d1afbf9434a3f838b6eed2048612 Mon Sep 17 00:00:00 2001 From: jneen Date: Tue, 9 Jun 2026 20:58:34 -0400 Subject: [PATCH 108/154] use the keywords api for matlab --- lib/rouge/lexers/matlab.rb | 22 +++++++++++----------- spec/visual/samples/matlab | 3 +++ 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/lib/rouge/lexers/matlab.rb b/lib/rouge/lexers/matlab.rb index 7d487e9f1d..be21331653 100644 --- a/lib/rouge/lexers/matlab.rb +++ b/lib/rouge/lexers/matlab.rb @@ -36,18 +36,18 @@ def self.keywords delegate Shell, m[2] end + ident = %r/[a-z][_a-z0-9]*/i - # TODO [jneen]: there are elements of BUILTINS with - # a . in them - this won't match any of those. - rule %r/[a-zA-Z][_a-zA-Z0-9]*/m do |m| - match = m[0] - if self.class.keywords.include? match - token Keyword - elsif BUILTINS.include? match - token Name::Builtin - else - token Name - end + # [jneen]: there are elements of BUILTINS with + # a . in them - the generic rule below won't match any of those. + keywords %r/#{ident}[.]#{ident}/ do + rule BUILTINS, Name::Builtin + end + + keywords ident do + rule :keywords, Keyword + rule BUILTINS, Name::Builtin + default Name end rule %r{[(){};:,\/\\\]\[]}, Punctuation diff --git a/spec/visual/samples/matlab b/spec/visual/samples/matlab index 0ddf565358..4b9f6c8431 100644 --- a/spec/visual/samples/matlab +++ b/spec/visual/samples/matlab @@ -16,6 +16,9 @@ end a = rand(30); b = rand(30); +c = H5F.get_name(foo); +c = H5F.some_weird_function_that_is_not_builtin(foo); + c = a .* b ./ a \ ... comment at end of line and continuation (b .* a + b - a); From 5a0d7a37c28f058f1197f75a384b34a0f5420dc7 Mon Sep 17 00:00:00 2001 From: jneen Date: Tue, 9 Jun 2026 21:06:53 -0400 Subject: [PATCH 109/154] use the keywords api for meson --- lib/rouge/lexers/meson.rb | 67 ++++++++++++++++----------------------- 1 file changed, 27 insertions(+), 40 deletions(-) diff --git a/lib/rouge/lexers/meson.rb b/lib/rouge/lexers/meson.rb index 046cff26ed..d8b72d018e 100644 --- a/lib/rouge/lexers/meson.rb +++ b/lib/rouge/lexers/meson.rb @@ -10,32 +10,26 @@ class Meson < RegexLexer filenames 'meson.build', 'meson_options.txt' mimetypes 'text/x-meson' - def self.keywords - @keywords ||= %w( - continue break elif else endif - if true false foreach endforeach - ) - end - - def self.builtin_variables - @builtin_variables ||= %w( - meson host_machine build_machine target_machine - ) - end - - def self.builtin_functions - @builtin_functions ||= %w( - add_global_arguments add_project_arguments - add_global_link_arguments add_project_link_arguments add_test_setup add_languages - alias_target assert benchmark both_libraries build_target configuration_data configure_file - custom_target declare_dependency dependency disabler environment error executable - generator gettext get_option get_variable files find_library find_program - include_directories import install_data install_headers install_man install_subdir - is_disabler is_variable jar join_paths library message option project - run_target run_command set_variable subdir subdir_done - subproject summary shared_library shared_module static_library test vcs_tag warning - ) - end + KEYWORDS = Set.new %w( + continue break elif else endif + if true false foreach endforeach + ) + + BUILTIN_VARIABLES = Set.new %w( + meson host_machine build_machine target_machine + ) + + BUILTIN_FUNCTIONS = Set.new %w( + add_global_arguments add_project_arguments + add_global_link_arguments add_project_link_arguments add_test_setup add_languages + alias_target assert benchmark both_libraries build_target configuration_data configure_file + custom_target declare_dependency dependency disabler environment error executable + generator gettext get_option get_variable files find_library find_program + include_directories import install_data install_headers install_man install_subdir + is_disabler is_variable jar join_paths library message option project + run_target run_command set_variable subdir subdir_done + subproject summary shared_library shared_module static_library test vcs_tag warning + ) identifier = /[[:alpha:]_][[:alnum:]_]*/ @@ -61,22 +55,15 @@ def current_string push :generic_string end - rule %r/(? Date: Tue, 9 Jun 2026 21:10:20 -0400 Subject: [PATCH 110/154] use the keywords api for minizinc --- lib/rouge/lexers/minizinc.rb | 66 ++++++++++++++---------------------- 1 file changed, 26 insertions(+), 40 deletions(-) diff --git a/lib/rouge/lexers/minizinc.rb b/lib/rouge/lexers/minizinc.rb index 937a140a00..006d81617c 100644 --- a/lib/rouge/lexers/minizinc.rb +++ b/lib/rouge/lexers/minizinc.rb @@ -13,37 +13,29 @@ class MiniZinc < RegexLexer filenames '*.mzn', '*.fzn', '*.dzn' mimetypes 'text/minizinc' - def self.builtins - @builtins = Set.new %w[ - abort abs acosh array_intersect array_union array1d array2d array3d - array4d array5d array6d asin assert atan bool2int card ceil concat - cos cosh dom dom_array dom_size fix exp floor index_set - index_set_1of2 index_set_2of2 index_set_1of3 index_set_2of3 - index_set_3of3 int2float is_fixed join lb lb_array length ln log log2 - log10 min max pow product round set2array show show_int show_float - sin sinh sqrt sum tan tanh trace ub ub_array - ] - end + BUILTINS = Set.new %w[ + abort abs acosh array_intersect array_union array1d array2d array3d + array4d array5d array6d asin assert atan bool2int card ceil concat + cos cosh dom dom_array dom_size fix exp floor index_set + index_set_1of2 index_set_2of2 index_set_1of3 index_set_2of3 + index_set_3of3 int2float is_fixed join lb lb_array length ln log log2 + log10 min max pow product round set2array show show_int show_float + sin sinh sqrt sum tan tanh trace ub ub_array + ] - def self.keywords - @keywords = Set.new %w[ - ann annotation any constraint else endif function for forall if - include list of op output minimize maximize par predicate record - satisfy solve test then type var where - ] - end + KEYWORDS = Set.new %w[ + ann annotation any constraint else endif function for forall if + include list of op output minimize maximize par predicate record + satisfy solve test then type var where + ] - def self.keywords_type - @keywords_type ||= Set.new %w( - array set bool enum float int string tuple - ) - end + KEYWORDS_TYPE = Set.new %w( + array set bool enum float int string tuple + ) - def self.operators - @operators ||= Set.new %w( - in subset superset union diff symdiff intersect - ) - end + OPERATORS = Set.new %w( + in subset superset union diff symdiff intersect + ) id = /[$a-zA-Z_]\w*/ @@ -62,18 +54,12 @@ def self.operators rule %r((true|false)\b), Keyword::Constant rule %r(([+-]?)\d+(\.(?!\.)\d*)?([eE][-+]?\d+)?), Literal::Number - rule id do |m| - if self.class.keywords.include? m[0] - token Keyword - elsif self.class.keywords_type.include? m[0] - token Keyword::Type - elsif self.class.builtins.include? m[0] - token Name::Builtin - elsif self.class.operators.include? m[0] - token Operator - else - token Name::Other - end + keywords id do + rule KEYWORDS, Keyword + rule KEYWORDS_TYPE, Keyword::Type + rule BUILTINS, Name::Builtin + rule OPERATORS, Operator + default Name end rule %r(::\s*([^\W\d]\w*)(\s*\([^\)]*\))?), Name::Decorator From 909750277ea18d78e41ae934391d02916cf3e71b Mon Sep 17 00:00:00 2001 From: jneen Date: Tue, 9 Jun 2026 21:26:32 -0400 Subject: [PATCH 111/154] use the keywords api for mosel, and adhere properly to the spec --- .rubocop_todo.yml | 1 - lib/rouge/lexers/mosel.rb | 183 ++++------------------------- lib/rouge/lexers/mosel/keywords.rb | 146 +++++++++++++++++++++++ 3 files changed, 171 insertions(+), 159 deletions(-) create mode 100644 lib/rouge/lexers/mosel/keywords.rb diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index 7c038364bd..b814261df9 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -30,7 +30,6 @@ Naming/VariableName: # Offense count: 113 Rouge/NoBuildingAlternationPatternInRegexp: Exclude: - - 'lib/rouge/lexers/mosel.rb' - 'lib/rouge/lexers/nix.rb' - 'lib/rouge/lexers/p4.rb' - 'lib/rouge/lexers/pascal.rb' diff --git a/lib/rouge/lexers/mosel.rb b/lib/rouge/lexers/mosel.rb index bad56d0f65..9c3bc7d8c7 100644 --- a/lib/rouge/lexers/mosel.rb +++ b/lib/rouge/lexers/mosel.rb @@ -15,149 +15,7 @@ class Mosel < RegexLexer id = /[a-zA-Z_][a-zA-Z0-9_]*/ - ############################################################################################################################ - # General language lements - ############################################################################################################################ - - core_keywords = %w( - and array as - boolean break - case count counter - declarations div do dynamic - elif else end evaluation exit - false forall forward from function - if imports in include initialisations initializations integer inter is_binary is_continuous is_free is_integer is_partint is_semcont is_semint is_sos1 is_sos2 - linctr list - max min mod model mpvar - next not of options or - package parameters procedure - public prod range real record repeat requirements - set string sum - then to true - union until uses - version - while with - ) - - core_functions = %w( - abs arctan assert - bitflip bitneg bitset bitshift bittest bitval - ceil cos create currentdate currenttime cuthead cuttail - delcell exists exit exp exportprob - fclose fflush finalize findfirst findlast floor fopen fselect fskipline - getact getcoeff getcoeffs getdual getfid getfirst gethead getfname getlast getobjval getparam getrcost getreadcnt getreverse getsize getslack getsol gettail gettype getvars - iseof ishidden isodd ln log - makesos1 makesos2 maxlist minlist - publish - random read readln reset reverse round - setcoeff sethidden setioerr setname setparam setrandseed settype sin splithead splittail sqrt strfmt substr - timestamp - unpublish - write writeln - ) - - ############################################################################################################################ - # mmxprs module elements - ############################################################################################################################ - - mmxprs_functions = %w( - addmipsol - basisstability - calcsolinfo clearmipdir clearmodcut command copysoltoinit - defdelayedrows defsecurevecs - estimatemarginals - fixglobal - getbstat getdualray getiis getiissense getiistype getinfcause getinfeas getlb getloadedlinctrs getloadedmpvars getname getprimalray getprobstat getrange getsensrng getsize getsol getub getvars - implies indicator isiisvalid isintegral loadbasis - loadmipsol loadprob - maximize minimize - postsolve - readbasis readdirs readsol refinemipsol rejectintsol repairinfeas resetbasis resetiis resetsol - savebasis savemipsol savesol savestate selectsol setbstat setcallback setcbcutoff setgndata setlb setmipdir setmodcut setsol setub setucbdata stopoptimize - unloadprob - writebasis writedirs writeprob writesol - xor - ) - - mmxpres_constants = %w(XPRS_OPT XPRS_UNF XPRS_INF XPRS_UNB XPRS_OTH) - - mmxprs_parameters = %w(XPRS_colorder XPRS_enumduplpol XPRS_enummaxsol XPRS_enumsols XPRS_fullversion XPRS_loadnames XPRS_problem XPRS_probname XPRS_verbose) - - - ############################################################################################################################ - # mmsystem module elements - ############################################################################################################################ - - mmsystem_functions = %w( - addmonths - copytext cuttext - deltext - endswith expandpath - fcopy fdelete findfiles findtext fmove - getasnumber getchar getcwd getdate getday getdaynum getdays getdirsep - getendparse setendparse - getenv getfsize getfstat getftime gethour getminute getmonth getmsec getpathsep - getqtype setqtype - getsecond - getsepchar setsepchar - getsize - getstart setstart - getsucc setsucc - getsysinfo getsysstat gettime - gettmpdir - gettrim settrim - getweekday getyear - inserttext isvalid - makedir makepath newtar - newzip nextfield - openpipe - parseextn parseint parsereal parsetext pastetext pathmatch pathsplit - qsort quote - readtextline regmatch regreplace removedir removefiles - setchar setdate setday setenv sethour - setminute setmonth setmsec setsecond settime setyear sleep startswith system - tarlist textfmt tolower toupper trim - untar unzip - ziplist - ) - - mmsystem_parameters = %w(datefmt datetimefmt monthnames sys_endparse sys_fillchar sys_pid sys_qtype sys_regcache sys_sepchar) - - ############################################################################################################################ - # mmjobs module elements - ############################################################################################################################ - - mmjobs_instance_mgmt_functions = %w( - clearaliases connect - disconnect - findxsrvs - getaliases getbanner gethostalias - sethostalias - ) - - mmjobs_model_mgmt_functions = %w( - compile - detach - getannidents getannotations getexitcode getgid getid getnode getrmtid getstatus getuid - load - reset resetmodpar run - setcontrol setdefstream setmodpar setworkdir stop - unload - ) - - mmjobs_synchornization_functions = %w( - dropnextevent - getclass getfromgid getfromid getfromuid getnextevent getvalue - isqueueempty - nullevent - peeknextevent - send setgid setuid - wait waitfor - ) - - mmjobs_functions = mmjobs_instance_mgmt_functions + mmjobs_model_mgmt_functions + mmjobs_synchornization_functions - - mmjobs_parameters = %w(conntmpl defaultnode fsrvdelay fsrvnbiter fsrvport jobid keepalive nodenumber parentnumber) + lazy { require_relative 'mosel/keywords' } state :whitespace do @@ -201,22 +59,31 @@ class Mosel < RegexLexer rule %r/(true|false)\b/i, Name::Builtin - rule %r/\b(#{core_keywords.join('|')})\b/i, Keyword - rule %r/\b(#{core_functions.join('|')})\b/, Name::Builtin - - - rule %r/\b(#{mmxprs_functions.join('|')})\b/, Name::Function - rule %r/\b(#{mmxpres_constants.join('|')})\b/, Name::Constant - rule %r/\b(#{mmxprs_parameters.join('|')})\b/i, Name::Property - - rule %r/\b(#{mmsystem_functions.join('|')})\b/i, Name::Function - rule %r/\b(#{mmsystem_parameters.join('|')})\b/, Name::Property - - rule %r/\b(#{mmjobs_functions.join('|')})\b/i, Name::Function - rule %r/\b(#{mmjobs_parameters.join('|')})\b/, Name::Property - - rule id, Name + # Handle the extremely special case of core keywords being valid if they are *all* upper case, + # but not *partially* upper case. + # + # ref: https://www.fico.com/fico-xpress-optimization/docs/latest/mosel/mosel_lang/dhtml/moselreflang.html?scroll=seclangintro_2 + # > Note that, although the lexical analyzer of Mosel is case-sensitive, + # > the reserved words are defined both as lower and upper case (i.e. AND + # > and and are keywords but not And). + rule id do |m| + fallthrough! unless CORE_KEYWORDS.include?(m[0].downcase) + fallthrough! unless m[0].match?(/\A[A-Z_]+\z|\A[a-z_]+\z/) + token Keyword + end + + keywords id do + rule CORE_FUNCTIONS, Name::Builtin + rule MMXPRS_FUNCTIONS, Name::Function + rule MMXPRES_CONSTANTS, Name::Constant + rule MMXPRS_PARAMETERS, Name::Property + rule MMSYSTEM_FUNCTIONS, Name::Function + rule MMSYSTEM_PARAMETERS, Name::Property + rule MMJOBS_FUNCTIONS, Name::Function + rule MMJOBS_PARAMETERS, Name::Property + default Name + end end state :root do diff --git a/lib/rouge/lexers/mosel/keywords.rb b/lib/rouge/lexers/mosel/keywords.rb new file mode 100644 index 0000000000..30d4f56ce4 --- /dev/null +++ b/lib/rouge/lexers/mosel/keywords.rb @@ -0,0 +1,146 @@ +# -*- coding: utf-8 -*- # +# frozen_string_literal: true + +module Rouge + module Lexers + class Mosel + ############################################################################################################################ + # General language lements + ############################################################################################################################ + + CORE_KEYWORDS = Set.new %w( + and array as + boolean break + case count counter + declarations div do dynamic + elif else end evaluation exit + false forall forward from function + if imports in include initialisations initializations integer inter is_binary is_continuous is_free is_integer is_partint is_semcont is_semint is_sos1 is_sos2 + linctr list + max min mod model mpvar + next not of options or + package parameters procedure + public prod range real record repeat requirements + set string sum + then to true + union until uses + version + while with + ) + + CORE_FUNCTIONS = Set.new %w( + abs arctan assert + bitflip bitneg bitset bitshift bittest bitval + ceil cos create currentdate currenttime cuthead cuttail + delcell exists exit exp exportprob + fclose fflush finalize findfirst findlast floor fopen fselect fskipline + getact getcoeff getcoeffs getdual getfid getfirst gethead getfname getlast getobjval getparam getrcost getreadcnt getreverse getsize getslack getsol gettail gettype getvars + iseof ishidden isodd ln log + makesos1 makesos2 maxlist minlist + publish + random read readln reset reverse round + setcoeff sethidden setioerr setname setparam setrandseed settype sin splithead splittail sqrt strfmt substr + timestamp + unpublish + write writeln + ) + + ############################################################################################################################ + # mmxprs module elements + ############################################################################################################################ + + MMXPRS_FUNCTIONS = Set.new %w( + addmipsol + basisstability + calcsolinfo clearmipdir clearmodcut command copysoltoinit + defdelayedrows defsecurevecs + estimatemarginals + fixglobal + getbstat getdualray getiis getiissense getiistype getinfcause getinfeas getlb getloadedlinctrs getloadedmpvars getname getprimalray getprobstat getrange getsensrng getsize getsol getub getvars + implies indicator isiisvalid isintegral loadbasis + loadmipsol loadprob + maximize minimize + postsolve + readbasis readdirs readsol refinemipsol rejectintsol repairinfeas resetbasis resetiis resetsol + savebasis savemipsol savesol savestate selectsol setbstat setcallback setcbcutoff setgndata setlb setmipdir setmodcut setsol setub setucbdata stopoptimize + unloadprob + writebasis writedirs writeprob writesol + xor + ) + + MMXPRES_CONSTANTS = Set.new %w(XPRS_OPT XPRS_UNF XPRS_INF XPRS_UNB XPRS_OTH) + + MMXPRS_PARAMETERS = Set.new %w(XPRS_colorder XPRS_enumduplpol XPRS_enummaxsol XPRS_enumsols XPRS_fullversion XPRS_loadnames XPRS_problem XPRS_probname XPRS_verbose) + + + ############################################################################################################################ + # mmsystem module elements + ############################################################################################################################ + + MMSYSTEM_FUNCTIONS = Set.new %w( + addmonths + copytext cuttext + deltext + endswith expandpath + fcopy fdelete findfiles findtext fmove + getasnumber getchar getcwd getdate getday getdaynum getdays getdirsep + getendparse setendparse + getenv getfsize getfstat getftime gethour getminute getmonth getmsec getpathsep + getqtype setqtype + getsecond + getsepchar setsepchar + getsize + getstart setstart + getsucc setsucc + getsysinfo getsysstat gettime + gettmpdir + gettrim settrim + getweekday getyear + inserttext isvalid + makedir makepath newtar + newzip nextfield + openpipe + parseextn parseint parsereal parsetext pastetext pathmatch pathsplit + qsort quote + readtextline regmatch regreplace removedir removefiles + setchar setdate setday setenv sethour + setminute setmonth setmsec setsecond settime setyear sleep startswith system + tarlist textfmt tolower toupper trim + untar unzip + ziplist + ) + + MMSYSTEM_PARAMETERS = Set.new %w(datefmt datetimefmt monthnames sys_endparse sys_fillchar sys_pid sys_qtype sys_regcache sys_sepchar) + + ############################################################################################################################ + # mmjobs module elements + ############################################################################################################################ + + MMJOBS_FUNCTIONS = Set.new %w( + clearaliases connect + disconnect + findxsrvs + getaliases getbanner gethostalias + sethostalias + + compile + detach + getannidents getannotations getexitcode getgid getid getnode getrmtid getstatus getuid + load + reset resetmodpar run + setcontrol setdefstream setmodpar setworkdir stop + unload + + dropnextevent + getclass getfromgid getfromid getfromuid getnextevent getvalue + isqueueempty + nullevent + peeknextevent + send setgid setuid + wait waitfor + ) + + MMJOBS_PARAMETERS = Set.new %w(conntmpl defaultnode fsrvdelay fsrvnbiter fsrvport jobid keepalive nodenumber parentnumber) + end + end +end From aecac3e9ad8d147adcb22c14393b6436c997a863 Mon Sep 17 00:00:00 2001 From: jneen Date: Tue, 9 Jun 2026 21:30:20 -0400 Subject: [PATCH 112/154] use the keywords api for nesasm --- lib/rouge/lexers/nesasm.rb | 51 +++++++++++++++----------------------- 1 file changed, 20 insertions(+), 31 deletions(-) diff --git a/lib/rouge/lexers/nesasm.rb b/lib/rouge/lexers/nesasm.rb index dda3e77020..dd4c21123b 100644 --- a/lib/rouge/lexers/nesasm.rb +++ b/lib/rouge/lexers/nesasm.rb @@ -10,28 +10,22 @@ class NesAsm < RegexLexer aliases 'nes' filenames '*.nesasm' - def self.keywords - @keywords ||= %w( - ADC AND ASL BIT BRK CMP CPX CPY DEC EOR INC JMP JSR LDA LDX LDY LSR - NOP ORA ROL ROR RTI RTS SBC STA STX STY TAX TXA DEX INX TAY TYA DEY - INY BPL BMI BVC BVS BCC BCS BNE BEQ CLC SEC CLI SEI CLV CLD SED TXS - TSX PHA PLA PHP PLP - ) - end + KEYWORDS = Set.new %w( + ADC AND ASL BIT BRK CMP CPX CPY DEC EOR INC JMP JSR LDA LDX LDY LSR + NOP ORA ROL ROR RTI RTS SBC STA STX STY TAX TXA DEX INX TAY TYA DEY + INY BPL BMI BVC BVS BCC BCS BNE BEQ CLC SEC CLI SEI CLV CLD SED TXS + TSX PHA PLA PHP PLP + ) - def self.keywords_type - @keywords_type ||= %w( - DB DW BYTE WORD - ) - end + KEYWORDS_TYPE = Set.new %w( + DB DW BYTE WORD + ) - def self.keywords_reserved - @keywords_reserved ||= %w( - INCBIN INCLUDE ORG BANK RSSET RS MACRO ENDM DS PROC ENDP PROCGROUP - ENDPROCGROUP INCCHR DEFCHR ZP BSS CODE DATA IF IFDEF IFNDEF ELSE - ENDIF FAIL INESPRG INESCHR INESMAP INESMIR FUNC - ) - end + KEYWORDS_RESERVED = Set.new %w( + INCBIN INCLUDE ORG BANK RSSET RS MACRO ENDM DS PROC ENDP PROCGROUP + ENDPROCGROUP INCCHR DEFCHR ZP BSS CODE DATA IF IFDEF IFNDEF ELSE + ENDIF FAIL INESPRG INESCHR INESMAP INESMIR FUNC + ) state :root do rule %r/\s+/m, Text @@ -43,18 +37,13 @@ def self.keywords_reserved rule %r/\#?\d+/, Num # 10 #10 rule %r([~&*+=\|?:<>/-]), Operator - rule %r/\#?\w+:?/i do |m| - name = m[0].upcase + keywords %r/\#?\w+:?/i do |m| + transform(&:upcase) - if self.class.keywords.include? name - token Keyword - elsif self.class.keywords_type.include? name - token Keyword::Type - elsif self.class.keywords_reserved.include? name - token Keyword::Reserved - else - token Name::Function - end + rule KEYWORDS, Keyword + rule KEYWORDS_TYPE, Keyword::Type + rule KEYWORDS_RESERVED, Keyword::Reserved + default Name::Function end rule %r/\#?(?:LOW|HIGH)\(.*\)/i, Keyword::Reserved # LOW() #HIGH() From 715796d8c36702b7be57c600235b09e4d7e71823 Mon Sep 17 00:00:00 2001 From: jneen Date: Tue, 9 Jun 2026 21:33:06 -0400 Subject: [PATCH 113/154] use the keywords api for nial --- lib/rouge/lexers/nial.rb | 40 +++++++++++++++------------------------- 1 file changed, 15 insertions(+), 25 deletions(-) diff --git a/lib/rouge/lexers/nial.rb b/lib/rouge/lexers/nial.rb index e6ecb72c93..ca28875599 100644 --- a/lib/rouge/lexers/nial.rb +++ b/lib/rouge/lexers/nial.rb @@ -120,36 +120,26 @@ def self.consts rule %r/\?[^\s()\[\]{}#,;]*/, Generic::Error rule %r/%[^;]+;/, Comment::Multiline rule %r/^#(.+\n)+\n/, Comment::Multiline - rule %r/:=|[\{\}\[\]\(\),:;]/ do |m| - if self.class.punctuations.include?(m[0]) - token Punctuation - else - token Text - end + keywords %r/:=|[\{\}\[\]\(\),:;]/ do + rule :punctuations, Punctuation + default Text end + # [".", "!", "#", "+", "*", "-", "<<", # "/", "<", ">>", "<=", ">", "=", ">=", "@", "|", "~="] - rule %r'>>|>=|<=|~=|[\.!#+*\-=>>|>=|<=|~=|[\.!#+*\-=> Date: Tue, 9 Jun 2026 21:40:34 -0400 Subject: [PATCH 114/154] use the keywords api for nim --- lib/rouge/lexers/nim.rb | 39 +++++++++++++++++++-------------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/lib/rouge/lexers/nim.rb b/lib/rouge/lexers/nim.rb index 4625846afd..7a8e77fb8e 100644 --- a/lib/rouge/lexers/nim.rb +++ b/lib/rouge/lexers/nim.rb @@ -12,7 +12,7 @@ class Nim < RegexLexer aliases 'nimrod' filenames '*.nim' - KEYWORDS = %w( + KEYWORDS = Set.new %w( addr as asm atomic bind block break case cast const continue converter defer discard distinct do elif else end enum except export func finally for from generic if import include interface iterator let @@ -20,29 +20,23 @@ class Nim < RegexLexer template try tuple type using var when while with without yield ) - OPWORDS = %w( + OPWORDS = Set.new %w( and or not xor shl shr div mod in notin is isnot ) - PSEUDOKEYWORDS = %w( + PSEUDOKEYWORDS = Set.new %w( nil true false ) - TYPES = %w( + TYPES = Set.new %w( int int8 int16 int32 int64 float float32 float64 bool char range array seq set string ) - NAMESPACE = %w( + NAMESPACE = Set.new %w( from import include ) - def self.underscorize(words) - words.map do |w| - w.gsub(/./) { |x| "#{Regexp.escape(x)}_?" } - end.join('|') - end - state :chars do rule(/\\([\\abcefnrtvl"\']|x[a-fA-F0-9]{2}|[0-9]{1,3})/, Str::Escape) rule(/'/, Str::Char, :pop!) @@ -119,16 +113,21 @@ def self.underscorize(words) # Char rule(/'/, Str::Char, :chars) - # Keywords - rule(%r[(#{Nim.underscorize(OPWORDS)})\b], Operator::Word) rule(/(p_?r_?o_?c_?\s)(?![\(\[\]])/, Keyword, :funcname) - rule(%r[(#{Nim.underscorize(KEYWORDS)})\b], Keyword) - rule(%r[(#{Nim.underscorize(NAMESPACE)})\b], Keyword::Namespace) - rule(/(v_?a_?r)\b/, Keyword::Declaration) - rule(%r[(#{Nim.underscorize(TYPES)})\b], Keyword::Type) - rule(%r[(#{Nim.underscorize(PSEUDOKEYWORDS)})\b], Keyword::Pseudo) - # Identifiers - rule(/\b((?![_\d])\w)(((?!_)\w)|(_(?!_)\w))*/, Name) + + # Keywords and Identifiers + keywords %r/(?![_\d])\w+/ do + # underscores can be used freely in idents + transform { |word| word.delete('_') } + + rule OPWORDS, Operator::Word + rule KEYWORDS, Keyword + rule NAMESPACE, Keyword::Namespace + rule Set['var'], Keyword::Declaration + rule TYPES, Keyword::Type + rule PSEUDOKEYWORDS, Keyword::Pseudo + default Name + end # Numbers # Note: Have to do this with a block to push multiple states first, From b55d1b09294b7caec03a0b557cbc30c3c8e29cdb Mon Sep 17 00:00:00 2001 From: jneen Date: Tue, 9 Jun 2026 21:51:19 -0400 Subject: [PATCH 115/154] use the keywords api for nix --- .rubocop_todo.yml | 7 ---- lib/rouge/lexers/nix.rb | 77 +++++++++++------------------------------ spec/lexers/nix_spec.rb | 31 ++++++++++++++++- 3 files changed, 50 insertions(+), 65 deletions(-) diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index b814261df9..96f20cf370 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -6,12 +6,6 @@ # Note that changes in the inspected code, or installation of new # versions of RuboCop, may require this file to be generated again. -# Offense count: 1 -# Configuration parameters: AllowComments, AllowEmptyLambdas. -Lint/EmptyBlock: - Exclude: - - 'lib/rouge/lexers/nix.rb' - # Offense count: 16 # Configuration parameters: EnforcedStyle, AllowedPatterns, ForbiddenIdentifiers, ForbiddenPatterns. # SupportedStyles: snake_case, camelCase @@ -30,7 +24,6 @@ Naming/VariableName: # Offense count: 113 Rouge/NoBuildingAlternationPatternInRegexp: Exclude: - - 'lib/rouge/lexers/nix.rb' - 'lib/rouge/lexers/p4.rb' - 'lib/rouge/lexers/pascal.rb' - 'lib/rouge/lexers/postscript.rb' diff --git a/lib/rouge/lexers/nix.rb b/lib/rouge/lexers/nix.rb index ba97eca00e..a3b14246ca 100644 --- a/lib/rouge/lexers/nix.rb +++ b/lib/rouge/lexers/nix.rb @@ -10,7 +10,7 @@ class Nix < RegexLexer aliases 'nixos' filenames '*.nix' - id_boundary = /(?![a-zA-Z0-9_'-])/ + ident = %r/[a-zA-Z_][a-zA-Z0-9_'-]*/ state :whitespaces do rule %r/^\s*\n\s*$/m, Text @@ -28,29 +28,17 @@ class Nix < RegexLexer end state :number do - rule %r/[0-9]/, Num::Integer - end - - state :null do - rule %r/null#{id_boundary}/, Keyword::Constant - end - - state :boolean do - rule %r/(?:true|false)#{id_boundary}/, Keyword::Constant - end - - state :binding do - rule %r/[a-zA-Z_][a-zA-Z0-9_'-]*/, Name::Variable + rule %r/[0-9]+/, Num::Integer end state :path do - word = "[a-zA-Z0-9\._-]+" - section = "(\/#{word})" - prefix = "[a-z\+]+:\/\/" - root = /#{section}+/.source - tilde = /~#{section}+/.source - basic = /#{word}(\/#{word})+/.source - url = /#{prefix}(\/?#{basic})/.source + word = /[a-zA-Z0-9._-]+/ + section = %r(/#{word}) + prefix = %r([a-z+]+://) + root = /#{section}+/ + tilde = /~#{section}+/ + basic = %r(#{word}(/#{word})+) + url = %r(#{prefix}(/?#{basic})) rule %r/(#{root}|#{tilde}|#{basic}|#{url})/, Str::Other end @@ -137,8 +125,6 @@ class Nix < RegexLexer state :expression do mixin :ignore mixin :comment - mixin :boolean - mixin :null mixin :number mixin :path mixin :string @@ -147,42 +133,13 @@ class Nix < RegexLexer mixin :accessor mixin :assignment mixin :delimiter - mixin :binding mixin :atom mixin :set mixin :list end state :keywords do - mixin :keywords_namespace - mixin :keywords_declaration - mixin :keywords_conditional - mixin :keywords_reserved - mixin :keywords_builtin - end - - state :keywords_namespace do - keywords = %w(with in inherit) - rule %r/(?:#{keywords.join('|')})#{id_boundary}/, Keyword::Namespace - end - - state :keywords_declaration do - keywords = %w(let) - rule %r/(?:#{keywords.join('|')})#{id_boundary}/, Keyword::Declaration - end - - state :keywords_conditional do - keywords = %w(if then else) - rule %r/(?:#{keywords.join('|')})#{id_boundary}/, Keyword - end - - state :keywords_reserved do - keywords = %w(rec assert map) - rule %r/(?:#{keywords.join('|')})#{id_boundary}/, Keyword::Reserved - end - - state :keywords_builtin do - keywords = %w( + builtins = Set.new %w( abort baseNameOf builtins @@ -194,7 +151,16 @@ class Nix < RegexLexer throw toString ) - rule %r/(?:#{keywords.join('|')})#{id_boundary}/, Keyword::Reserved + + keywords ident do + rule Set['with', 'in', 'inherit'], Keyword::Namespace + rule Set['let'], Keyword::Declaration + rule Set['null', 'true', 'false'], Keyword::Constant + rule Set['if', 'then', 'else'], Keyword + rule Set['rec', 'assert', 'map'], Keyword::Reserved + rule builtins, Name::Builtin + default Name + end end state :ignore do @@ -205,9 +171,6 @@ class Nix < RegexLexer mixin :ignore mixin :expression end - - start do - end end end end diff --git a/spec/lexers/nix_spec.rb b/spec/lexers/nix_spec.rb index c36b40a5d4..befd3cc486 100644 --- a/spec/lexers/nix_spec.rb +++ b/spec/lexers/nix_spec.rb @@ -8,7 +8,36 @@ include Support::Lexing it 'recognizes Nix identifier boundaries (#2176)' do - assert_tokens_equal "let\n if' = -1;\n trueTest = 1;\n falseTest = 0;\nin { }", ["Keyword.Declaration", "let"], ["Text", "\n "], ["Name.Variable", "if'"], ["Text", " "], ["Operator", "="], ["Text", " "], ["Operator", "-"], ["Literal.Number.Integer", "1"], ["Punctuation", ";"], ["Text", "\n "], ["Name.Variable", "trueTest"], ["Text", " "], ["Operator", "="], ["Text", " "], ["Literal.Number.Integer", "1"], ["Punctuation", ";"], ["Text", "\n "], ["Name.Variable", "falseTest"], ["Text", " "], ["Operator", "="], ["Text", " "], ["Literal.Number.Integer", "0"], ["Punctuation", ";"], ["Text", "\n"], ["Keyword.Namespace", "in"], ["Text", " "], ["Punctuation", "{"], ["Text", " "], ["Punctuation", "}"] + assert_tokens_equal "let\n if' = -1;\n trueTest = 1;\n falseTest = 0;\nin { }", + ["Keyword.Declaration", "let"], + ["Text", "\n "], + ["Name", "if'"], + ["Text", " "], + ["Operator", "="], + ["Text", " "], + ["Operator", "-"], + ["Literal.Number.Integer", "1"], + ["Punctuation", ";"], + ["Text", "\n "], + ["Name", "trueTest"], + ["Text", " "], + ["Operator", "="], + ["Text", " "], + ["Literal.Number.Integer", "1"], + ["Punctuation", ";"], + ["Text", "\n "], + ["Name", "falseTest"], + ["Text", " "], + ["Operator", "="], + ["Text", " "], + ["Literal.Number.Integer", "0"], + ["Punctuation", ";"], + ["Text", "\n"], + ["Keyword.Namespace", "in"], + ["Text", " "], + ["Punctuation", "{"], + ["Text", " "], + ["Punctuation", "}"] end end end From 19e9b84f4a897b0d8bb28189d53980732126ab40 Mon Sep 17 00:00:00 2001 From: jneen Date: Tue, 9 Jun 2026 22:33:51 -0400 Subject: [PATCH 116/154] use the keywords api for objective-c and objective-cpp --- lib/rouge/lexers/objective_c.rb | 2 +- lib/rouge/lexers/objective_c/common.rb | 40 ++++++++++++-------------- lib/rouge/lexers/objective_cpp.rb | 2 +- 3 files changed, 20 insertions(+), 24 deletions(-) diff --git a/lib/rouge/lexers/objective_c.rb b/lib/rouge/lexers/objective_c.rb index 4d39cf12f7..6c03fc086f 100644 --- a/lib/rouge/lexers/objective_c.rb +++ b/lib/rouge/lexers/objective_c.rb @@ -7,7 +7,7 @@ module Rouge module Lexers class ObjectiveC < C - extend ObjectiveCCommon + include ObjectiveCCommon tag 'objective_c' title "Objective-C" diff --git a/lib/rouge/lexers/objective_c/common.rb b/lib/rouge/lexers/objective_c/common.rb index 2cb8107944..b6c2c7ce0a 100644 --- a/lib/rouge/lexers/objective_c/common.rb +++ b/lib/rouge/lexers/objective_c/common.rb @@ -4,23 +4,22 @@ module Rouge module Lexers module ObjectiveCCommon - def at_keywords - @at_keywords ||= %w( - selector private protected public encode synchronized try - throw catch finally end property synthesize dynamic optional - interface implementation import autoreleasepool - ) - end + AT_BUILTINS = Set.new %w(true false YES NO) - def at_builtins - @at_builtins ||= %w(true false YES NO) - end + AT_KEYWORDS = Set.new %w( + selector private protected public encode synchronized try + throw catch finally end property synthesize dynamic optional + interface implementation import autoreleasepool + ) - def builtins - @builtins ||= %w(YES NO nil) - end + BUILTINS = Set.new %w(YES NO nil) + + def self.included(base) + # override for the C/C++ lexers + def base.builtins + BUILTINS + end - def self.extended(base) base.prepend :statements do rule %r/@"/, base::Str, :string rule %r/@'(\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,2}|\\.|[^\\'\n]')/, @@ -36,14 +35,11 @@ def self.extended(base) rule %r/@(?:interface|implementation)\b/, base::Keyword, :objc_classname rule %r/@(?:class|protocol)\b/, base::Keyword, :forward_classname - rule %r/@([[:alnum:]]+)/ do |m| - if base.at_keywords.include? m[1] - token base::Keyword - elsif base.at_builtins.include? m[1] - token base::Name::Builtin - else - token base::Error - end + keywords %r/@([[:alnum:]]+)/ do |m| + group 1 + rule AT_KEYWORDS, base::Keyword + rule AT_BUILTINS, base::Name::Builtin + default base::Error end rule %r/[?]/, base::Punctuation, :ternary diff --git a/lib/rouge/lexers/objective_cpp.rb b/lib/rouge/lexers/objective_cpp.rb index 20ca823a43..32ea9b0035 100644 --- a/lib/rouge/lexers/objective_cpp.rb +++ b/lib/rouge/lexers/objective_cpp.rb @@ -8,7 +8,7 @@ module Rouge module Lexers class ObjectiveCpp < Cpp - extend ObjectiveCCommon + include ObjectiveCCommon tag 'objective_cpp' title "Objective-C++" From 0420530b1e0bec264f986abc51fc8582cb818c9c Mon Sep 17 00:00:00 2001 From: jneen Date: Tue, 9 Jun 2026 22:37:53 -0400 Subject: [PATCH 117/154] use the keywords api for ocaml and rescript --- lib/rouge/lexers/ocaml.rb | 14 ++------------ lib/rouge/lexers/ocaml/common.rb | 9 +++++++++ lib/rouge/lexers/rescript.rb | 13 +------------ 3 files changed, 12 insertions(+), 24 deletions(-) diff --git a/lib/rouge/lexers/ocaml.rb b/lib/rouge/lexers/ocaml.rb index 34d3eb01b9..ab74cea783 100644 --- a/lib/rouge/lexers/ocaml.rb +++ b/lib/rouge/lexers/ocaml.rb @@ -25,18 +25,8 @@ def self.keywords rule %r/`#{@@id}/, Name::Tag rule @@upper_id, Name::Class rule %r/[(][*](?![)])/, Comment, :comment - rule @@id do |m| - match = m[0] - if self.class.keywords.include? match - token Keyword - elsif self.class.word_operators.include? match - token Operator::Word - elsif self.class.primitives.include? match - token Keyword::Type - else - token Name - end - end + + mixin :keywords_and_names rule %r/[(){}\[\];]+/, Punctuation rule @@operator, Operator diff --git a/lib/rouge/lexers/ocaml/common.rb b/lib/rouge/lexers/ocaml/common.rb index bd6f93d9ef..f17884c46c 100644 --- a/lib/rouge/lexers/ocaml/common.rb +++ b/lib/rouge/lexers/ocaml/common.rb @@ -48,6 +48,15 @@ def self.primitives rule @@id, Name, :pop! rule %r/[({\[]/, Punctuation, :pop! end + + state :keywords_and_names do + keywords @@id do + rule :keywords, Keyword + rule :word_operators, Operator::Word + rule :primitives, Keyword::Type + default Name + end + end end end end diff --git a/lib/rouge/lexers/rescript.rb b/lib/rouge/lexers/rescript.rb index 33f61a4763..f6df8bb496 100644 --- a/lib/rouge/lexers/rescript.rb +++ b/lib/rouge/lexers/rescript.rb @@ -55,18 +55,7 @@ def self.word_operators rule %r(/\*), Comment::Multiline, :comment # Keywords and identifiers - rule @@id do |m| - match = m[0] - if self.class.keywords.include? match - token Keyword - elsif self.class.word_operators.include? match - token Operator::Word - elsif self.class.types.include? match - token Keyword::Type - else - token Name - end - end + mixin :keywords_and_names # Braces rule %r/[(){}\[\];]+/, Punctuation From d75b2900347bd40bf80cd3e3614012b7bc7b2373 Mon Sep 17 00:00:00 2001 From: jneen Date: Tue, 9 Jun 2026 22:40:36 -0400 Subject: [PATCH 118/154] use the keywords api for OCL --- lib/rouge/lexers/ocl.rb | 86 +++++++++++++++++------------------------ 1 file changed, 35 insertions(+), 51 deletions(-) diff --git a/lib/rouge/lexers/ocl.rb b/lib/rouge/lexers/ocl.rb index 6511ffa03b..50ccde1907 100644 --- a/lib/rouge/lexers/ocl.rb +++ b/lib/rouge/lexers/ocl.rb @@ -8,47 +8,37 @@ class OCL < RegexLexer filenames '*.ocl' mimetypes 'text/x-ocl' - def self.keywords - @keywords ||= Set.new %w( - context pre post inv init body def derive if then else endif import - package endpackage let in - ) - end + KEYWORDS = Set.new %w( + context pre post inv init body def derive if then else endif import + package endpackage let in + ) - def self.keywords_type - @keywords_type ||= Set.new %w( - Boolean Integer UnlimitedNatural Real String OrderedSet Tuple Bag Set - Sequence OclInvalid OclVoid TupleType OclState Collection OclMessage - ) - end + KEYWORDS_TYPE = Set.new %w( + Boolean Integer UnlimitedNatural Real String OrderedSet Tuple Bag Set + Sequence OclInvalid OclVoid TupleType OclState Collection OclMessage + ) - def self.builtins - @builtins ||= Set.new %w( - self null result true false invalid @pre - ) - end + BUILTINS = Set.new %w( + self null result true false invalid @pre + ) - def self.operators - @operators ||= Set.new %w( - or xor and not implies - ) - end + OPERATORS = Set.new %w( + or xor and not implies + ) - def self.functions - @functions ||= Set.new %w( - oclAsSet oclIsNew oclIsUndefined oclIsInvalid oclAsType oclIsTypeOf - oclIsKindOf oclInState oclType oclLocale hasReturned result - isSignalSent isOperationCallabs floor round max min toString div mod - size substring concat toInteger toReal toUpperCase toLowerCase - indexOf equalsIgnoreCase at characters toBoolean includes excludes - count includesAll excludesAll isEmpty notEmpty sum product - selectByKind selectByType asBag asSequence asOrderedSet asSet flatten - union intersection including excluding symmetricDifferencecount - append prepend insertAt subOrderedSet first last reverse subSequence - any closure collect collectNested exists forAll isUnique iterate one - reject select sortedBy allInstances average conformsTo - ) - end + FUNCTIONS = Set.new %w( + oclAsSet oclIsNew oclIsUndefined oclIsInvalid oclAsType oclIsTypeOf + oclIsKindOf oclInState oclType oclLocale hasReturned result + isSignalSent isOperationCallabs floor round max min toString div mod + size substring concat toInteger toReal toUpperCase toLowerCase + indexOf equalsIgnoreCase at characters toBoolean includes excludes + count includesAll excludesAll isEmpty notEmpty sum product + selectByKind selectByType asBag asSequence asOrderedSet asSet flatten + union intersection including excluding symmetricDifferencecount + append prepend insertAt subOrderedSet first last reverse subSequence + any closure collect collectNested exists forAll isUnique iterate one + reject select sortedBy allInstances average conformsTo + ) state :single_string do rule %r/\\./, Str::Escape @@ -63,20 +53,14 @@ def self.functions rule %r/'/, Str::Single, :single_string rule %r([-|+*/<>=~!@#%&?^]), Operator rule %r/[;:()\[\],.]/, Punctuation - rule %r/[a-zA-Z]\w*/ do |m| - if self.class.operators.include? m[0] - token Operator - elsif self.class.keywords_type.include? m[0] - token Keyword::Declaration - elsif self.class.keywords.include? m[0] - token Keyword - elsif self.class.builtins.include? m[0] - token Name::Builtin - elsif self.class.functions.include? m[0] - token Name::Function - else - token Name - end + + keywords %r/[a-zA-Z]\w*/ do + rule OPERATORS, Operator + rule KEYWORDS_TYPE, Keyword::Declaration + rule KEYWORDS, Keyword + rule BUILTINS, Name::Builtin + rule FUNCTIONS, Name::Function + default Name end end end From 325ab24a8481697f00207f401f671982eee81248 Mon Sep 17 00:00:00 2001 From: jneen Date: Tue, 9 Jun 2026 22:47:53 -0400 Subject: [PATCH 119/154] use the keywords api for openedge --- .rubocop_todo.yml | 1 - lib/rouge/lexers/openedge.rb | 513 +------------------------- lib/rouge/lexers/openedge/keywords.rb | 492 ++++++++++++++++++++++++ 3 files changed, 501 insertions(+), 505 deletions(-) create mode 100644 lib/rouge/lexers/openedge/keywords.rb diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index 96f20cf370..8eaee61074 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -39,7 +39,6 @@ Rouge/NoBuildingAlternationPatternInRegexp: # Offense count: 20 Rouge/NoHugeCollections: Exclude: - - 'lib/rouge/lexers/openedge.rb' - 'lib/rouge/lexers/plsql.rb' - 'lib/rouge/lexers/racket.rb' - 'lib/rouge/lexers/sas.rb' diff --git a/lib/rouge/lexers/openedge.rb b/lib/rouge/lexers/openedge.rb index a3c6213c4d..8ecbb14119 100644 --- a/lib/rouge/lexers/openedge.rb +++ b/lib/rouge/lexers/openedge.rb @@ -12,500 +12,9 @@ class OpenEdge < RegexLexer title 'OpenEdge ABL' desc 'The OpenEdge ABL programming language' - id = /[a-zA-Z_&{}!][a-zA-Z0-9_\-&!}]*/ - - def self.keywords - @keywords ||= Set.new %w( - ABORT ABS ABSO ABSOL ABSOLU ABSOLUT ABSOLUTE ABSTRACT ACCELERATOR ACCEPT-CHANGES ACCEPT-ROW-CHANGES - ACCUM ACCUMU ACCUMUL ACCUMULA ACCUMULAT ACCUMULATE ACROSS ACTIVE ACTIVE-FORM ACTIVE-WINDOW ACTOR ADD - ADD-BUFFER ADD-CALC-COL ADD-CALC-COLU ADD-CALC-COLUM ADD-CALC-COLUMN ADD-COLUMNS-FROM - ADD-EVENTS-PROC ADD-EVENTS-PROCE ADD-EVENTS-PROCED ADD-EVENTS-PROCEDU ADD-EVENTS-PROCEDUR - ADD-EVENTS-PROCEDURE ADD-FIELDS-FROM ADD-FIRST ADD-HEADER-ENTRY ADD-INDEX-FIELD ADD-INTERVAL - ADD-LAST ADD-LIKE-COLUMN ADD-LIKE-FIELD ADD-LIKE-INDEX ADD-NEW-FIELD ADD-NEW-INDEX - ADD-SCHEMA-LOCATION ADD-SOURCE-BUFFER ADD-SUPER-PROC ADD-SUPER-PROCE ADD-SUPER-PROCED - ADD-SUPER-PROCEDU ADD-SUPER-PROCEDUR ADD-SUPER-PROCEDURE ADM-DATA ADVISE AFTER-BUFFER AFTER-ROWID - AFTER-TABLE ALERT-BOX ALIAS ALL ALLOW-COLUMN-SEARCHING ALLOW-PREV-DESERIALIZATION ALLOW-REPLICATION - ALTER ALTERNATE-KEY ALWAYS-ON-TOP AMBIG AMBIGU AMBIGUO AMBIGUOU AMBIGUOUS ANALYZ ANALYZE AND - ANSI-ONLY ANY ANY-KEY ANY-PRINTABLE ANYWHERE APPEND APPEND-CHILD APPEND-LINE APPL-ALERT APPL-ALERT- - APPL-ALERT-B APPL-ALERT-BO APPL-ALERT-BOX APPL-ALERT-BOXE APPL-ALERT-BOXES APPL-CONTEXT-ID - APPLICATION APPLY APPLY-CALLBACK APPSERVER-INFO APPSERVER-PASSWORD APPSERVER-USERID ARRAY-M ARRAY-ME - ARRAY-MES ARRAY-MESS ARRAY-MESSA ARRAY-MESSAG ARRAY-MESSAGE AS ASC ASCE ASCEN ASCEND ASCENDI - ASCENDIN ASCENDING AS-CURSOR ASK-OVERWRITE ASSEMBLY ASSIGN ASYNCHRONOUS ASYNC-REQUEST-COUNT - ASYNC-REQUEST-HANDLE AT ATTACH ATTACH-DATA-SOURCE ATTACHED-PAIRLIST ATTACHMENT ATTR ATTR- - ATTRIBUTE-NAMES ATTRIBUTE-TYPE ATTR-S ATTR-SP ATTR-SPA ATTR-SPAC ATTR-SPACE AUDIT-CONTROL - AUDIT-ENABLED AUDIT-EVENT-CONTEXT AUDIT-POLICY AUTHENTICATION-FAILED AUTHORIZATION AUTO-COMP - AUTO-COMPL AUTO-COMPLE AUTO-COMPLET AUTO-COMPLETI AUTO-COMPLETIO AUTO-COMPLETION AUTO-DELETE - AUTO-DELETE-XML AUTO-ENDKEY AUTO-END-KEY AUTO-GO AUTO-IND AUTO-INDE AUTO-INDEN AUTO-INDENT AUTOMATIC - AUTO-RESIZE AUTO-RET AUTO-RETU AUTO-RETUR AUTO-RETURN AUTO-SYNCHRONIZE AUTO-VAL AUTO-VALI AUTO-VALID - AUTO-VALIDA AUTO-VALIDAT AUTO-VALIDATE AUTO-Z AUTO-ZA AUTO-ZAP AVAIL AVAILA AVAILAB AVAILABL - AVAILABLE AVAILABLE-FORMATS AVE AVER AVERA AVERAG AVERAGE AVG BACK BACKG BACKGR BACKGRO BACKGROU - BACKGROUN BACKGROUND BACKSPACE BACK-TAB BACKWARD BACKWARDS BASE64 BASE64-DECODE BASE64-ENCODE - BASE-ADE BASE-KEY BASIC-LOGGING BATCH BATCH-MODE BATCH-SIZE BEFORE-BUFFER BEFORE-H BEFORE-HI - BEFORE-HID BEFORE-HIDE BEFORE-ROWID BEFORE-TABLE BEGIN-EVENT-GROUP BEGINS BELL BETWEEN BGC BGCO - BGCOL BGCOLO BGCOLOR BIG-ENDIAN BINARY BIND BIND-WHERE BLANK BLOB BLOCK BLOCK-ITERATION-DISPLAY - BLOCK-LEV BLOCK-LEVE BLOCK-LEVEL BORDER-B BORDER-BO BORDER-BOT BORDER-BOTT BORDER-BOTTO - BORDER-BOTTOM BORDER-BOTTOM-C BORDER-BOTTOM-CH BORDER-BOTTOM-CHA BORDER-BOTTOM-CHAR - BORDER-BOTTOM-CHARS BORDER-BOTTOM-P BORDER-BOTTOM-PI BORDER-BOTTOM-PIX BORDER-BOTTOM-PIXE - BORDER-BOTTOM-PIXEL BORDER-BOTTOM-PIXELS BORDER-L BORDER-LE BORDER-LEF BORDER-LEFT BORDER-LEFT-C - BORDER-LEFT-CH BORDER-LEFT-CHA BORDER-LEFT-CHAR BORDER-LEFT-CHARS BORDER-LEFT-P BORDER-LEFT-PI - BORDER-LEFT-PIX BORDER-LEFT-PIXE BORDER-LEFT-PIXEL BORDER-LEFT-PIXELS BORDER-R BORDER-RI BORDER-RIG - BORDER-RIGH BORDER-RIGHT BORDER-RIGHT-C BORDER-RIGHT-CH BORDER-RIGHT-CHA BORDER-RIGHT-CHAR - BORDER-RIGHT-CHARS BORDER-RIGHT-P BORDER-RIGHT-PI BORDER-RIGHT-PIX BORDER-RIGHT-PIXE - BORDER-RIGHT-PIXEL BORDER-RIGHT-PIXELS BORDER-T BORDER-TO BORDER-TOP BORDER-TOP-C BORDER-TOP-CH - BORDER-TOP-CHA BORDER-TOP-CHAR BORDER-TOP-CHARS BORDER-TOP-P BORDER-TOP-PI BORDER-TOP-PIX - BORDER-TOP-PIXE BORDER-TOP-PIXEL BORDER-TOP-PIXELS BOTH BOTTOM BOTTOM-COLUMN BOX BOX-SELECT - BOX-SELECTA BOX-SELECTAB BOX-SELECTABL BOX-SELECTABLE BREAK BREAK-LINE BROWSE BROWSE-COLUMN-DATA-TYPES - BROWSE-COLUMN-FORMATS BROWSE-COLUMN-LABELS BROWSE-HEADER BTOS BUFFER BUFFER-CHARS BUFFER-COMP - BUFFER-COMPA BUFFER-COMPAR BUFFER-COMPARE BUFFER-COPY BUFFER-CREATE BUFFER-DELETE BUFFER-FIELD - BUFFER-GROUP-ID BUFFER-GROUP-NAME BUFFER-HANDLE BUFFER-LINES BUFFER-N BUFFER-NA BUFFER-NAM - BUFFER-NAME BUFFER-PARTITION-ID BUFFER-RELEAS BUFFER-RELEASE BUFFER-TENANT-ID BUFFER-TENANT-NAME - BUFFER-VALIDATE BUFFER-VALUE BUTTON BUTTONS BY BY-POINTER BY-REFERENCE BYTE BYTES-READ BYTES-WRITTEN - BY-VALUE BY-VARIANT-POINT BY-VARIANT-POINTE BY-VARIANT-POINTER CACHE CACHE-SIZE CALL CALL-NAME - CALL-TYPE CANCEL-BREAK CANCEL-BUTTON CANCELLED CANCEL-PICK CANCEL-REQUESTS CANCEL-REQUESTS-AFTER - CAN-CREA CAN-CREAT CAN-CREATE CAN-DELE CAN-DELET CAN-DELETE CAN-DO CAN-DO-DOMAIN-SUPPORT CAN-FIND - CAN-QUERY CAN-READ CAN-SET CAN-WRIT CAN-WRITE CAPS CAREFUL-PAINT CASE CASE-SEN CASE-SENS CASE-SENSI - CASE-SENSIT CASE-SENSITI CASE-SENSITIV CASE-SENSITIVE CAST CATCH CDECL CENTER CENTERE CENTERED - CHAINED CHAR CHARA CHARAC CHARACT CHARACTE CHARACTER CHARACTER_LENGTH CHARSET CHECK CHECKED - CHECK-MEM-STOMP CHILD-BUFFER CHILD-NUM CHOICES CHOOSE CHR CLASS CLASS-TYPE CLEAR CLEAR-APPL-CONTEXT - CLEAR-LOG CLEAR-SELECT CLEAR-SELECTI CLEAR-SELECTIO CLEAR-SELECTION CLEAR-SORT-ARROW - CLEAR-SORT-ARROWS CLIENT-CONNECTION-ID CLIENT-PRINCIPAL CLIENT-TTY CLIENT-TYPE CLIENT-WORKSTATION - CLIPBOARD CLOB CLONE-NODE CLOSE CLOSE-LOG CODE CODEBASE-LOCATOR CODEPAGE CODEPAGE-CONVERT COL - COLLATE COL-OF COLON COLON-ALIGN COLON-ALIGNE COLON-ALIGNED COLOR COLOR-TABLE COLUMN COLUMN-BGC - COLUMN-BGCO COLUMN-BGCOL COLUMN-BGCOLO COLUMN-BGCOLOR COLUMN-CODEPAGE COLUMN-DCOLOR COLUMN-FGC - COLUMN-FGCO COLUMN-FGCOL COLUMN-FGCOLO COLUMN-FGCOLOR COLUMN-FONT COLUMN-LAB COLUMN-LABE - COLUMN-LABEL COLUMN-LABEL-BGC COLUMN-LABEL-BGCO COLUMN-LABEL-BGCOL COLUMN-LABEL-BGCOLO - COLUMN-LABEL-BGCOLOR COLUMN-LABEL-DCOLOR COLUMN-LABEL-FGC COLUMN-LABEL-FGCO COLUMN-LABEL-FGCOL - COLUMN-LABEL-FGCOLO COLUMN-LABEL-FGCOLOR COLUMN-LABEL-FONT COLUMN-LABEL-HEIGHT-C - COLUMN-LABEL-HEIGHT-CH COLUMN-LABEL-HEIGHT-CHA COLUMN-LABEL-HEIGHT-CHAR COLUMN-LABEL-HEIGHT-CHARS - COLUMN-LABEL-HEIGHT-P COLUMN-LABEL-HEIGHT-PI COLUMN-LABEL-HEIGHT-PIX COLUMN-LABEL-HEIGHT-PIXE - COLUMN-LABEL-HEIGHT-PIXEL COLUMN-LABEL-HEIGHT-PIXELS COLUMN-MOVABLE COLUMN-OF COLUMN-PFC COLUMN-PFCO - COLUMN-PFCOL COLUMN-PFCOLO COLUMN-PFCOLOR COLUMN-READ-ONLY COLUMN-RESIZABLE COLUMNS COLUMN-SC - COLUMN-SCR COLUMN-SCRO COLUMN-SCROL COLUMN-SCROLL COLUMN-SCROLLI COLUMN-SCROLLIN COLUMN-SCROLLING - COMBO-BOX COM-HANDLE COMMAND COMPARE COMPARES COMPILE COMPILER COMPLETE COMPONENT-HANDLE - COMPONENT-SELF COM-SELF CONFIG-NAME CONNECT CONNECTED CONSTRAINED CONSTRUCTOR CONTAINER-EVENT - CONTAINS CONTENTS CONTEXT CONTEXT-HELP CONTEXT-HELP-FILE CONTEXT-HELP-ID CONTEXT-POP CONTEXT-POPU - CONTEXT-POPUP CONTROL CONTROL-BOX CONTROL-CONT CONTROL-CONTA CONTROL-CONTAI CONTROL-CONTAIN - CONTROL-CONTAINE CONTROL-CONTAINER CONTROL-FRAM CONTROL-FRAME CONVERT CONVERT-3D CONVERT-3D- - CONVERT-3D-C CONVERT-3D-CO CONVERT-3D-COL CONVERT-3D-COLO CONVERT-3D-COLOR CONVERT-3D-COLORS - CONVERT-TO-OFFS CONVERT-TO-OFFSE CONVERT-TO-OFFSET COPY COPY-DATASET COPY-LOB COPY-SAX-ATTRIBUTES - COPY-TEMP-TABLE COUNT COUNT-OF COVERAGE CPCASE CPCOLL CPINT CPINTE CPINTER CPINTERN CPINTERNA - CPINTERNAL CPLOG CPPRINT CPRCODEIN CPRCODEOUT CPSTREAM CPTERM CRC-VAL CRC-VALU CRC-VALUE CREATE - CREATE-LIKE CREATE-LIKE-SEQUENTIAL CREATE-NODE CREATE-NODE-NAMESPACE CREATE-ON-ADD - CREATE-RESULT-LIST-ENTRY CREATE-TEST-FILE CTOS CURRENT CURRENT_DATE CURRENT-CHANGED CURRENT-COLUMN - CURRENT-ENV CURRENT-ENVI CURRENT-ENVIR CURRENT-ENVIRO CURRENT-ENVIRON CURRENT-ENVIRONM - CURRENT-ENVIRONME CURRENT-ENVIRONMEN CURRENT-ENVIRONMENT CURRENT-ITERATION CURRENT-LANG - CURRENT-LANGU CURRENT-LANGUA CURRENT-LANGUAG CURRENT-LANGUAGE CURRENT-QUERY CURRENT-REQUEST-INFO - CURRENT-RESPONSE-INFO CURRENT-RESULT-ROW CURRENT-ROW-MODIFIED CURRENT-VALUE CURRENT-WINDOW CURS - CURSO CURSOR CURSOR-CHAR CURSOR-DOWN CURSOR-LEFT CURSOR-LINE CURSOR-OFFSET CURSOR-RIGHT CURSOR-UP - CUT DATA-B DATABASE DATA-BI DATA-BIN DATA-BIND DATA-ENTRY-RET DATA-ENTRY-RETU DATA-ENTRY-RETUR - DATA-ENTRY-RETURN DATA-REFRESH-LINE DATA-REFRESH-PAGE DATA-REL DATA-RELA DATA-RELAT DATA-RELATI - DATA-RELATIO DATA-RELATION DATASERVERS DATASET DATASET-HANDLE DATA-SOURCE DATA-SOURCE-COMPLETE-MAP - DATA-SOURCE-MODIFIED DATA-SOURCE-ROWID DATA-T DATA-TY DATA-TYP DATA-TYPE DATE DATE-F DATE-FO - DATE-FOR DATE-FORM DATE-FORMA DATE-FORMAT DATETIME DATETIME-TZ DAY DBCODEPAGE DBCOLLATION DB-CONTEXT - DB-LIST DBNAME DBPARAM DB-REFERENCES DB-REMOTE-HOST DBREST DBRESTR DBRESTRI DBRESTRIC DBRESTRICT - DBRESTRICTI DBRESTRICTIO DBRESTRICTION DBRESTRICTIONS DBTASKID DBTYPE DBVERS DBVERSI DBVERSIO - DBVERSION DCOLOR DDE DDE-ERROR DDE-I DDE-ID DDE-ITEM DDE-NAME DDE-NOTIFY DDE-TOPIC DEBLANK DEBU - DEBUG DEBUG-ALERT DEBUGGER DEBUG-LIST DEBUG-SET-TENANT DEC DECI DECIM DECIMA DECIMAL DECIMALS - DECLARE DECLARE-NAMESPACE DECRYPT DEF DEFAULT DEFAULT-ACTION DEFAULT-BUFFER-HANDLE DEFAULT-BUT - DEFAULT-BUTT DEFAULT-BUTTO DEFAULT-BUTTON DEFAULT-COMMIT DEFAULT-EX DEFAULT-EXT DEFAULT-EXTE - DEFAULT-EXTEN DEFAULT-EXTENS DEFAULT-EXTENSI DEFAULT-EXTENSIO DEFAULT-EXTENSION DEFAULT-NOXL - DEFAULT-NOXLA DEFAULT-NOXLAT DEFAULT-NOXLATE DEFAULT-POP-UP DEFAULT-STRING DEFAULT-VALUE - DEFAULT-WINDOW DEFAUT-B DEFER-LOB-FETCH DEFI DEFIN DEFINE DEFINED DEFINE-USER-EVENT-MANAGER DEL - DELEGATE DELETE DELETE-CHAR DELETE-CHARACTER DELETE-COLUMN DELETE-CURRENT-ROW DELETE-END-LINE - DELETE-FIELD DELETE-HEADER-ENTRY DELETE-LINE DELETE-NODE DELETE-RESULT-LIST-ENTRY - DELETE-SELECTED-ROW DELETE-SELECTED-ROWS DELETE-WORD DELIMITER DESC DESCE DESCEN DESCEND DESCENDI - DESCENDIN DESCENDING DESCRIPT DESCRIPTI DESCRIPTIO DESCRIPTION DESELECT DESELECT-EXTEND - DESELECT-FOCUSED-ROW DESELECTION DESELECTION-EXTEND DESELECT-ROWS DESELECT-SELECTED-ROW DESTRUCTOR - DETACH DETACH-DATA-SOURCE DIALOG-BOX DIALOG-HELP DICT DICTI DICTIO DICTION DICTIONA DICTIONAR - DICTIONARY DIR DIRECTORY DISABLE DISABLE-AUTO-ZAP DISABLE-CONNECTIONS DISABLED DISABLE-DUMP-TRIGGERS - DISABLE-LOAD-TRIGGERS DISCON DISCONN DISCONNE DISCONNEC DISCONNECT DISMISS-MENU DISP DISPL DISPLA - DISPLAY DISPLAY-MESSAGE DISPLAY-T DISPLAY-TIMEZONE DISPLAY-TY DISPLAY-TYP DISPLAY-TYPE DISTINCT - DLL-CALL-TYPE DO DOMAIN-DESCRIPTION DOMAIN-NAME DOMAIN-TYPE DOS DOS-END DOTNET-CLR-LOADED DOUBLE - DOWN DRAG-ENABLED DROP DROP-DOWN DROP-DOWN-LIST DROP-FILE-NOTIFY DROP-TARGET DS-CLOSE-CURSOR - DSLOG-MANAGER DUMP DUMP-LOGGING-NOW DYNAMIC DYNAMIC-CAST DYNAMIC-CURRENT-VALUE DYNAMIC-ENUM - DYNAMIC-FUNC DYNAMIC-FUNCT DYNAMIC-FUNCTI DYNAMIC-FUNCTIO DYNAMIC-FUNCTION DYNAMIC-INVOKE - DYNAMIC-NEW DYNAMIC-NEXT-VALUE DYNAMIC-PROPERTY EACH ECHO EDGE EDGE-C EDGE-CH EDGE-CHA EDGE-CHAR - EDGE-CHARS EDGE-P EDGE-PI EDGE-PIX EDGE-PIXE EDGE-PIXEL EDGE-PIXELS EDIT-CAN-PASTE EDIT-CAN-UNDO - EDIT-CLEAR EDIT-COPY EDIT-CUT EDITING EDITOR EDITOR-BACKTAB EDITOR-TAB EDIT-PASTE EDIT-UNDO ELSE - EMPTY EMPTY-DATASET EMPTY-SELECTION EMPTY-TEMP-TABLE ENABLE ENABLE-CONNECTIONS ENABLED - ENABLED-FIELDS ENCODE ENCODE-DOMAIN-ACCESS-CODE ENCODING ENCRYPT ENCRYPT-AUDIT-MAC-KEY - ENCRYPTION-SALT END END-BOX-SELECTION END-DOCUMENT END-ELEMENT END-ERROR END-EVENT-GROUP - END-FILE-DROP ENDKEY END-KEY END-MOVE END-RESIZE END-ROW-RESIZE END-SEARCH END-USER-PROMPT ENTERED - ENTER-MENUBAR ENTITY-EXPANSION-LIMIT ENTRY ENTRY-TYPES-LIST ENUM EQ ERROR ERROR-COL ERROR-COLU - ERROR-COLUM ERROR-COLUMN ERROR-OBJECT ERROR-OBJECT-DETAIL ERROR-ROW ERROR-STACK-TRACE ERROR-STAT - ERROR-STATU ERROR-STATUS ERROR-STRING ESCAPE ETIME EVENT EVENT-GROUP-ID EVENT-PROCEDURE - EVENT-PROCEDURE-CONTEXT EVENTS EVENT-T EVENT-TY EVENT-TYP EVENT-TYPE EXCEPT EXCLUSIVE EXCLUSIVE-ID - EXCLUSIVE-L EXCLUSIVE-LO EXCLUSIVE-LOC EXCLUSIVE-LOCK EXCLUSIVE-WEB EXCLUSIVE-WEB- EXCLUSIVE-WEB-U - EXCLUSIVE-WEB-US EXCLUSIVE-WEB-USE EXCLUSIVE-WEB-USER EXECUTE EXECUTION-LOG EXISTS EXIT EXIT-CODE - EXP EXPAND EXPANDABLE EXPIRE EXPLICIT EXPORT EXPORT-PRINCIPAL EXTENDED EXTENT EXTERNAL EXTRACT FALSE - FALSE-LEAKS FETCH FETCH-SELECTED-ROW FGC FGCO FGCOL FGCOLO FGCOLOR FIELD FIELDS FILE FILE-ACCESS-D - FILE-ACCESS-DA FILE-ACCESS-DAT FILE-ACCESS-DATE FILE-ACCESS-T FILE-ACCESS-TI FILE-ACCESS-TIM - FILE-ACCESS-TIME FILE-CREATE-D FILE-CREATE-DA FILE-CREATE-DAT FILE-CREATE-DATE FILE-CREATE-T - FILE-CREATE-TI FILE-CREATE-TIM FILE-CREATE-TIME FILE-INFO FILE-INFOR FILE-INFORM FILE-INFORMA - FILE-INFORMAT FILE-INFORMATI FILE-INFORMATIO FILE-INFORMATION FILE-MOD-D FILE-MOD-DA FILE-MOD-DAT - FILE-MOD-DATE FILE-MOD-T FILE-MOD-TI FILE-MOD-TIM FILE-MOD-TIME FILENAME FILE-NAME FILE-OFF - FILE-OFFS FILE-OFFSE FILE-OFFSET FILE-SIZE FILE-TYPE FILL FILLED FILL-IN FILL-MODE FILL-WHERE-STRING - FILTERS FINAL FINALLY FIND FIND-BY-ROWID FIND-CASE-SENSITIVE FIND-CURRENT FINDER FIND-FIRST - FIND-GLOBAL FIND-LAST FIND-NEXT FIND-NEXT-OCCURRENCE FIND-PREVIOUS FIND-PREV-OCCURRENCE FIND-SELECT - FIND-UNIQUE FIND-WRAP-AROUND FIREHOSE-CURSOR FIRST FIRST-ASYNC FIRST-ASYNC- FIRST-ASYNCH-REQUEST - FIRST-ASYNC-R FIRST-ASYNC-RE FIRST-ASYNC-REQ FIRST-ASYNC-REQU FIRST-ASYNC-REQUE FIRST-ASYNC-REQUES - FIRST-ASYNC-REQUEST FIRST-BUFFER FIRST-CHILD FIRST-COLUMN FIRST-DATASET FIRST-DATA-SOURCE FIRST-FORM - FIRST-OBJECT FIRST-OF FIRST-PROC FIRST-PROCE FIRST-PROCED FIRST-PROCEDU FIRST-PROCEDUR - FIRST-PROCEDURE FIRST-QUERY FIRST-SERV FIRST-SERVE FIRST-SERVER FIRST-SERVER-SOCKET FIRST-SOCKET - FIRST-TAB-I FIRST-TAB-IT FIRST-TAB-ITE FIRST-TAB-ITEM FIT-LAST-COLUMN FIX-CODEPAGE FIXED-ONLY FLAGS - FLAT-BUTTON FLOAT FOCUS FOCUSED-ROW FOCUSED-ROW-SELECTED FOCUS-IN FONT FONT-TABLE FOR FORCE-FILE - FORE FOREG FOREGR FOREGRO FOREGROU FOREGROUN FOREGROUND FOREIGN-KEY-HIDDEN FORM FORMA FORMAT - FORMATTE FORMATTED FORM-INPUT FORM-LONG-INPUT FORWARD FORWARD-ONLY FORWARDS FRAGMEN FRAGMENT FRAM - FRAME FRAME-COL FRAME-DB FRAME-DOWN FRAME-FIELD FRAME-FILE FRAME-INDE FRAME-INDEX FRAME-LINE - FRAME-NAME FRAME-ROW FRAME-SPA FRAME-SPAC FRAME-SPACI FRAME-SPACIN FRAME-SPACING FRAME-VAL - FRAME-VALU FRAME-VALUE FRAME-X FRAME-Y FREQUENCY FROM FROM-C FROM-CH FROM-CHA FROM-CHAR FROM-CHARS - FROM-CUR FROM-CURR FROM-CURRE FROM-CURREN FROM-CURRENT FROMNOREORDER FROM-P FROM-PI FROM-PIX - FROM-PIXE FROM-PIXEL FROM-PIXELS FULL-HEIGHT FULL-HEIGHT-C FULL-HEIGHT-CH FULL-HEIGHT-CHA - FULL-HEIGHT-CHAR FULL-HEIGHT-CHARS FULL-HEIGHT-P FULL-HEIGHT-PI FULL-HEIGHT-PIX FULL-HEIGHT-PIXE - FULL-HEIGHT-PIXEL FULL-HEIGHT-PIXELS FULL-PATHN FULL-PATHNA FULL-PATHNAM FULL-PATHNAME FULL-WIDTH - FULL-WIDTH- FULL-WIDTH-C FULL-WIDTH-CH FULL-WIDTH-CHA FULL-WIDTH-CHAR FULL-WIDTH-CHARS FULL-WIDTH-P - FULL-WIDTH-PI FULL-WIDTH-PIX FULL-WIDTH-PIXE FULL-WIDTH-PIXEL FULL-WIDTH-PIXELS FUNCTION - FUNCTION-CALL-TYPE GATEWAY GATEWAYS GE GENERATE-MD5 GENERATE-PBE-KEY GENERATE-PBE-SALT - GENERATE-RANDOM-KEY GENERATE-UUID GET GET-ATTR-CALL-TYPE GET-ATTRIBUTE GET-ATTRIBUTE-NODE - GET-BINARY-DATA GET-BITS GET-BLUE GET-BLUE- GET-BLUE-V GET-BLUE-VA GET-BLUE-VAL GET-BLUE-VALU - GET-BLUE-VALUE GET-BROWSE-COL GET-BROWSE-COLU GET-BROWSE-COLUM GET-BROWSE-COLUMN GET-BUFFER-HANDLE - GETBYTE GET-BYTE GET-BYTE-ORDER GET-BYTES GET-BYTES-AVAILABLE GET-CALLBACK-PROC-CONTEXT - GET-CALLBACK-PROC-NAME GET-CGI-LIST GET-CGI-LONG-VALUE GET-CGI-VALUE GET-CHANGES GET-CHILD - GET-CHILD-REL GET-CHILD-RELA GET-CHILD-RELAT GET-CHILD-RELATI GET-CHILD-RELATIO GET-CHILD-RELATION - GET-CLASS GET-CLIENT GET-CODEPAGE GET-CODEPAGES GET-COLL GET-COLLA GET-COLLAT GET-COLLATI - GET-COLLATIO GET-COLLATION GET-COLLATIONS GET-COLUMN GET-CONFIG-VALUE GET-CURR GET-CURRE GET-CURREN - GET-CURRENT GET-DATASET-BUFFER GET-DB-CLIENT GET-DIR GET-DOCUMENT-ELEMENT GET-DOUBLE - GET-DROPPED-FILE GET-DYNAMIC GET-EFFECTIVE-TENANT-ID GET-EFFECTIVE-TENANT-NAME GET-ERROR-COLUMN - GET-ERROR-ROW GET-FILE GET-FILE-NAME GET-FILE-OFFSE GET-FILE-OFFSET GET-FIRS GET-FIRST GET-FLOAT - GET-GREEN GET-GREEN- GET-GREEN-V GET-GREEN-VA GET-GREEN-VAL GET-GREEN-VALU GET-GREEN-VALUE - GET-HEADER-ENTR GET-HEADER-ENTRY GET-INDEX-BY-NAMESPACE-NAME GET-INDEX-BY-QNAME GET-INT64 - GET-ITERATION GET-KEY-VAL GET-KEY-VALU GET-KEY-VALUE GET-LAST GET-LOCALNAME-BY-INDEX GET-LONG - GET-MESSAGE GET-MESSAGE-TYPE GET-NEXT GET-NODE GET-NUMBER GET-PARENT GET-POINTER-VALUE GET-PREV - GET-PRINTERS GET-PROPERTY GET-QNAME-BY-INDEX GET-RED GET-RED- GET-RED-V GET-RED-VA GET-RED-VAL - GET-RED-VALU GET-RED-VALUE GET-REL GET-RELA GET-RELAT GET-RELATI GET-RELATIO GET-RELATION - GET-REPOSITIONED-ROW GET-RGB GET-RGB- GET-RGB-V GET-RGB-VA GET-RGB-VAL GET-RGB-VALU GET-RGB-VALUE - GET-ROW GET-SAFE-USER GET-SELECTED GET-SELECTED- GET-SELECTED-W GET-SELECTED-WI GET-SELECTED-WID - GET-SELECTED-WIDG GET-SELECTED-WIDGE GET-SELECTED-WIDGET GET-SERIALIZED GET-SHORT GET-SIGNATURE - GET-SIZE GET-SOCKET-OPTION GET-SOURCE-BUFFER GET-STRING GET-TAB-ITEM GET-TEXT-HEIGHT - GET-TEXT-HEIGHT-C GET-TEXT-HEIGHT-CH GET-TEXT-HEIGHT-CHA GET-TEXT-HEIGHT-CHAR GET-TEXT-HEIGHT-CHARS - GET-TEXT-HEIGHT-P GET-TEXT-HEIGHT-PI GET-TEXT-HEIGHT-PIX GET-TEXT-HEIGHT-PIXE GET-TEXT-HEIGHT-PIXEL - GET-TEXT-HEIGHT-PIXELS GET-TEXT-WIDTH GET-TEXT-WIDTH-C GET-TEXT-WIDTH-CH GET-TEXT-WIDTH-CHA - GET-TEXT-WIDTH-CHAR GET-TEXT-WIDTH-CHARS GET-TEXT-WIDTH-P GET-TEXT-WIDTH-PI GET-TEXT-WIDTH-PIX - GET-TEXT-WIDTH-PIXE GET-TEXT-WIDTH-PIXEL GET-TEXT-WIDTH-PIXELS GET-TOP-BUFFER GET-TYPE-BY-INDEX - GET-TYPE-BY-NAMESPACE-NAME GET-TYPE-BY-QNAME GET-UNSIGNED-LONG GET-UNSIGNED-SHORT GET-URI-BY-INDEX - GET-VALUE-BY-INDEX GET-VALUE-BY-NAMESPACE-NAME GET-VALUE-BY-QNAME GET-WAIT GET-WAIT- GET-WAIT-S - GET-WAIT-ST GET-WAIT-STA GET-WAIT-STAT GET-WAIT-STATE GLOBAL GO GO-ON GO-PEND GO-PENDI GO-PENDIN - GO-PENDING GOTO GRANT GRANT-ARCHIVE GRAPHIC-E GRAPHIC-ED GRAPHIC-EDG GRAPHIC-EDGE GRAYED - GRID-FACTOR-H GRID-FACTOR-HO GRID-FACTOR-HOR GRID-FACTOR-HORI GRID-FACTOR-HORIZ GRID-FACTOR-HORIZO - GRID-FACTOR-HORIZON GRID-FACTOR-HORIZONT GRID-FACTOR-HORIZONTA GRID-FACTOR-HORIZONTAL GRID-FACTOR-V - GRID-FACTOR-VE GRID-FACTOR-VER GRID-FACTOR-VERT GRID-FACTOR-VERTI GRID-FACTOR-VERTIC - GRID-FACTOR-VERTICA GRID-FACTOR-VERTICAL GRID-SET GRID-SNAP GRID-UNIT-HEIGHT GRID-UNIT-HEIGHT-C - GRID-UNIT-HEIGHT-CH GRID-UNIT-HEIGHT-CHA GRID-UNIT-HEIGHT-CHAR GRID-UNIT-HEIGHT-CHARS - GRID-UNIT-HEIGHT-P GRID-UNIT-HEIGHT-PI GRID-UNIT-HEIGHT-PIX GRID-UNIT-HEIGHT-PIXE - GRID-UNIT-HEIGHT-PIXEL GRID-UNIT-HEIGHT-PIXELS GRID-UNIT-WIDTH GRID-UNIT-WIDTH-C GRID-UNIT-WIDTH-CH - GRID-UNIT-WIDTH-CHA GRID-UNIT-WIDTH-CHAR GRID-UNIT-WIDTH-CHARS GRID-UNIT-WIDTH-P GRID-UNIT-WIDTH-PI - GRID-UNIT-WIDTH-PIX GRID-UNIT-WIDTH-PIXE GRID-UNIT-WIDTH-PIXEL GRID-UNIT-WIDTH-PIXELS GRID-VISIBLE - GROUP GROUP-BOX GT GUID HANDLE HANDLER HAS-LOBS HAS-RECORDS HAVING HEADER HEIGHT HEIGHT-C HEIGHT-CH - HEIGHT-CHA HEIGHT-CHAR HEIGHT-CHARS HEIGHT-P HEIGHT-PI HEIGHT-PIX HEIGHT-PIXE HEIGHT-PIXEL - HEIGHT-PIXELS HELP HELP-CON HELP-CONT HELP-CONTE HELP-CONTEX HELP-CONTEXT HELPFILE-N HELPFILE-NA - HELPFILE-NAM HELPFILE-NAME HELP-TOPIC HEX-DECODE HEX-ENCODE HIDDEN HIDE HINT HOME HORI HORIZ - HORIZ-END HORIZ-HOME HORIZO HORIZON HORIZONT HORIZONTA HORIZONTAL HORIZ-SCROLL-DRAG HOST-BYTE-ORDER - HTML-CHARSET HTML-END-OF-LINE HTML-END-OF-PAGE HTML-FRAME-BEGIN HTML-FRAME-END HTML-HEADER-BEGIN - HTML-HEADER-END HTML-TITLE-BEGIN HTML-TITLE-END HWND ICFPARAM ICFPARAME ICFPARAMET ICFPARAMETE - ICFPARAMETER ICON IF IGNORE-CURRENT-MOD IGNORE-CURRENT-MODI IGNORE-CURRENT-MODIF - IGNORE-CURRENT-MODIFI IGNORE-CURRENT-MODIFIE IGNORE-CURRENT-MODIFIED IMAGE IMAGE-DOWN - IMAGE-INSENSITIVE IMAGE-SIZE IMAGE-SIZE-C IMAGE-SIZE-CH IMAGE-SIZE-CHA IMAGE-SIZE-CHAR - IMAGE-SIZE-CHARS IMAGE-SIZE-P IMAGE-SIZE-PI IMAGE-SIZE-PIX IMAGE-SIZE-PIXE IMAGE-SIZE-PIXEL - IMAGE-SIZE-PIXELS IMAGE-UP IMMEDIATE-DISPLAY IMPLEMENTS IMPORT IMPORT-NODE IMPORT-PRINCIPAL IN - INCREMENT-EXCLUSIVE-ID INDEX INDEXED-REPOSITION INDEX-HINT INDEX-INFO INDEX-INFOR INDEX-INFORM - INDEX-INFORMA INDEX-INFORMAT INDEX-INFORMATI INDEX-INFORMATIO INDEX-INFORMATION INDICATOR INFO INFOR - INFORM INFORMA INFORMAT INFORMATI INFORMATIO INFORMATION IN-HANDLE INHERIT-BGC INHERIT-BGCO - INHERIT-BGCOL INHERIT-BGCOLO INHERIT-BGCOLOR INHERIT-COLOR-MODE INHERIT-FGC INHERIT-FGCO - INHERIT-FGCOL INHERIT-FGCOLO INHERIT-FGCOLOR INHERITS INIT INITIAL INITIAL-DIR INITIAL-FILTER - INITIALIZE INITIALIZE-DOCUMENT-TYPE INITIATE INNER INNER-CHARS INNER-LINES INPUT INPUT-O INPUT-OU - INPUT-OUT INPUT-OUTP INPUT-OUTPU INPUT-OUTPUT INPUT-VALUE INSERT INSERT-ATTRIBUTE INSERT-B INSERT-BA - INSERT-BAC INSERT-BACK INSERT-BACKT INSERT-BACKTA INSERT-BACKTAB INSERT-BEFORE INSERT-COLUMN - INSERT-FIELD INSERT-FIELD-DATA INSERT-FIELD-LABEL INSERT-FILE INSERT-MODE INSERT-ROW INSERT-STRING - INSERT-T INSERT-TA INSERT-TAB INSTANTIATING-PROCEDURE INT INT64 INTE INTEG INTEGE INTEGER INTERFACE - INTERNAL-ENTRIES INTERVAL INTO INVOKE IS IS-ATTR IS-ATTR- IS-ATTR-S IS-ATTR-SP IS-ATTR-SPA - IS-ATTR-SPAC IS-ATTR-SPACE IS-CLAS IS-CLASS IS-CODEPAGE-FIXED IS-COLUMN-CODEPAGE IS-DB-MULTI-TENANT - IS-JSON IS-LEAD-BYTE IS-MULTI-TENANT ISO-DATE IS-OPEN IS-PARAMETER-SET IS-PARTITIONE IS-PARTITIONED - IS-ROW-SELECTED IS-SELECTED IS-XML ITEM ITEMS-PER-ROW ITERATION-CHANGED JOIN JOIN-BY-SQLDB - JOIN-ON-SELECT KBLABEL KEEP-CONNECTION-OPEN KEEP-FRAME-Z KEEP-FRAME-Z- KEEP-FRAME-Z-O - KEEP-FRAME-Z-OR KEEP-FRAME-Z-ORD KEEP-FRAME-Z-ORDE KEEP-FRAME-Z-ORDER KEEP-MESSAGES - KEEP-SECURITY-CACHE KEEP-TAB-ORDER KEY KEYCACHE-JOIN KEYCODE KEY-CODE KEYFUNC KEY-FUNC KEYFUNCT - KEY-FUNCT KEYFUNCTI KEY-FUNCTI KEYFUNCTIO KEY-FUNCTIO KEYFUNCTION KEY-FUNCTION KEYLABEL KEY-LABEL - KEYS KEYWORD KEYWORD-ALL LABEL LABEL-BGC LABEL-BGCO LABEL-BGCOL LABEL-BGCOLO LABEL-BGCOLOR LABEL-DC - LABEL-DCO LABEL-DCOL LABEL-DCOLO LABEL-DCOLOR LABEL-FGC LABEL-FGCO LABEL-FGCOL LABEL-FGCOLO - LABEL-FGCOLOR LABEL-FONT LABEL-PFC LABEL-PFCO LABEL-PFCOL LABEL-PFCOLO LABEL-PFCOLOR LABELS - LABELS-HAVE-COLONS LANDSCAPE LANGUAGE LANGUAGES LARGE LARGE-TO-SMALL LAST LAST-ASYNC LAST-ASYNC- - LAST-ASYNCH-REQUEST LAST-ASYNC-R LAST-ASYNC-RE LAST-ASYNC-REQ LAST-ASYNC-REQU LAST-ASYNC-REQUE - LAST-ASYNC-REQUES LAST-ASYNC-REQUEST LAST-BATCH LAST-CHILD LAST-EVEN LAST-EVENT LAST-FORM LASTKEY - LAST-KEY LAST-OBJECT LAST-OF LAST-PROCE LAST-PROCED LAST-PROCEDU LAST-PROCEDUR LAST-PROCEDURE - LAST-SERV LAST-SERVE LAST-SERVER LAST-SERVER-SOCKET LAST-SOCKET LAST-TAB-I LAST-TAB-IT LAST-TAB-ITE - LAST-TAB-ITEM LC LDBNAME LE LEADING LEAK-DETECTION LEAVE LEFT LEFT-ALIGN LEFT-ALIGNE LEFT-ALIGNED - LEFT-END LEFT-TRIM LENGTH LIBRARY LIBRARY-CALLING-CONVENTION LIKE LIKE-SEQUENTIAL LINE LINE-COUNT - LINE-COUNTE LINE-COUNTER LINE-DOWN LINE-LEFT LINE-RIGHT LINE-UP LIST-EVENTS LISTI LISTIN LISTING - LISTINGS LIST-ITEM-PAIRS LIST-ITEMS LIST-PROPERTY-NAMES LIST-QUERY-ATTRS LIST-SET-ATTRS LIST-WIDGETS - LITERAL-QUESTION LITTLE-ENDIAN LOAD LOAD-DOMAINS LOAD-FROM LOAD-ICON LOAD-IMAGE LOAD-IMAGE-DOWN - LOAD-IMAGE-INSENSITIVE LOAD-IMAGE-UP LOAD-MOUSE-P LOAD-MOUSE-PO LOAD-MOUSE-POI LOAD-MOUSE-POIN - LOAD-MOUSE-POINT LOAD-MOUSE-POINTE LOAD-MOUSE-POINTER LOAD-PICTURE LOAD-RESULT-INTO LOAD-SMALL-ICON - LOB-DIR LOCAL-HOST LOCAL-NAME LOCAL-PORT LOCAL-VERSION-INFO LOCATOR-COLUMN-NUMBER - LOCATOR-LINE-NUMBER LOCATOR-PUBLIC-ID LOCATOR-SYSTEM-ID LOCATOR-TYPE LOCKED LOCK-REGISTRATION LOG - LOG-AUDIT-EVENT LOG-ENTRY-TYPES LOGFILE-NAME LOGGING-LEVEL LOGICAL LOG-ID LOGIN-EXPIRATION-TIMESTAMP - LOGIN-HOST LOGIN-STATE LOG-MANAGER LOGOUT LOG-THRESHOLD LONG LONGCH LONGCHA LONGCHAR - LONGCHAR-TO-NODE-VALUE LOOKAHEAD LOOKUP LOWER LT MACHINE-CLASS MAIN-MENU MANDATORY MANUAL-HIGHLIGHT - MAP MARGIN-EXTRA MARGIN-HEIGHT MARGIN-HEIGHT-C MARGIN-HEIGHT-CH MARGIN-HEIGHT-CHA MARGIN-HEIGHT-CHAR - MARGIN-HEIGHT-CHARS MARGIN-HEIGHT-P MARGIN-HEIGHT-PI MARGIN-HEIGHT-PIX MARGIN-HEIGHT-PIXE - MARGIN-HEIGHT-PIXEL MARGIN-HEIGHT-PIXELS MARGIN-WIDTH MARGIN-WIDTH-C MARGIN-WIDTH-CH - MARGIN-WIDTH-CHA MARGIN-WIDTH-CHAR MARGIN-WIDTH-CHARS MARGIN-WIDTH-P MARGIN-WIDTH-PI - MARGIN-WIDTH-PIX MARGIN-WIDTH-PIXE MARGIN-WIDTH-PIXEL MARGIN-WIDTH-PIXELS MARK-NEW MARK-ROW-STATE - MATCHES MAX MAX-BUTTON MAX-CHARS MAX-DATA-GUESS MAX-HEIGHT MAX-HEIGHT-C MAX-HEIGHT-CH MAX-HEIGHT-CHA - MAX-HEIGHT-CHAR MAX-HEIGHT-CHARS MAX-HEIGHT-P MAX-HEIGHT-PI MAX-HEIGHT-PIX MAX-HEIGHT-PIXE - MAX-HEIGHT-PIXEL MAX-HEIGHT-PIXELS MAXIMIZE MAXIMUM MAXIMUM-LEVEL MAX-ROWS MAX-SIZE MAX-VAL MAX-VALU - MAX-VALUE MAX-WIDTH MAX-WIDTH-C MAX-WIDTH-CH MAX-WIDTH-CHA MAX-WIDTH-CHAR MAX-WIDTH-CHARS - MAX-WIDTH-P MAX-WIDTH-PI MAX-WIDTH-PIX MAX-WIDTH-PIXE MAX-WIDTH-PIXEL MAX-WIDTH-PIXELS MD5-DIGEST - MD5-VALUE MEMBER MEMPTR MEMPTR-TO-NODE-VALUE MENU MENUBAR MENU-BAR MENU-DROP MENU-ITEM MENU-K - MENU-KE MENU-KEY MENU-M MENU-MO MENU-MOU MENU-MOUS MENU-MOUSE MERGE-BY-FIELD MERGE-CHANGES - MERGE-ROW-CHANGES MESSAGE MESSAGE-AREA MESSAGE-AREA-FONT MESSAGE-AREA-MSG MESSAGE-DIGEST - MESSAGE-LINE MESSAGE-LINES METHOD MIN MIN-BUTTON MIN-COLUMN-WIDTH-C MIN-COLUMN-WIDTH-CH - MIN-COLUMN-WIDTH-CHA MIN-COLUMN-WIDTH-CHAR MIN-COLUMN-WIDTH-CHARS MIN-COLUMN-WIDTH-P - MIN-COLUMN-WIDTH-PI MIN-COLUMN-WIDTH-PIX MIN-COLUMN-WIDTH-PIXE MIN-COLUMN-WIDTH-PIXEL - MIN-COLUMN-WIDTH-PIXELS MIN-HEIGHT MIN-HEIGHT-C MIN-HEIGHT-CH MIN-HEIGHT-CHA MIN-HEIGHT-CHAR - MIN-HEIGHT-CHARS MIN-HEIGHT-P MIN-HEIGHT-PI MIN-HEIGHT-PIX MIN-HEIGHT-PIXE MIN-HEIGHT-PIXEL - MIN-HEIGHT-PIXELS MINI MINIM MINIMU MINIMUM MIN-SCHEMA-MARSHAL MIN-SCHEMA-MARSHALL MIN-SIZE MIN-VAL - MIN-VALU MIN-VALUE MIN-WIDTH MIN-WIDTH-C MIN-WIDTH-CH MIN-WIDTH-CHA MIN-WIDTH-CHAR MIN-WIDTH-CHARS - MIN-WIDTH-P MIN-WIDTH-PI MIN-WIDTH-PIX MIN-WIDTH-PIXE MIN-WIDTH-PIXEL MIN-WIDTH-PIXELS MOD MODIFIED - MODULO MONTH MOUSE MOUSE-P MOUSE-PO MOUSE-POI MOUSE-POIN MOUSE-POINT MOUSE-POINTE MOUSE-POINTER - MOVABLE MOVE MOVE-AFTER MOVE-AFTER- MOVE-AFTER-T MOVE-AFTER-TA MOVE-AFTER-TAB MOVE-AFTER-TAB- - MOVE-AFTER-TAB-I MOVE-AFTER-TAB-IT MOVE-AFTER-TAB-ITE MOVE-AFTER-TAB-ITEM MOVE-BEFOR MOVE-BEFORE - MOVE-BEFORE- MOVE-BEFORE-T MOVE-BEFORE-TA MOVE-BEFORE-TAB MOVE-BEFORE-TAB- MOVE-BEFORE-TAB-I - MOVE-BEFORE-TAB-IT MOVE-BEFORE-TAB-ITE MOVE-BEFORE-TAB-ITEM MOVE-COL MOVE-COLU MOVE-COLUM - MOVE-COLUMN MOVE-TO-B MOVE-TO-BO MOVE-TO-BOT MOVE-TO-BOTT MOVE-TO-BOTTO MOVE-TO-BOTTOM MOVE-TO-EOF - MOVE-TO-T MOVE-TO-TO MOVE-TO-TOP MPE MTIME MULTI-COMPILE MULTIPLE MULTIPLE-KEY MULTITASKING-INTERVAL - MUST-EXIST MUST-UNDERSTAND NAME NAMESPACE-PREFIX NAMESPACE-URI NATIVE NE NEEDS-APPSERVER-PROMPT - NEEDS-PROMPT NESTED NEW NEW-INSTANCE NEW-LINE NEW-ROW NEXT NEXT-COL NEXT-COLU NEXT-COLUM NEXT-COLUMN - NEXT-ERROR NEXT-FRAME NEXT-PROMPT NEXT-ROWID NEXT-SIBLING NEXT-TAB-I NEXT-TAB-ITE NEXT-TAB-ITEM - NEXT-VALUE NEXT-WORD NO NO-APPLY NO-ARRAY-M NO-ARRAY-ME NO-ARRAY-MES NO-ARRAY-MESS NO-ARRAY-MESSA - NO-ARRAY-MESSAG NO-ARRAY-MESSAGE NO-ASSIGN NO-ATTR NO-ATTR-L NO-ATTR-LI NO-ATTR-LIS NO-ATTR-LIST - NO-ATTR-S NO-ATTR-SP NO-ATTR-SPA NO-ATTR-SPAC NO-ATTR-SPACE NO-AUTO-TRI NO-AUTO-TRIM - NO-AUTO-VALIDATE NO-BIND-WHERE NO-BOX NO-COLUMN-SC NO-COLUMN-SCR NO-COLUMN-SCRO NO-COLUMN-SCROL - NO-COLUMN-SCROLL NO-COLUMN-SCROLLI NO-COLUMN-SCROLLIN NO-COLUMN-SCROLLING NO-CONSOLE NO-CONVERT - NO-CONVERT-3D NO-CONVERT-3D- NO-CONVERT-3D-C NO-CONVERT-3D-CO NO-CONVERT-3D-COL NO-CONVERT-3D-COLO - NO-CONVERT-3D-COLOR NO-CONVERT-3D-COLORS NO-CURRENT-VALUE NO-DEBUG NODE-TYPE NODE-VALUE - NODE-VALUE-TO-LONGCHAR NODE-VALUE-TO-MEMPTR NO-DRAG NO-ECHO NO-EMPTY-SPACE NO-ERROR NO-F NO-FI - NO-FIL NO-FILL NO-FIREHOSE-CURSOR NO-FOCUS NO-HELP NO-HIDE NO-INDEX-HINT NO-INHERIT-BGC - NO-INHERIT-BGCO NO-INHERIT-BGCOL NO-INHERIT-BGCOLO NO-INHERIT-BGCOLOR NO-INHERIT-FGC NO-INHERIT-FGCO - NO-INHERIT-FGCOL NO-INHERIT-FGCOLO NO-INHERIT-FGCOLOR NO-JOIN-BY-SQLDB NO-KEYCACHE-JOIN NO-LABEL - NO-LABELS NO-LOBS NO-LOCK NO-LOOKAHEAD NO-MAP NO-MES NO-MESS NO-MESSA NO-MESSAG NO-MESSAGE - NONAMESPACE-SCHEMA-LOCATION NONE NON-SERIALIZABLE NO-PAUSE NO-PREFE NO-PREFET NO-PREFETC NO-PREFETCH - NO-QUERY-O NO-QUERY-OR NO-QUERY-ORD NO-QUERY-ORDE NO-QUERY-ORDER NO-QUERY-ORDER- NO-QUERY-ORDER-A - NO-QUERY-ORDER-AD NO-QUERY-ORDER-ADD NO-QUERY-ORDER-ADDE NO-QUERY-ORDER-ADDED NO-QUERY-U NO-QUERY-UN - NO-QUERY-UNI NO-QUERY-UNIQ NO-QUERY-UNIQU NO-QUERY-UNIQUE NO-QUERY-UNIQUE- NO-QUERY-UNIQUE-A - NO-QUERY-UNIQUE-AD NO-QUERY-UNIQUE-ADD NO-QUERY-UNIQUE-ADDE NO-QUERY-UNIQUE-ADDED NO-RETURN-VAL - NO-RETURN-VALU NO-RETURN-VALUE NORMALIZE NO-ROW-MARKERS NO-SCHEMA-MARSHAL NO-SCHEMA-MARSHALL - NO-SCROLLBAR-V NO-SCROLLBAR-VE NO-SCROLLBAR-VER NO-SCROLLBAR-VERT NO-SCROLLBAR-VERTI - NO-SCROLLBAR-VERTIC NO-SCROLLBAR-VERTICA NO-SCROLLBAR-VERTICAL NO-SCROLLING NO-SEPARATE-CONNECTION - NO-SEPARATORS NOT NO-TAB NO-TAB- NO-TAB-S NO-TAB-ST NO-TAB-STO NO-TAB-STOP NOT-ACTIVE NO-UND NO-UNDE - NO-UNDER NO-UNDERL NO-UNDERLI NO-UNDERLIN NO-UNDERLINE NO-UNDO NO-VAL NO-VALI NO-VALID NO-VALIDA - NO-VALIDAT NO-VALIDATE NOW NO-WAIT NO-WORD-WRAP NULL NUM-ALI NUM-ALIA NUM-ALIAS NUM-ALIASE - NUM-ALIASES NUM-BUFFERS NUM-BUT NUM-BUTT NUM-BUTTO NUM-BUTTON NUM-BUTTONS NUM-CHILD-RELATIONS - NUM-CHILDREN NUM-COL NUM-COLU NUM-COLUM NUM-COLUMN NUM-COLUMNS NUM-COPIES NUM-DBS NUM-DROPPED-FILES - NUM-ENTRIES NUMERIC NUMERIC-DEC NUMERIC-DECI NUMERIC-DECIM NUMERIC-DECIMA NUMERIC-DECIMAL - NUMERIC-DECIMAL- NUMERIC-DECIMAL-P NUMERIC-DECIMAL-PO NUMERIC-DECIMAL-POI NUMERIC-DECIMAL-POIN - NUMERIC-DECIMAL-POINT NUMERIC-F NUMERIC-FO NUMERIC-FOR NUMERIC-FORM NUMERIC-FORMA NUMERIC-FORMAT - NUMERIC-SEP NUMERIC-SEPA NUMERIC-SEPAR NUMERIC-SEPARA NUMERIC-SEPARAT NUMERIC-SEPARATO - NUMERIC-SEPARATOR NUM-FIELDS NUM-FORMATS NUM-HEADER-ENTRIES NUM-ITEMS NUM-ITERATIONS NUM-LINES - NUM-LOCKED-COL NUM-LOCKED-COLU NUM-LOCKED-COLUM NUM-LOCKED-COLUMN NUM-LOCKED-COLUMNS NUM-LOG-FILES - NUM-MESSAGES NUM-PARAMETERS NUM-REFERENCES NUM-RELATIONS NUM-REPL NUM-REPLA NUM-REPLAC NUM-REPLACE - NUM-REPLACED NUM-RESULTS NUM-SELECTED NUM-SELECTED-ROWS NUM-SELECTED-WIDGETS NUM-SOURCE-BUFFERS - NUM-TABS NUM-TOP-BUFFERS NUM-TO-RETAIN NUM-VISIBLE-COL NUM-VISIBLE-COLU NUM-VISIBLE-COLUM - NUM-VISIBLE-COLUMN NUM-VISIBLE-COLUMNS OBJECT OCTET_LENGTH OCTET-LENGTH OF OFF OFF-END OFF-HOME OK - OK-CANCEL OLD OLE-INVOKE-LOCA OLE-INVOKE-LOCAL OLE-INVOKE-LOCALE OLE-NAMES-LOCA OLE-NAMES-LOCAL - OLE-NAMES-LOCALE ON ON-FRAME ON-FRAME- ON-FRAME-B ON-FRAME-BO ON-FRAME-BOR ON-FRAME-BORD - ON-FRAME-BORDE ON-FRAME-BORDER OPEN OPEN-LINE-ABOVE OPSYS OPTION OPTIONS OPTIONS-FILE OR - ORDERED-JOIN ORDINAL ORIENTATION ORIGIN-HANDLE ORIGIN-ROWID OS2 OS400 OS-APPEND OS-COMMAND OS-COPY - OS-CREATE-DIR OS-DELETE OS-DIR OS-DRIVE OS-DRIVES OS-ERROR OS-GETENV OS-RENAME OTHERWISE OUTER - OUTER-JOIN OUT-OF-DATA OUTPUT OVERLAY OVERRIDE OWNER OWNER-DOCUMENT PACKAGE-PRIVATE - PACKAGE-PROTECTED PAGE PAGE-BOT PAGE-BOTT PAGE-BOTTO PAGE-BOTTOM PAGED PAGE-DOWN PAGE-LEFT PAGE-NUM - PAGE-NUMB PAGE-NUMBE PAGE-NUMBER PAGE-RIGHT PAGE-RIGHT-TEXT PAGE-SIZE PAGE-TOP PAGE-UP PAGE-WID - PAGE-WIDT PAGE-WIDTH PARAM PARAME PARAMET PARAMETE PARAMETER PARENT PARENT-BUFFER - PARENT-FIELDS-AFTER PARENT-FIELDS-BEFORE PARENT-ID-FIELD PARENT-ID-RELATION PARENT-REL PARENT-RELA - PARENT-RELAT PARENT-RELATI PARENT-RELATIO PARENT-RELATION PARENT-WINDOW-CLOSE PARSE-STATUS - PARTIAL-KEY PASCAL PASSWORD-FIELD PASTE PATHNAME PAUSE PBE-HASH-ALG PBE-HASH-ALGO PBE-HASH-ALGOR - PBE-HASH-ALGORI PBE-HASH-ALGORIT PBE-HASH-ALGORITH PBE-HASH-ALGORITHM PBE-KEY-ROUNDS PDBNAME PERF - PERFO PERFOR PERFORM PERFORMA PERFORMAN PERFORMANC PERFORMANCE PERSIST PERSISTE PERSISTEN PERSISTENT - PERSISTENT-CACHE-DISABLED PERSISTENT-PROCEDURE PFC PFCO PFCOL PFCOLO PFCOLOR PICK PICK-AREA - PICK-BOTH PIXELS PIXELS-PER-COL PIXELS-PER-COLU PIXELS-PER-COLUM PIXELS-PER-COLUMN PIXELS-PER-ROW - POPUP-M POPUP-ME POPUP-MEN POPUP-MENU POPUP-O POPUP-ON POPUP-ONL POPUP-ONLY PORTRAIT POSITION - PRECISION PREFER-DATASET PREPARED PREPARE-STRING PREPROC PREPROCE PREPROCES PREPROCESS PRESEL - PRESELE PRESELEC PRESELECT PREV PREV-COL PREV-COLU PREV-COLUM PREV-COLUMN PREV-FRAME PREV-SIBLING - PREV-TAB-I PREV-TAB-IT PREV-TAB-ITE PREV-TAB-ITEM PREV-WORD PRIMARY PRIMARY-PASSPHRASE PRINTER - PRINTER-CONTROL-HANDLE PRINTER-HDC PRINTER-NAME PRINTER-PORT PRINTER-SETUP PRIVATE PRIVATE-D - PRIVATE-DA PRIVATE-DAT PRIVATE-DATA PRIVILEGES PROCE PROCED PROCEDU PROCEDUR PROCEDURE - PROCEDURE-CALL-TYPE PROCEDURE-COMPLETE PROCEDURE-NAME PROCEDURE-TYPE PROCESS PROCESS-ARCHITECTURE - PROC-HA PROC-HAN PROC-HAND PROC-HANDL PROC-HANDLE PROC-ST PROC-STA PROC-STAT PROC-STATU PROC-STATUS - PROC-TEXT PROC-TEXT-BUFFER PROFILE-FILE PROFILER PROFILING PROGRAM-NAME PROGRESS PROGRESS-S - PROGRESS-SO PROGRESS-SOU PROGRESS-SOUR PROGRESS-SOURC PROGRESS-SOURCE PROMPT PROMPT-F PROMPT-FO - PROMPT-FOR PROMSGS PROPATH PROPERTY PROTECTED PROVERS PROVERSI PROVERSIO PROVERSION PROXY - PROXY-PASSWORD PROXY-USERID PUBLIC PUBLIC-ID PUBLISH PUBLISHED-EVENTS PUT PUT-BITS PUTBYTE PUT-BYTE - PUT-BYTES PUT-DOUBLE PUT-FLOAT PUT-INT64 PUT-KEY-VAL PUT-KEY-VALU PUT-KEY-VALUE PUT-LONG PUT-SHORT - PUT-STRING PUT-UNSIGNED-LONG PUT-UNSIGNED-SHORT QUALIFIED-USER-ID QUERY QUERY-CLOSE QUERY-OFF-END - QUERY-OPEN QUERY-PREPARE QUERY-TUNING QUESTION QUIT QUOTER RADIO-BUTTONS RADIO-SET RANDOM RAW - RAW-TRANSFER RCODE-INFO RCODE-INFOR RCODE-INFORM RCODE-INFORMA RCODE-INFORMAT RCODE-INFORMATI - RCODE-INFORMATIO RCODE-INFORMATION READ READ-AVAILABLE READ-EXACT-NUM READ-FILE READ-JSON READKEY - READ-ONLY READ-RESPONSE READ-XML READ-XMLSCHEMA REAL RECALL RECID RECORD-LEN RECORD-LENG - RECORD-LENGT RECORD-LENGTH RECT RECTA RECTAN RECTANG RECTANGL RECTANGLE RECURSIVE REFERENCE-ONLY - REFRESH REFRESHABLE REFRESH-AUDIT-POLICY REGISTER-DOMAIN REINSTATE REJECT-CHANGES REJECTED - REJECT-ROW-CHANGES RELATION-FI RELATION-FIE RELATION-FIEL RELATION-FIELD RELATION-FIELDS - RELATIONS-ACTIVE RELEASE REMOTE REMOTE-HOST REMOTE-PORT REMOVE-ATTRIBUTE REMOVE-CHILD - REMOVE-EVENTS-PROC REMOVE-EVENTS-PROCE REMOVE-EVENTS-PROCED REMOVE-EVENTS-PROCEDU - REMOVE-EVENTS-PROCEDUR REMOVE-EVENTS-PROCEDURE REMOVE-SUPER-PROC REMOVE-SUPER-PROCE - REMOVE-SUPER-PROCED REMOVE-SUPER-PROCEDU REMOVE-SUPER-PROCEDUR REMOVE-SUPER-PROCEDURE REPEAT REPLACE - REPLACE-CHILD REPLACE-SELECTION-TEXT REPLICATION-CREATE REPLICATION-DELETE REPLICATION-WRITE REPORTS - REPOSITION REPOSITION-BACK REPOSITION-BACKW REPOSITION-BACKWA REPOSITION-BACKWAR REPOSITION-BACKWARD - REPOSITION-BACKWARDS REPOSITION-FORW REPOSITION-FORWA REPOSITION-FORWAR REPOSITION-FORWARD - REPOSITION-FORWARDS REPOSITION-MODE REPOSITION-PARENT-REL REPOSITION-PARENT-RELA - REPOSITION-PARENT-RELAT REPOSITION-PARENT-RELATI REPOSITION-PARENT-RELATIO REPOSITION-PARENT-RELATION - REPOSITION-TO-ROW REPOSITION-TO-ROWID REQUEST REQUEST-INFO RESET RESIZA RESIZAB RESIZABL RESIZABLE - RESIZE RESPONSE-INFO RESTART-ROW RESTART-ROWID RESULT RESUME-DISPLAY RETAIN RETAIN-S RETAIN-SH - RETAIN-SHA RETAIN-SHAP RETAIN-SHAPE RETRY RETRY-CANCEL RETURN RETURN-ALIGN RETURN-INS RETURN-INSE - RETURN-INSER RETURN-INSERT RETURN-INSERTE RETURN-INSERTED RETURNS RETURN-TO-START-DI - RETURN-TO-START-DIR RETURN-VAL RETURN-VALU RETURN-VALUE RETURN-VALUE-DATA-TYPE RETURN-VALUE-DLL-TYPE - REVERSE-FROM REVERT REVOKE RGB-V RGB-VA RGB-VAL RGB-VALU RGB-VALUE RIGHT RIGHT-ALIGN RIGHT-ALIGNE - RIGHT-ALIGNED RIGHT-END RIGHT-TRIM R-INDEX ROLE ROLES ROUND ROUNDED ROUTINE-LEVEL ROW ROW-CREATED - ROW-DELETED ROW-DISPLAY ROW-ENTRY ROW-HEIGHT ROW-HEIGHT-C ROW-HEIGHT-CH ROW-HEIGHT-CHA - ROW-HEIGHT-CHAR ROW-HEIGHT-CHARS ROW-HEIGHT-P ROW-HEIGHT-PI ROW-HEIGHT-PIX ROW-HEIGHT-PIXE - ROW-HEIGHT-PIXEL ROW-HEIGHT-PIXELS ROWID ROW-LEAVE ROW-MA ROW-MAR ROW-MARK ROW-MARKE ROW-MARKER - ROW-MARKERS ROW-MODIFIED ROW-OF ROW-RESIZABLE ROW-STATE ROW-UNMODIFIED RULE RULE-ROW RULE-Y RUN - RUN-PROC RUN-PROCE RUN-PROCED RUN-PROCEDU RUN-PROCEDUR RUN-PROCEDURE SAVE SAVE-AS SAVE-FILE - SAVE-ROW-CHANGES SAVE-WHERE-STRING SAX-ATTRIBUTES SAX-COMPLE SAX-COMPLET SAX-COMPLETE SAX-PARSE - SAX-PARSE-FIRST SAX-PARSE-NEXT SAX-PARSER-ERROR SAX-READER SAX-RUNNING SAX-UNINITIALIZED - SAX-WRITE-BEGIN SAX-WRITE-COMPLETE SAX-WRITE-CONTENT SAX-WRITE-ELEMENT SAX-WRITE-ERROR - SAX-WRITE-IDLE SAX-WRITER SAX-WRITE-TAG SAX-XML SCHEMA SCHEMA-CHANGE SCHEMA-LOCATION SCHEMA-MARSHAL - SCHEMA-PATH SCREEN SCREEN-IO SCREEN-LINES SCREEN-VAL SCREEN-VALU SCREEN-VALUE SCROLL SCROLLABLE - SCROLLBAR-DRAG SCROLLBAR-H SCROLLBAR-HO SCROLLBAR-HOR SCROLLBAR-HORI SCROLLBAR-HORIZ - SCROLLBAR-HORIZO SCROLLBAR-HORIZON SCROLLBAR-HORIZONT SCROLLBAR-HORIZONTA SCROLLBAR-HORIZONTAL - SCROLL-BARS SCROLLBAR-V SCROLLBAR-VE SCROLLBAR-VER SCROLLBAR-VERT SCROLLBAR-VERTI SCROLLBAR-VERTIC - SCROLLBAR-VERTICA SCROLLBAR-VERTICAL SCROLL-DELTA SCROLLED-ROW-POS SCROLLED-ROW-POSI - SCROLLED-ROW-POSIT SCROLLED-ROW-POSITI SCROLLED-ROW-POSITIO SCROLLED-ROW-POSITION SCROLL-HORIZONTAL - SCROLLING SCROLL-LEFT SCROLL-MODE SCROLL-NOTIFY SCROLL-OFFSET SCROLL-RIGHT SCROLL-TO-CURRENT-ROW - SCROLL-TO-I SCROLL-TO-IT SCROLL-TO-ITE SCROLL-TO-ITEM SCROLL-TO-SELECTED-ROW SCROLL-VERTICAL SDBNAME - SEAL SEAL-TIMESTAMP SEARCH SEARCH-SELF SEARCH-TARGER SEARCH-TARGET SECTION SECURITY-POLICY SEEK - SELECT SELECTABLE SELECT-ALL SELECTED SELECTED-ITEMS SELECT-EXTEND SELECT-FOCUSED-ROW SELECTION - SELECTION-END SELECTION-EXTEND SELECTION-LIST SELECTION-START SELECTION-TEXT SELECT-NEXT-ROW - SELECT-ON-JOIN SELECT-PREV-ROW SELECT-REPOSITIONED-ROW SELECT-ROW SELF SEND SEND-SQL - SEND-SQL-STATEMENT SENSITIVE SEPARATE-CONNECTION SEPARATOR-FGC SEPARATOR-FGCO SEPARATOR-FGCOL - SEPARATOR-FGCOLO SEPARATOR-FGCOLOR SEPARATORS SERIALIZABLE SERIALIZE-HIDDEN SERIALIZE-NAME - SERIALIZE-ROW SERVER SERVER-CONNECTION-BO SERVER-CONNECTION-BOU SERVER-CONNECTION-BOUN - SERVER-CONNECTION-BOUND SERVER-CONNECTION-BOUND-RE SERVER-CONNECTION-BOUND-REQ SERVER-CONNECTION-BOUND-REQU - SERVER-CONNECTION-BOUND-REQUE SERVER-CONNECTION-BOUND-REQUES SERVER-CONNECTION-BOUND-REQUEST - SERVER-CONNECTION-CO SERVER-CONNECTION-CON SERVER-CONNECTION-CONT SERVER-CONNECTION-CONTE - SERVER-CONNECTION-CONTEX SERVER-CONNECTION-CONTEXT SERVER-CONNECTION-ID SERVER-OPERATING-MODE - SERVER-SOCKET SESSION SESSION-END SESSION-ID SET SET-ACTOR SET-APPL-CONTEXT SET-ATTR-CALL-TYPE - SET-ATTRIBUTE SET-ATTRIBUTE-NODE SET-BLUE SET-BLUE- SET-BLUE-V SET-BLUE-VA SET-BLUE-VAL - SET-BLUE-VALU SET-BLUE-VALUE SET-BREAK SET-BUFFERS SET-BYTE-ORDER SET-CALLBACK SET-CALLBACK-PROCEDURE - SET-CELL-FOCUS SET-CLIENT SET-COMMIT SET-CONNECT-PROCEDURE SET-CONTENTS SET-CURRENT-VALUE - SET-DB-CLIENT SET-DB-LOGGING SET-DYNAMIC SET-EFFECTIVE-TENANT SET-EVENT-MANAGER-OPTION SET-GREEN - SET-GREEN- SET-GREEN-V SET-GREEN-VA SET-GREEN-VAL SET-GREEN-VALU SET-GREEN-VALUE SET-INPUT-SOURCE - SET-MUST-UNDERSTAND SET-NODE SET-NUMERIC-FORM SET-NUMERIC-FORMA SET-NUMERIC-FORMAT SET-OPTION - SET-OUTPUT-DESTINATION SET-PARAMETER SET-POINTER-VAL SET-POINTER-VALU SET-POINTER-VALUE SET-PROPERTY - SET-READ-RESPONSE-PROCEDURE SET-RED SET-RED- SET-RED-V SET-RED-VA SET-RED-VAL SET-RED-VALU - SET-RED-VALUE SET-REPOSITIONED-ROW SET-RGB SET-RGB- SET-RGB-V SET-RGB-VA SET-RGB-VAL SET-RGB-VALU - SET-RGB-VALUE SET-ROLE SET-ROLLBACK SET-SAFE-USER SET-SELECTION SET-SERIALIZED SET-SIZE - SET-SOCKET-OPTION SET-SORT-ARROW SET-STATE SETTINGS SETUSER SETUSERI SETUSERID SET-WAIT SET-WAIT- - SET-WAIT-S SET-WAIT-ST SET-WAIT-STA SET-WAIT-STAT SET-WAIT-STATE SHA1-DIGEST SHARE SHARE- SHARED - SHARE-L SHARE-LO SHARE-LOC SHARE-LOCK SHORT SHOW-IN-TASK SHOW-IN-TASKB SHOW-IN-TASKBA - SHOW-IN-TASKBAR SHOW-STAT SHOW-STATS SIDE-LAB SIDE-LABE SIDE-LABEL SIDE-LABEL-H SIDE-LABEL-HA - SIDE-LABEL-HAN SIDE-LABEL-HAND SIDE-LABEL-HANDL SIDE-LABEL-HANDLE SIDE-LABELS SIGNATURE - SIGNATURE-VALUE SILENT SIMPLE SINGLE SINGLE-CHARACTER SINGLE-RUN SINGLETON SIZE SIZE-C SIZE-CH - SIZE-CHA SIZE-CHAR SIZE-CHARS SIZE-P SIZE-PI SIZE-PIX SIZE-PIXE SIZE-PIXEL SIZE-PIXELS SKIP - SKIP-DELETED-REC SKIP-DELETED-RECO SKIP-DELETED-RECOR SKIP-DELETED-RECORD SKIP-GROUP-DUPLICATES - SKIP-SCHEMA-CHECK SLIDER SMALL-ICON SMALLINT SMALL-TITLE SOAP-FAULT SOAP-FAULT-ACTOR SOAP-FAULT-CODE - SOAP-FAULT-DETAIL SOAP-FAULT-MISUNDERSTOOD-HEADER SOAP-FAULT-NODE SOAP-FAULT-ROLE SOAP-FAULT-STRING - SOAP-FAULT-SUBCODE SOAP-HEADER SOAP-HEADER-ENTRYREF SOAP-VERSION SOCKET SOME SORT SORT-ASCENDING - SORT-NUMBER SOURCE SOURCE-PROCEDURE SPACE SQL SQRT SSL-SERVER-NAME STANDALONE START - START-BOX-SELECTION START-DOCUMENT START-ELEMENT START-EXTEND-BOX-SELECTION STARTING START-MEM-CHECK - START-MOVE START-RESIZE START-ROW-RESIZE START-SEARCH STARTUP-PARAMETERS STATE-DETAIL STATIC - STATISTICS STATUS STATUS-AREA STATUS-AREA-FONT STATUS-AREA-MSG STDCALL STOMP-DETECTION - STOMP-FREQUENCY STOP STOP-AFTER STOP-DISPLAY STOP-MEM-CHECK STOP-OBJECT STOP-PARSING STOPPE STOPPED - STORED-PROC STORED-PROCE STORED-PROCED STORED-PROCEDU STORED-PROCEDUR STORED-PROCEDURE STREAM - STREAM-HANDLE STREAM-IO STRETCH-TO-FIT STRICT STRICT-ENTITY-RESOLUTION STRING STRING-VALUE - STRING-XREF SUB- SUB-AVE SUB-AVER SUB-AVERA SUB-AVERAG SUB-AVERAGE SUB-COUNT SUB-MAX SUB-MAXI - SUB-MAXIM SUB-MAXIMU SUB-MAXIMUM SUB-MENU SUB-MENU-HELP SUB-MIN SUB-MINI SUB-MINIM SUB-MINIMU - SUB-MINIMUM SUBSCRIBE SUBST SUBSTI SUBSTIT SUBSTITU SUBSTITUT SUBSTITUTE SUBSTR SUBSTRI SUBSTRIN - SUBSTRING SUB-TOTAL SUBTYPE SUM SUMMARY SUM-MAX SUPER SUPER-PROC SUPER-PROCE SUPER-PROCED - SUPER-PROCEDU SUPER-PROCEDUR SUPER-PROCEDURE SUPER-PROCEDURES SUPPRESS-NAMESPACE-PROCESSING - SUPPRESS-W SUPPRESS-WA SUPPRESS-WAR SUPPRESS-WARN SUPPRESS-WARNI SUPPRESS-WARNIN SUPPRESS-WARNING - SUPPRESS-WARNINGS SUPPRESS-WARNINGS-LIST SUSPEND SYMMETRIC-ENCRYPTION-ALGORITHM SYMMETRIC-ENCRYPTION-IV - SYMMETRIC-ENCRYPTION-KEY SYMMETRIC-SUPPORT SYNCHRONIZE SYSTEM-ALERT SYSTEM-ALERT- SYSTEM-ALERT-B - SYSTEM-ALERT-BO SYSTEM-ALERT-BOX SYSTEM-ALERT-BOXE SYSTEM-ALERT-BOXES SYSTEM-DIALOG SYSTEM-HELP - SYSTEM-ID TAB TABLE TABLE-CRC-LIST TABLE-HANDLE TABLE-LIST TABLE-NUM TABLE-NUMB TABLE-NUMBE - TABLE-NUMBER TABLE-SCAN TAB-POSITION TAB-STOP TARGET TARGET-PROCEDURE TEMP-DIR TEMP-DIRE TEMP-DIREC - TEMP-DIRECT TEMP-DIRECTO TEMP-DIRECTOR TEMP-DIRECTORY TEMP-TABLE TEMP-TABLE-PREPAR - TEMP-TABLE-PREPARE TENANT TENANT-ID TENANT-NAME TENANT-NAME-TO-ID TENANT-WHERE TERM TERMINAL - TERMINATE TEXT TEXT-CURSOR TEXT-SEG TEXT-SEG- TEXT-SEG-G TEXT-SEG-GR TEXT-SEG-GRO TEXT-SEG-GROW - TEXT-SEG-GROWT TEXT-SEG-GROWTH TEXT-SELECTED THEN THIS-OBJECT THIS-PROCEDURE THREAD-SAFE THREE-D - THROUGH THROW THRU TIC-MARKS TIME TIME-SOURCE TIMEZONE TITLE TITLE-BGC TITLE-BGCO TITLE-BGCOL - TITLE-BGCOLO TITLE-BGCOLOR TITLE-DC TITLE-DCO TITLE-DCOL TITLE-DCOLO TITLE-DCOLOR TITLE-FGC - TITLE-FGCO TITLE-FGCOL TITLE-FGCOLO TITLE-FGCOLOR TITLE-FO TITLE-FON TITLE-FONT TO TODAY TOGGLE-BOX - TOOLTIP TOOLTIPS TOP TOP-COLUMN TOPIC TOP-NAV-QUERY TOP-ONLY TO-ROWID TOTAL TRACE-FILTER TRACING - TRACKING-CHANGES TRAILING TRANS TRANSACT TRANSACTI TRANSACTIO TRANSACTION TRANSACTION-MODE - TRANS-INIT-PROC TRANS-INIT-PROCE TRANS-INIT-PROCED TRANS-INIT-PROCEDU TRANS-INIT-PROCEDUR - TRANS-INIT-PROCEDURE TRANSPAR TRANSPARE TRANSPAREN TRANSPARENT TRIGGER TRIGGERS TRIM TRUE TRUNC - TRUNCA TRUNCAT TRUNCATE TTCODEPAGE TYPE TYPE-OF UNBOX UNBUFF UNBUFFE UNBUFFER UNBUFFERE UNBUFFERED - UNDERL UNDERLI UNDERLIN UNDERLINE UNDO UNDO-THROW-SCOPE UNFORM UNFORMA UNFORMAT UNFORMATT UNFORMATTE - UNFORMATTED UNION UNIQUE UNIQUE-ID UNIQUE-MATCH UNIX UNIX-END UNLESS-HIDDEN UNLOAD UNSIGNED-BYTE - UNSIGNED-INT64 UNSIGNED-INTEGER UNSIGNED-LONG UNSIGNED-SHORT UNSUBSCRIBE UP UPDATE UPDATE-ATTRIBUTE - UPPER URL URL-DECODE URL-ENCODE URL-PASSWORD URL-USERID USE USE-DIC USE-DICT USE-DICT- USE-DICT-E - USE-DICT-EX USE-DICT-EXP USE-DICT-EXPS USE-FILENAME USE-INDEX USER USER-DATA USE-REVVIDEO USERID - USER-ID USE-TEXT USE-UNDERLINE USE-WIDGET-POOL USING UTC-OFFSET V6DISPLAY V6FRAME VALIDATE - VALIDATE-DOMAIN-ACCESS-CODE VALIDATE-EXPRESSIO VALIDATE-EXPRESSION VALIDATE-MESSAGE VALIDATE-SEAL - VALIDATE-XML VALIDATION-ENABLED VALID-EVENT VALID-HANDLE VALID-OBJECT VALUE VALUE-CHANGED VALUES VAR - VARI VARIA VARIAB VARIABL VARIABLE VERB VERBO VERBOS VERBOSE VERSION VERT VERTI VERTIC VERTICA - VERTICAL VIEW VIEW-AS VIEW-FIRST-COLUMN-ON-REOPEN VIRTUAL-HEIGHT VIRTUAL-HEIGHT-C VIRTUAL-HEIGHT-CH - VIRTUAL-HEIGHT-CHA VIRTUAL-HEIGHT-CHAR VIRTUAL-HEIGHT-CHARS VIRTUAL-HEIGHT-P VIRTUAL-HEIGHT-PI - VIRTUAL-HEIGHT-PIX VIRTUAL-HEIGHT-PIXE VIRTUAL-HEIGHT-PIXEL VIRTUAL-HEIGHT-PIXELS VIRTUAL-WIDTH - VIRTUAL-WIDTH-C VIRTUAL-WIDTH-CH VIRTUAL-WIDTH-CHA VIRTUAL-WIDTH-CHAR VIRTUAL-WIDTH-CHARS - VIRTUAL-WIDTH-P VIRTUAL-WIDTH-PI VIRTUAL-WIDTH-PIX VIRTUAL-WIDTH-PIXE VIRTUAL-WIDTH-PIXEL - VIRTUAL-WIDTH-PIXELS VISIBLE VMS VOID WAIT WAIT-FOR WARNING WC-ADMIN-APP WEB-CON WEB-CONT WEB-CONTE - WEB-CONTEX WEB-CONTEXT WEB-NOTIFY WEEKDAY WHEN WHERE WHERE-STRING WHILE WIDGET WIDGET-E WIDGET-EN - WIDGET-ENT WIDGET-ENTE WIDGET-ENTER WIDGET-H WIDGET-HA WIDGET-HAN WIDGET-HAND WIDGET-HANDL - WIDGET-HANDLE WIDGET-ID WIDGET-L WIDGET-LE WIDGET-LEA WIDGET-LEAV WIDGET-LEAVE WIDGET-POOL WIDTH - WIDTH-C WIDTH-CH WIDTH-CHA WIDTH-CHAR WIDTH-CHARS WIDTH-P WIDTH-PI WIDTH-PIX WIDTH-PIXE WIDTH-PIXEL - WIDTH-PIXELS WINDOW WINDOW-CLOSE WINDOW-DELAYED-MIN WINDOW-DELAYED-MINI WINDOW-DELAYED-MINIM - WINDOW-DELAYED-MINIMI WINDOW-DELAYED-MINIMIZ WINDOW-DELAYED-MINIMIZE WINDOW-MAXIM WINDOW-MAXIMI - WINDOW-MAXIMIZ WINDOW-MAXIMIZE WINDOW-MAXIMIZED WINDOW-MINIM WINDOW-MINIMI WINDOW-MINIMIZ - WINDOW-MINIMIZE WINDOW-MINIMIZED WINDOW-NAME WINDOW-NORMAL WINDOW-RESIZED WINDOW-RESTORED WINDOW-STA - WINDOW-STAT WINDOW-STATE WINDOW-SYS WINDOW-SYST WINDOW-SYSTE WINDOW-SYSTEM WITH WORD-INDEX WORD-WRAP - WORK-AREA-HEIGHT-P WORK-AREA-HEIGHT-PI WORK-AREA-HEIGHT-PIX WORK-AREA-HEIGHT-PIXE - WORK-AREA-HEIGHT-PIXEL WORK-AREA-HEIGHT-PIXELS WORK-AREA-WIDTH-P WORK-AREA-WIDTH-PI - WORK-AREA-WIDTH-PIX WORK-AREA-WIDTH-PIXE WORK-AREA-WIDTH-PIXEL WORK-AREA-WIDTH-PIXELS WORK-AREA-X - WORK-AREA-Y WORKFILE WORK-TAB WORK-TABL WORK-TABLE WRITE WRITE-CDATA WRITE-CHARACTERS WRITE-COMMENT - WRITE-DATA WRITE-DATA-ELEMENT WRITE-EMPTY-ELEMENT WRITE-ENTITY-REF WRITE-EXTERNAL-DTD WRITE-FRAGMENT - WRITE-JSON WRITE-MESSAGE WRITE-PROCESSING-INSTRUCTION WRITE-STATUS WRITE-XML WRITE-XMLSCHEMA X XCODE - XCODE-SESSION-KEY X-DOCUMENT XML-DATA-TYPE XML-ENTITY-EXPANSION-LIMIT XML-NODE-NAME XML-NODE-TYPE - XML-SCHEMA-PAT XML-SCHEMA-PATH XML-STRICT-ENTITY-RESOLUTION XML-SUPPRESS-NAMESPACE-PROCESSING - X-NODEREF X-OF XOR XREF XREF-XML Y YEAR YEAR-OFFSET YES YES-NO YES-NO-CANCEL Y-OF - ) - end - - def self.keywords_prepro - @keywords_prepro ||= Set.new %w( - &ANALYZE-SUSPEND &ANALYZE-RESUME - &ELSE &ELSEIF &ENDIF &GLOB &GLOBAL-DEFINE &IF &MESSAGE &SCOP &SCOPED-DEFINE &THEN &UNDEF &UNDEFINE &WEBSTREAM - {&BATCH} {&BATCH-MODE} {&FILE-NAME} {&LINE-NUMBE} {&LINE-NUMBER} {&OPSYS} {&PROCESS-ARCHITECTURE} {&SEQUENCE} - {&WINDOW-SYS} {&WINDOW-SYSTEM} - ) - end + lazy { require_relative 'openedge/keywords' } - def self.keywords_type - @keywords_type ||= Set.new %w( - BLOB CHARACTER CHAR CLOB COM-HANDLE DATE DATETIME DATETIME-TZ DECIMAL - DEC HANDLE INT64 INTEGER INT LOGICAL LONGCHAR MEMPTR RAW RECID ROWID - WIDGET-HANDLE VOID - ) - end + id = /[a-zA-Z_&{}!][a-zA-Z0-9_\-&!}]*/ state :statements do mixin :singlelinecomment @@ -614,17 +123,13 @@ def self.keywords_type state :root do mixin :statements - rule id do |m| - name = m[0].upcase - if self.class.keywords_prepro.include? name - token Comment::Preproc - elsif self.class.keywords.include? name - token Keyword - elsif self.class.keywords_type.include? name - token Keyword::Type - else - token Name::Variable - end + keywords id do + transform(&:upcase) + + rule KEYWORDS_PREPRO, Comment::Preproc + rule KEYWORDS, Keyword + rule KEYWORDS_TYPE, Keyword::Type + default Name end end end diff --git a/lib/rouge/lexers/openedge/keywords.rb b/lib/rouge/lexers/openedge/keywords.rb new file mode 100644 index 0000000000..fc6c5fb59a --- /dev/null +++ b/lib/rouge/lexers/openedge/keywords.rb @@ -0,0 +1,492 @@ +module Rouge + module Lexers + class OpenEdge + KEYWORDS = Set.new %w( + ABORT ABS ABSO ABSOL ABSOLU ABSOLUT ABSOLUTE ABSTRACT ACCELERATOR ACCEPT-CHANGES ACCEPT-ROW-CHANGES + ACCUM ACCUMU ACCUMUL ACCUMULA ACCUMULAT ACCUMULATE ACROSS ACTIVE ACTIVE-FORM ACTIVE-WINDOW ACTOR ADD + ADD-BUFFER ADD-CALC-COL ADD-CALC-COLU ADD-CALC-COLUM ADD-CALC-COLUMN ADD-COLUMNS-FROM + ADD-EVENTS-PROC ADD-EVENTS-PROCE ADD-EVENTS-PROCED ADD-EVENTS-PROCEDU ADD-EVENTS-PROCEDUR + ADD-EVENTS-PROCEDURE ADD-FIELDS-FROM ADD-FIRST ADD-HEADER-ENTRY ADD-INDEX-FIELD ADD-INTERVAL + ADD-LAST ADD-LIKE-COLUMN ADD-LIKE-FIELD ADD-LIKE-INDEX ADD-NEW-FIELD ADD-NEW-INDEX + ADD-SCHEMA-LOCATION ADD-SOURCE-BUFFER ADD-SUPER-PROC ADD-SUPER-PROCE ADD-SUPER-PROCED + ADD-SUPER-PROCEDU ADD-SUPER-PROCEDUR ADD-SUPER-PROCEDURE ADM-DATA ADVISE AFTER-BUFFER AFTER-ROWID + AFTER-TABLE ALERT-BOX ALIAS ALL ALLOW-COLUMN-SEARCHING ALLOW-PREV-DESERIALIZATION ALLOW-REPLICATION + ALTER ALTERNATE-KEY ALWAYS-ON-TOP AMBIG AMBIGU AMBIGUO AMBIGUOU AMBIGUOUS ANALYZ ANALYZE AND + ANSI-ONLY ANY ANY-KEY ANY-PRINTABLE ANYWHERE APPEND APPEND-CHILD APPEND-LINE APPL-ALERT APPL-ALERT- + APPL-ALERT-B APPL-ALERT-BO APPL-ALERT-BOX APPL-ALERT-BOXE APPL-ALERT-BOXES APPL-CONTEXT-ID + APPLICATION APPLY APPLY-CALLBACK APPSERVER-INFO APPSERVER-PASSWORD APPSERVER-USERID ARRAY-M ARRAY-ME + ARRAY-MES ARRAY-MESS ARRAY-MESSA ARRAY-MESSAG ARRAY-MESSAGE AS ASC ASCE ASCEN ASCEND ASCENDI + ASCENDIN ASCENDING AS-CURSOR ASK-OVERWRITE ASSEMBLY ASSIGN ASYNCHRONOUS ASYNC-REQUEST-COUNT + ASYNC-REQUEST-HANDLE AT ATTACH ATTACH-DATA-SOURCE ATTACHED-PAIRLIST ATTACHMENT ATTR ATTR- + ATTRIBUTE-NAMES ATTRIBUTE-TYPE ATTR-S ATTR-SP ATTR-SPA ATTR-SPAC ATTR-SPACE AUDIT-CONTROL + AUDIT-ENABLED AUDIT-EVENT-CONTEXT AUDIT-POLICY AUTHENTICATION-FAILED AUTHORIZATION AUTO-COMP + AUTO-COMPL AUTO-COMPLE AUTO-COMPLET AUTO-COMPLETI AUTO-COMPLETIO AUTO-COMPLETION AUTO-DELETE + AUTO-DELETE-XML AUTO-ENDKEY AUTO-END-KEY AUTO-GO AUTO-IND AUTO-INDE AUTO-INDEN AUTO-INDENT AUTOMATIC + AUTO-RESIZE AUTO-RET AUTO-RETU AUTO-RETUR AUTO-RETURN AUTO-SYNCHRONIZE AUTO-VAL AUTO-VALI AUTO-VALID + AUTO-VALIDA AUTO-VALIDAT AUTO-VALIDATE AUTO-Z AUTO-ZA AUTO-ZAP AVAIL AVAILA AVAILAB AVAILABL + AVAILABLE AVAILABLE-FORMATS AVE AVER AVERA AVERAG AVERAGE AVG BACK BACKG BACKGR BACKGRO BACKGROU + BACKGROUN BACKGROUND BACKSPACE BACK-TAB BACKWARD BACKWARDS BASE64 BASE64-DECODE BASE64-ENCODE + BASE-ADE BASE-KEY BASIC-LOGGING BATCH BATCH-MODE BATCH-SIZE BEFORE-BUFFER BEFORE-H BEFORE-HI + BEFORE-HID BEFORE-HIDE BEFORE-ROWID BEFORE-TABLE BEGIN-EVENT-GROUP BEGINS BELL BETWEEN BGC BGCO + BGCOL BGCOLO BGCOLOR BIG-ENDIAN BINARY BIND BIND-WHERE BLANK BLOB BLOCK BLOCK-ITERATION-DISPLAY + BLOCK-LEV BLOCK-LEVE BLOCK-LEVEL BORDER-B BORDER-BO BORDER-BOT BORDER-BOTT BORDER-BOTTO + BORDER-BOTTOM BORDER-BOTTOM-C BORDER-BOTTOM-CH BORDER-BOTTOM-CHA BORDER-BOTTOM-CHAR + BORDER-BOTTOM-CHARS BORDER-BOTTOM-P BORDER-BOTTOM-PI BORDER-BOTTOM-PIX BORDER-BOTTOM-PIXE + BORDER-BOTTOM-PIXEL BORDER-BOTTOM-PIXELS BORDER-L BORDER-LE BORDER-LEF BORDER-LEFT BORDER-LEFT-C + BORDER-LEFT-CH BORDER-LEFT-CHA BORDER-LEFT-CHAR BORDER-LEFT-CHARS BORDER-LEFT-P BORDER-LEFT-PI + BORDER-LEFT-PIX BORDER-LEFT-PIXE BORDER-LEFT-PIXEL BORDER-LEFT-PIXELS BORDER-R BORDER-RI BORDER-RIG + BORDER-RIGH BORDER-RIGHT BORDER-RIGHT-C BORDER-RIGHT-CH BORDER-RIGHT-CHA BORDER-RIGHT-CHAR + BORDER-RIGHT-CHARS BORDER-RIGHT-P BORDER-RIGHT-PI BORDER-RIGHT-PIX BORDER-RIGHT-PIXE + BORDER-RIGHT-PIXEL BORDER-RIGHT-PIXELS BORDER-T BORDER-TO BORDER-TOP BORDER-TOP-C BORDER-TOP-CH + BORDER-TOP-CHA BORDER-TOP-CHAR BORDER-TOP-CHARS BORDER-TOP-P BORDER-TOP-PI BORDER-TOP-PIX + BORDER-TOP-PIXE BORDER-TOP-PIXEL BORDER-TOP-PIXELS BOTH BOTTOM BOTTOM-COLUMN BOX BOX-SELECT + BOX-SELECTA BOX-SELECTAB BOX-SELECTABL BOX-SELECTABLE BREAK BREAK-LINE BROWSE BROWSE-COLUMN-DATA-TYPES + BROWSE-COLUMN-FORMATS BROWSE-COLUMN-LABELS BROWSE-HEADER BTOS BUFFER BUFFER-CHARS BUFFER-COMP + BUFFER-COMPA BUFFER-COMPAR BUFFER-COMPARE BUFFER-COPY BUFFER-CREATE BUFFER-DELETE BUFFER-FIELD + BUFFER-GROUP-ID BUFFER-GROUP-NAME BUFFER-HANDLE BUFFER-LINES BUFFER-N BUFFER-NA BUFFER-NAM + BUFFER-NAME BUFFER-PARTITION-ID BUFFER-RELEAS BUFFER-RELEASE BUFFER-TENANT-ID BUFFER-TENANT-NAME + BUFFER-VALIDATE BUFFER-VALUE BUTTON BUTTONS BY BY-POINTER BY-REFERENCE BYTE BYTES-READ BYTES-WRITTEN + BY-VALUE BY-VARIANT-POINT BY-VARIANT-POINTE BY-VARIANT-POINTER CACHE CACHE-SIZE CALL CALL-NAME + CALL-TYPE CANCEL-BREAK CANCEL-BUTTON CANCELLED CANCEL-PICK CANCEL-REQUESTS CANCEL-REQUESTS-AFTER + CAN-CREA CAN-CREAT CAN-CREATE CAN-DELE CAN-DELET CAN-DELETE CAN-DO CAN-DO-DOMAIN-SUPPORT CAN-FIND + CAN-QUERY CAN-READ CAN-SET CAN-WRIT CAN-WRITE CAPS CAREFUL-PAINT CASE CASE-SEN CASE-SENS CASE-SENSI + CASE-SENSIT CASE-SENSITI CASE-SENSITIV CASE-SENSITIVE CAST CATCH CDECL CENTER CENTERE CENTERED + CHAINED CHAR CHARA CHARAC CHARACT CHARACTE CHARACTER CHARACTER_LENGTH CHARSET CHECK CHECKED + CHECK-MEM-STOMP CHILD-BUFFER CHILD-NUM CHOICES CHOOSE CHR CLASS CLASS-TYPE CLEAR CLEAR-APPL-CONTEXT + CLEAR-LOG CLEAR-SELECT CLEAR-SELECTI CLEAR-SELECTIO CLEAR-SELECTION CLEAR-SORT-ARROW + CLEAR-SORT-ARROWS CLIENT-CONNECTION-ID CLIENT-PRINCIPAL CLIENT-TTY CLIENT-TYPE CLIENT-WORKSTATION + CLIPBOARD CLOB CLONE-NODE CLOSE CLOSE-LOG CODE CODEBASE-LOCATOR CODEPAGE CODEPAGE-CONVERT COL + COLLATE COL-OF COLON COLON-ALIGN COLON-ALIGNE COLON-ALIGNED COLOR COLOR-TABLE COLUMN COLUMN-BGC + COLUMN-BGCO COLUMN-BGCOL COLUMN-BGCOLO COLUMN-BGCOLOR COLUMN-CODEPAGE COLUMN-DCOLOR COLUMN-FGC + COLUMN-FGCO COLUMN-FGCOL COLUMN-FGCOLO COLUMN-FGCOLOR COLUMN-FONT COLUMN-LAB COLUMN-LABE + COLUMN-LABEL COLUMN-LABEL-BGC COLUMN-LABEL-BGCO COLUMN-LABEL-BGCOL COLUMN-LABEL-BGCOLO + COLUMN-LABEL-BGCOLOR COLUMN-LABEL-DCOLOR COLUMN-LABEL-FGC COLUMN-LABEL-FGCO COLUMN-LABEL-FGCOL + COLUMN-LABEL-FGCOLO COLUMN-LABEL-FGCOLOR COLUMN-LABEL-FONT COLUMN-LABEL-HEIGHT-C + COLUMN-LABEL-HEIGHT-CH COLUMN-LABEL-HEIGHT-CHA COLUMN-LABEL-HEIGHT-CHAR COLUMN-LABEL-HEIGHT-CHARS + COLUMN-LABEL-HEIGHT-P COLUMN-LABEL-HEIGHT-PI COLUMN-LABEL-HEIGHT-PIX COLUMN-LABEL-HEIGHT-PIXE + COLUMN-LABEL-HEIGHT-PIXEL COLUMN-LABEL-HEIGHT-PIXELS COLUMN-MOVABLE COLUMN-OF COLUMN-PFC COLUMN-PFCO + COLUMN-PFCOL COLUMN-PFCOLO COLUMN-PFCOLOR COLUMN-READ-ONLY COLUMN-RESIZABLE COLUMNS COLUMN-SC + COLUMN-SCR COLUMN-SCRO COLUMN-SCROL COLUMN-SCROLL COLUMN-SCROLLI COLUMN-SCROLLIN COLUMN-SCROLLING + COMBO-BOX COM-HANDLE COMMAND COMPARE COMPARES COMPILE COMPILER COMPLETE COMPONENT-HANDLE + COMPONENT-SELF COM-SELF CONFIG-NAME CONNECT CONNECTED CONSTRAINED CONSTRUCTOR CONTAINER-EVENT + CONTAINS CONTENTS CONTEXT CONTEXT-HELP CONTEXT-HELP-FILE CONTEXT-HELP-ID CONTEXT-POP CONTEXT-POPU + CONTEXT-POPUP CONTROL CONTROL-BOX CONTROL-CONT CONTROL-CONTA CONTROL-CONTAI CONTROL-CONTAIN + CONTROL-CONTAINE CONTROL-CONTAINER CONTROL-FRAM CONTROL-FRAME CONVERT CONVERT-3D CONVERT-3D- + CONVERT-3D-C CONVERT-3D-CO CONVERT-3D-COL CONVERT-3D-COLO CONVERT-3D-COLOR CONVERT-3D-COLORS + CONVERT-TO-OFFS CONVERT-TO-OFFSE CONVERT-TO-OFFSET COPY COPY-DATASET COPY-LOB COPY-SAX-ATTRIBUTES + COPY-TEMP-TABLE COUNT COUNT-OF COVERAGE CPCASE CPCOLL CPINT CPINTE CPINTER CPINTERN CPINTERNA + CPINTERNAL CPLOG CPPRINT CPRCODEIN CPRCODEOUT CPSTREAM CPTERM CRC-VAL CRC-VALU CRC-VALUE CREATE + CREATE-LIKE CREATE-LIKE-SEQUENTIAL CREATE-NODE CREATE-NODE-NAMESPACE CREATE-ON-ADD + CREATE-RESULT-LIST-ENTRY CREATE-TEST-FILE CTOS CURRENT CURRENT_DATE CURRENT-CHANGED CURRENT-COLUMN + CURRENT-ENV CURRENT-ENVI CURRENT-ENVIR CURRENT-ENVIRO CURRENT-ENVIRON CURRENT-ENVIRONM + CURRENT-ENVIRONME CURRENT-ENVIRONMEN CURRENT-ENVIRONMENT CURRENT-ITERATION CURRENT-LANG + CURRENT-LANGU CURRENT-LANGUA CURRENT-LANGUAG CURRENT-LANGUAGE CURRENT-QUERY CURRENT-REQUEST-INFO + CURRENT-RESPONSE-INFO CURRENT-RESULT-ROW CURRENT-ROW-MODIFIED CURRENT-VALUE CURRENT-WINDOW CURS + CURSO CURSOR CURSOR-CHAR CURSOR-DOWN CURSOR-LEFT CURSOR-LINE CURSOR-OFFSET CURSOR-RIGHT CURSOR-UP + CUT DATA-B DATABASE DATA-BI DATA-BIN DATA-BIND DATA-ENTRY-RET DATA-ENTRY-RETU DATA-ENTRY-RETUR + DATA-ENTRY-RETURN DATA-REFRESH-LINE DATA-REFRESH-PAGE DATA-REL DATA-RELA DATA-RELAT DATA-RELATI + DATA-RELATIO DATA-RELATION DATASERVERS DATASET DATASET-HANDLE DATA-SOURCE DATA-SOURCE-COMPLETE-MAP + DATA-SOURCE-MODIFIED DATA-SOURCE-ROWID DATA-T DATA-TY DATA-TYP DATA-TYPE DATE DATE-F DATE-FO + DATE-FOR DATE-FORM DATE-FORMA DATE-FORMAT DATETIME DATETIME-TZ DAY DBCODEPAGE DBCOLLATION DB-CONTEXT + DB-LIST DBNAME DBPARAM DB-REFERENCES DB-REMOTE-HOST DBREST DBRESTR DBRESTRI DBRESTRIC DBRESTRICT + DBRESTRICTI DBRESTRICTIO DBRESTRICTION DBRESTRICTIONS DBTASKID DBTYPE DBVERS DBVERSI DBVERSIO + DBVERSION DCOLOR DDE DDE-ERROR DDE-I DDE-ID DDE-ITEM DDE-NAME DDE-NOTIFY DDE-TOPIC DEBLANK DEBU + DEBUG DEBUG-ALERT DEBUGGER DEBUG-LIST DEBUG-SET-TENANT DEC DECI DECIM DECIMA DECIMAL DECIMALS + DECLARE DECLARE-NAMESPACE DECRYPT DEF DEFAULT DEFAULT-ACTION DEFAULT-BUFFER-HANDLE DEFAULT-BUT + DEFAULT-BUTT DEFAULT-BUTTO DEFAULT-BUTTON DEFAULT-COMMIT DEFAULT-EX DEFAULT-EXT DEFAULT-EXTE + DEFAULT-EXTEN DEFAULT-EXTENS DEFAULT-EXTENSI DEFAULT-EXTENSIO DEFAULT-EXTENSION DEFAULT-NOXL + DEFAULT-NOXLA DEFAULT-NOXLAT DEFAULT-NOXLATE DEFAULT-POP-UP DEFAULT-STRING DEFAULT-VALUE + DEFAULT-WINDOW DEFAUT-B DEFER-LOB-FETCH DEFI DEFIN DEFINE DEFINED DEFINE-USER-EVENT-MANAGER DEL + DELEGATE DELETE DELETE-CHAR DELETE-CHARACTER DELETE-COLUMN DELETE-CURRENT-ROW DELETE-END-LINE + DELETE-FIELD DELETE-HEADER-ENTRY DELETE-LINE DELETE-NODE DELETE-RESULT-LIST-ENTRY + DELETE-SELECTED-ROW DELETE-SELECTED-ROWS DELETE-WORD DELIMITER DESC DESCE DESCEN DESCEND DESCENDI + DESCENDIN DESCENDING DESCRIPT DESCRIPTI DESCRIPTIO DESCRIPTION DESELECT DESELECT-EXTEND + DESELECT-FOCUSED-ROW DESELECTION DESELECTION-EXTEND DESELECT-ROWS DESELECT-SELECTED-ROW DESTRUCTOR + DETACH DETACH-DATA-SOURCE DIALOG-BOX DIALOG-HELP DICT DICTI DICTIO DICTION DICTIONA DICTIONAR + DICTIONARY DIR DIRECTORY DISABLE DISABLE-AUTO-ZAP DISABLE-CONNECTIONS DISABLED DISABLE-DUMP-TRIGGERS + DISABLE-LOAD-TRIGGERS DISCON DISCONN DISCONNE DISCONNEC DISCONNECT DISMISS-MENU DISP DISPL DISPLA + DISPLAY DISPLAY-MESSAGE DISPLAY-T DISPLAY-TIMEZONE DISPLAY-TY DISPLAY-TYP DISPLAY-TYPE DISTINCT + DLL-CALL-TYPE DO DOMAIN-DESCRIPTION DOMAIN-NAME DOMAIN-TYPE DOS DOS-END DOTNET-CLR-LOADED DOUBLE + DOWN DRAG-ENABLED DROP DROP-DOWN DROP-DOWN-LIST DROP-FILE-NOTIFY DROP-TARGET DS-CLOSE-CURSOR + DSLOG-MANAGER DUMP DUMP-LOGGING-NOW DYNAMIC DYNAMIC-CAST DYNAMIC-CURRENT-VALUE DYNAMIC-ENUM + DYNAMIC-FUNC DYNAMIC-FUNCT DYNAMIC-FUNCTI DYNAMIC-FUNCTIO DYNAMIC-FUNCTION DYNAMIC-INVOKE + DYNAMIC-NEW DYNAMIC-NEXT-VALUE DYNAMIC-PROPERTY EACH ECHO EDGE EDGE-C EDGE-CH EDGE-CHA EDGE-CHAR + EDGE-CHARS EDGE-P EDGE-PI EDGE-PIX EDGE-PIXE EDGE-PIXEL EDGE-PIXELS EDIT-CAN-PASTE EDIT-CAN-UNDO + EDIT-CLEAR EDIT-COPY EDIT-CUT EDITING EDITOR EDITOR-BACKTAB EDITOR-TAB EDIT-PASTE EDIT-UNDO ELSE + EMPTY EMPTY-DATASET EMPTY-SELECTION EMPTY-TEMP-TABLE ENABLE ENABLE-CONNECTIONS ENABLED + ENABLED-FIELDS ENCODE ENCODE-DOMAIN-ACCESS-CODE ENCODING ENCRYPT ENCRYPT-AUDIT-MAC-KEY + ENCRYPTION-SALT END END-BOX-SELECTION END-DOCUMENT END-ELEMENT END-ERROR END-EVENT-GROUP + END-FILE-DROP ENDKEY END-KEY END-MOVE END-RESIZE END-ROW-RESIZE END-SEARCH END-USER-PROMPT ENTERED + ENTER-MENUBAR ENTITY-EXPANSION-LIMIT ENTRY ENTRY-TYPES-LIST ENUM EQ ERROR ERROR-COL ERROR-COLU + ERROR-COLUM ERROR-COLUMN ERROR-OBJECT ERROR-OBJECT-DETAIL ERROR-ROW ERROR-STACK-TRACE ERROR-STAT + ERROR-STATU ERROR-STATUS ERROR-STRING ESCAPE ETIME EVENT EVENT-GROUP-ID EVENT-PROCEDURE + EVENT-PROCEDURE-CONTEXT EVENTS EVENT-T EVENT-TY EVENT-TYP EVENT-TYPE EXCEPT EXCLUSIVE EXCLUSIVE-ID + EXCLUSIVE-L EXCLUSIVE-LO EXCLUSIVE-LOC EXCLUSIVE-LOCK EXCLUSIVE-WEB EXCLUSIVE-WEB- EXCLUSIVE-WEB-U + EXCLUSIVE-WEB-US EXCLUSIVE-WEB-USE EXCLUSIVE-WEB-USER EXECUTE EXECUTION-LOG EXISTS EXIT EXIT-CODE + EXP EXPAND EXPANDABLE EXPIRE EXPLICIT EXPORT EXPORT-PRINCIPAL EXTENDED EXTENT EXTERNAL EXTRACT FALSE + FALSE-LEAKS FETCH FETCH-SELECTED-ROW FGC FGCO FGCOL FGCOLO FGCOLOR FIELD FIELDS FILE FILE-ACCESS-D + FILE-ACCESS-DA FILE-ACCESS-DAT FILE-ACCESS-DATE FILE-ACCESS-T FILE-ACCESS-TI FILE-ACCESS-TIM + FILE-ACCESS-TIME FILE-CREATE-D FILE-CREATE-DA FILE-CREATE-DAT FILE-CREATE-DATE FILE-CREATE-T + FILE-CREATE-TI FILE-CREATE-TIM FILE-CREATE-TIME FILE-INFO FILE-INFOR FILE-INFORM FILE-INFORMA + FILE-INFORMAT FILE-INFORMATI FILE-INFORMATIO FILE-INFORMATION FILE-MOD-D FILE-MOD-DA FILE-MOD-DAT + FILE-MOD-DATE FILE-MOD-T FILE-MOD-TI FILE-MOD-TIM FILE-MOD-TIME FILENAME FILE-NAME FILE-OFF + FILE-OFFS FILE-OFFSE FILE-OFFSET FILE-SIZE FILE-TYPE FILL FILLED FILL-IN FILL-MODE FILL-WHERE-STRING + FILTERS FINAL FINALLY FIND FIND-BY-ROWID FIND-CASE-SENSITIVE FIND-CURRENT FINDER FIND-FIRST + FIND-GLOBAL FIND-LAST FIND-NEXT FIND-NEXT-OCCURRENCE FIND-PREVIOUS FIND-PREV-OCCURRENCE FIND-SELECT + FIND-UNIQUE FIND-WRAP-AROUND FIREHOSE-CURSOR FIRST FIRST-ASYNC FIRST-ASYNC- FIRST-ASYNCH-REQUEST + FIRST-ASYNC-R FIRST-ASYNC-RE FIRST-ASYNC-REQ FIRST-ASYNC-REQU FIRST-ASYNC-REQUE FIRST-ASYNC-REQUES + FIRST-ASYNC-REQUEST FIRST-BUFFER FIRST-CHILD FIRST-COLUMN FIRST-DATASET FIRST-DATA-SOURCE FIRST-FORM + FIRST-OBJECT FIRST-OF FIRST-PROC FIRST-PROCE FIRST-PROCED FIRST-PROCEDU FIRST-PROCEDUR + FIRST-PROCEDURE FIRST-QUERY FIRST-SERV FIRST-SERVE FIRST-SERVER FIRST-SERVER-SOCKET FIRST-SOCKET + FIRST-TAB-I FIRST-TAB-IT FIRST-TAB-ITE FIRST-TAB-ITEM FIT-LAST-COLUMN FIX-CODEPAGE FIXED-ONLY FLAGS + FLAT-BUTTON FLOAT FOCUS FOCUSED-ROW FOCUSED-ROW-SELECTED FOCUS-IN FONT FONT-TABLE FOR FORCE-FILE + FORE FOREG FOREGR FOREGRO FOREGROU FOREGROUN FOREGROUND FOREIGN-KEY-HIDDEN FORM FORMA FORMAT + FORMATTE FORMATTED FORM-INPUT FORM-LONG-INPUT FORWARD FORWARD-ONLY FORWARDS FRAGMEN FRAGMENT FRAM + FRAME FRAME-COL FRAME-DB FRAME-DOWN FRAME-FIELD FRAME-FILE FRAME-INDE FRAME-INDEX FRAME-LINE + FRAME-NAME FRAME-ROW FRAME-SPA FRAME-SPAC FRAME-SPACI FRAME-SPACIN FRAME-SPACING FRAME-VAL + FRAME-VALU FRAME-VALUE FRAME-X FRAME-Y FREQUENCY FROM FROM-C FROM-CH FROM-CHA FROM-CHAR FROM-CHARS + FROM-CUR FROM-CURR FROM-CURRE FROM-CURREN FROM-CURRENT FROMNOREORDER FROM-P FROM-PI FROM-PIX + FROM-PIXE FROM-PIXEL FROM-PIXELS FULL-HEIGHT FULL-HEIGHT-C FULL-HEIGHT-CH FULL-HEIGHT-CHA + FULL-HEIGHT-CHAR FULL-HEIGHT-CHARS FULL-HEIGHT-P FULL-HEIGHT-PI FULL-HEIGHT-PIX FULL-HEIGHT-PIXE + FULL-HEIGHT-PIXEL FULL-HEIGHT-PIXELS FULL-PATHN FULL-PATHNA FULL-PATHNAM FULL-PATHNAME FULL-WIDTH + FULL-WIDTH- FULL-WIDTH-C FULL-WIDTH-CH FULL-WIDTH-CHA FULL-WIDTH-CHAR FULL-WIDTH-CHARS FULL-WIDTH-P + FULL-WIDTH-PI FULL-WIDTH-PIX FULL-WIDTH-PIXE FULL-WIDTH-PIXEL FULL-WIDTH-PIXELS FUNCTION + FUNCTION-CALL-TYPE GATEWAY GATEWAYS GE GENERATE-MD5 GENERATE-PBE-KEY GENERATE-PBE-SALT + GENERATE-RANDOM-KEY GENERATE-UUID GET GET-ATTR-CALL-TYPE GET-ATTRIBUTE GET-ATTRIBUTE-NODE + GET-BINARY-DATA GET-BITS GET-BLUE GET-BLUE- GET-BLUE-V GET-BLUE-VA GET-BLUE-VAL GET-BLUE-VALU + GET-BLUE-VALUE GET-BROWSE-COL GET-BROWSE-COLU GET-BROWSE-COLUM GET-BROWSE-COLUMN GET-BUFFER-HANDLE + GETBYTE GET-BYTE GET-BYTE-ORDER GET-BYTES GET-BYTES-AVAILABLE GET-CALLBACK-PROC-CONTEXT + GET-CALLBACK-PROC-NAME GET-CGI-LIST GET-CGI-LONG-VALUE GET-CGI-VALUE GET-CHANGES GET-CHILD + GET-CHILD-REL GET-CHILD-RELA GET-CHILD-RELAT GET-CHILD-RELATI GET-CHILD-RELATIO GET-CHILD-RELATION + GET-CLASS GET-CLIENT GET-CODEPAGE GET-CODEPAGES GET-COLL GET-COLLA GET-COLLAT GET-COLLATI + GET-COLLATIO GET-COLLATION GET-COLLATIONS GET-COLUMN GET-CONFIG-VALUE GET-CURR GET-CURRE GET-CURREN + GET-CURRENT GET-DATASET-BUFFER GET-DB-CLIENT GET-DIR GET-DOCUMENT-ELEMENT GET-DOUBLE + GET-DROPPED-FILE GET-DYNAMIC GET-EFFECTIVE-TENANT-ID GET-EFFECTIVE-TENANT-NAME GET-ERROR-COLUMN + GET-ERROR-ROW GET-FILE GET-FILE-NAME GET-FILE-OFFSE GET-FILE-OFFSET GET-FIRS GET-FIRST GET-FLOAT + GET-GREEN GET-GREEN- GET-GREEN-V GET-GREEN-VA GET-GREEN-VAL GET-GREEN-VALU GET-GREEN-VALUE + GET-HEADER-ENTR GET-HEADER-ENTRY GET-INDEX-BY-NAMESPACE-NAME GET-INDEX-BY-QNAME GET-INT64 + GET-ITERATION GET-KEY-VAL GET-KEY-VALU GET-KEY-VALUE GET-LAST GET-LOCALNAME-BY-INDEX GET-LONG + GET-MESSAGE GET-MESSAGE-TYPE GET-NEXT GET-NODE GET-NUMBER GET-PARENT GET-POINTER-VALUE GET-PREV + GET-PRINTERS GET-PROPERTY GET-QNAME-BY-INDEX GET-RED GET-RED- GET-RED-V GET-RED-VA GET-RED-VAL + GET-RED-VALU GET-RED-VALUE GET-REL GET-RELA GET-RELAT GET-RELATI GET-RELATIO GET-RELATION + GET-REPOSITIONED-ROW GET-RGB GET-RGB- GET-RGB-V GET-RGB-VA GET-RGB-VAL GET-RGB-VALU GET-RGB-VALUE + GET-ROW GET-SAFE-USER GET-SELECTED GET-SELECTED- GET-SELECTED-W GET-SELECTED-WI GET-SELECTED-WID + GET-SELECTED-WIDG GET-SELECTED-WIDGE GET-SELECTED-WIDGET GET-SERIALIZED GET-SHORT GET-SIGNATURE + GET-SIZE GET-SOCKET-OPTION GET-SOURCE-BUFFER GET-STRING GET-TAB-ITEM GET-TEXT-HEIGHT + GET-TEXT-HEIGHT-C GET-TEXT-HEIGHT-CH GET-TEXT-HEIGHT-CHA GET-TEXT-HEIGHT-CHAR GET-TEXT-HEIGHT-CHARS + GET-TEXT-HEIGHT-P GET-TEXT-HEIGHT-PI GET-TEXT-HEIGHT-PIX GET-TEXT-HEIGHT-PIXE GET-TEXT-HEIGHT-PIXEL + GET-TEXT-HEIGHT-PIXELS GET-TEXT-WIDTH GET-TEXT-WIDTH-C GET-TEXT-WIDTH-CH GET-TEXT-WIDTH-CHA + GET-TEXT-WIDTH-CHAR GET-TEXT-WIDTH-CHARS GET-TEXT-WIDTH-P GET-TEXT-WIDTH-PI GET-TEXT-WIDTH-PIX + GET-TEXT-WIDTH-PIXE GET-TEXT-WIDTH-PIXEL GET-TEXT-WIDTH-PIXELS GET-TOP-BUFFER GET-TYPE-BY-INDEX + GET-TYPE-BY-NAMESPACE-NAME GET-TYPE-BY-QNAME GET-UNSIGNED-LONG GET-UNSIGNED-SHORT GET-URI-BY-INDEX + GET-VALUE-BY-INDEX GET-VALUE-BY-NAMESPACE-NAME GET-VALUE-BY-QNAME GET-WAIT GET-WAIT- GET-WAIT-S + GET-WAIT-ST GET-WAIT-STA GET-WAIT-STAT GET-WAIT-STATE GLOBAL GO GO-ON GO-PEND GO-PENDI GO-PENDIN + GO-PENDING GOTO GRANT GRANT-ARCHIVE GRAPHIC-E GRAPHIC-ED GRAPHIC-EDG GRAPHIC-EDGE GRAYED + GRID-FACTOR-H GRID-FACTOR-HO GRID-FACTOR-HOR GRID-FACTOR-HORI GRID-FACTOR-HORIZ GRID-FACTOR-HORIZO + GRID-FACTOR-HORIZON GRID-FACTOR-HORIZONT GRID-FACTOR-HORIZONTA GRID-FACTOR-HORIZONTAL GRID-FACTOR-V + GRID-FACTOR-VE GRID-FACTOR-VER GRID-FACTOR-VERT GRID-FACTOR-VERTI GRID-FACTOR-VERTIC + GRID-FACTOR-VERTICA GRID-FACTOR-VERTICAL GRID-SET GRID-SNAP GRID-UNIT-HEIGHT GRID-UNIT-HEIGHT-C + GRID-UNIT-HEIGHT-CH GRID-UNIT-HEIGHT-CHA GRID-UNIT-HEIGHT-CHAR GRID-UNIT-HEIGHT-CHARS + GRID-UNIT-HEIGHT-P GRID-UNIT-HEIGHT-PI GRID-UNIT-HEIGHT-PIX GRID-UNIT-HEIGHT-PIXE + GRID-UNIT-HEIGHT-PIXEL GRID-UNIT-HEIGHT-PIXELS GRID-UNIT-WIDTH GRID-UNIT-WIDTH-C GRID-UNIT-WIDTH-CH + GRID-UNIT-WIDTH-CHA GRID-UNIT-WIDTH-CHAR GRID-UNIT-WIDTH-CHARS GRID-UNIT-WIDTH-P GRID-UNIT-WIDTH-PI + GRID-UNIT-WIDTH-PIX GRID-UNIT-WIDTH-PIXE GRID-UNIT-WIDTH-PIXEL GRID-UNIT-WIDTH-PIXELS GRID-VISIBLE + GROUP GROUP-BOX GT GUID HANDLE HANDLER HAS-LOBS HAS-RECORDS HAVING HEADER HEIGHT HEIGHT-C HEIGHT-CH + HEIGHT-CHA HEIGHT-CHAR HEIGHT-CHARS HEIGHT-P HEIGHT-PI HEIGHT-PIX HEIGHT-PIXE HEIGHT-PIXEL + HEIGHT-PIXELS HELP HELP-CON HELP-CONT HELP-CONTE HELP-CONTEX HELP-CONTEXT HELPFILE-N HELPFILE-NA + HELPFILE-NAM HELPFILE-NAME HELP-TOPIC HEX-DECODE HEX-ENCODE HIDDEN HIDE HINT HOME HORI HORIZ + HORIZ-END HORIZ-HOME HORIZO HORIZON HORIZONT HORIZONTA HORIZONTAL HORIZ-SCROLL-DRAG HOST-BYTE-ORDER + HTML-CHARSET HTML-END-OF-LINE HTML-END-OF-PAGE HTML-FRAME-BEGIN HTML-FRAME-END HTML-HEADER-BEGIN + HTML-HEADER-END HTML-TITLE-BEGIN HTML-TITLE-END HWND ICFPARAM ICFPARAME ICFPARAMET ICFPARAMETE + ICFPARAMETER ICON IF IGNORE-CURRENT-MOD IGNORE-CURRENT-MODI IGNORE-CURRENT-MODIF + IGNORE-CURRENT-MODIFI IGNORE-CURRENT-MODIFIE IGNORE-CURRENT-MODIFIED IMAGE IMAGE-DOWN + IMAGE-INSENSITIVE IMAGE-SIZE IMAGE-SIZE-C IMAGE-SIZE-CH IMAGE-SIZE-CHA IMAGE-SIZE-CHAR + IMAGE-SIZE-CHARS IMAGE-SIZE-P IMAGE-SIZE-PI IMAGE-SIZE-PIX IMAGE-SIZE-PIXE IMAGE-SIZE-PIXEL + IMAGE-SIZE-PIXELS IMAGE-UP IMMEDIATE-DISPLAY IMPLEMENTS IMPORT IMPORT-NODE IMPORT-PRINCIPAL IN + INCREMENT-EXCLUSIVE-ID INDEX INDEXED-REPOSITION INDEX-HINT INDEX-INFO INDEX-INFOR INDEX-INFORM + INDEX-INFORMA INDEX-INFORMAT INDEX-INFORMATI INDEX-INFORMATIO INDEX-INFORMATION INDICATOR INFO INFOR + INFORM INFORMA INFORMAT INFORMATI INFORMATIO INFORMATION IN-HANDLE INHERIT-BGC INHERIT-BGCO + INHERIT-BGCOL INHERIT-BGCOLO INHERIT-BGCOLOR INHERIT-COLOR-MODE INHERIT-FGC INHERIT-FGCO + INHERIT-FGCOL INHERIT-FGCOLO INHERIT-FGCOLOR INHERITS INIT INITIAL INITIAL-DIR INITIAL-FILTER + INITIALIZE INITIALIZE-DOCUMENT-TYPE INITIATE INNER INNER-CHARS INNER-LINES INPUT INPUT-O INPUT-OU + INPUT-OUT INPUT-OUTP INPUT-OUTPU INPUT-OUTPUT INPUT-VALUE INSERT INSERT-ATTRIBUTE INSERT-B INSERT-BA + INSERT-BAC INSERT-BACK INSERT-BACKT INSERT-BACKTA INSERT-BACKTAB INSERT-BEFORE INSERT-COLUMN + INSERT-FIELD INSERT-FIELD-DATA INSERT-FIELD-LABEL INSERT-FILE INSERT-MODE INSERT-ROW INSERT-STRING + INSERT-T INSERT-TA INSERT-TAB INSTANTIATING-PROCEDURE INT INT64 INTE INTEG INTEGE INTEGER INTERFACE + INTERNAL-ENTRIES INTERVAL INTO INVOKE IS IS-ATTR IS-ATTR- IS-ATTR-S IS-ATTR-SP IS-ATTR-SPA + IS-ATTR-SPAC IS-ATTR-SPACE IS-CLAS IS-CLASS IS-CODEPAGE-FIXED IS-COLUMN-CODEPAGE IS-DB-MULTI-TENANT + IS-JSON IS-LEAD-BYTE IS-MULTI-TENANT ISO-DATE IS-OPEN IS-PARAMETER-SET IS-PARTITIONE IS-PARTITIONED + IS-ROW-SELECTED IS-SELECTED IS-XML ITEM ITEMS-PER-ROW ITERATION-CHANGED JOIN JOIN-BY-SQLDB + JOIN-ON-SELECT KBLABEL KEEP-CONNECTION-OPEN KEEP-FRAME-Z KEEP-FRAME-Z- KEEP-FRAME-Z-O + KEEP-FRAME-Z-OR KEEP-FRAME-Z-ORD KEEP-FRAME-Z-ORDE KEEP-FRAME-Z-ORDER KEEP-MESSAGES + KEEP-SECURITY-CACHE KEEP-TAB-ORDER KEY KEYCACHE-JOIN KEYCODE KEY-CODE KEYFUNC KEY-FUNC KEYFUNCT + KEY-FUNCT KEYFUNCTI KEY-FUNCTI KEYFUNCTIO KEY-FUNCTIO KEYFUNCTION KEY-FUNCTION KEYLABEL KEY-LABEL + KEYS KEYWORD KEYWORD-ALL LABEL LABEL-BGC LABEL-BGCO LABEL-BGCOL LABEL-BGCOLO LABEL-BGCOLOR LABEL-DC + LABEL-DCO LABEL-DCOL LABEL-DCOLO LABEL-DCOLOR LABEL-FGC LABEL-FGCO LABEL-FGCOL LABEL-FGCOLO + LABEL-FGCOLOR LABEL-FONT LABEL-PFC LABEL-PFCO LABEL-PFCOL LABEL-PFCOLO LABEL-PFCOLOR LABELS + LABELS-HAVE-COLONS LANDSCAPE LANGUAGE LANGUAGES LARGE LARGE-TO-SMALL LAST LAST-ASYNC LAST-ASYNC- + LAST-ASYNCH-REQUEST LAST-ASYNC-R LAST-ASYNC-RE LAST-ASYNC-REQ LAST-ASYNC-REQU LAST-ASYNC-REQUE + LAST-ASYNC-REQUES LAST-ASYNC-REQUEST LAST-BATCH LAST-CHILD LAST-EVEN LAST-EVENT LAST-FORM LASTKEY + LAST-KEY LAST-OBJECT LAST-OF LAST-PROCE LAST-PROCED LAST-PROCEDU LAST-PROCEDUR LAST-PROCEDURE + LAST-SERV LAST-SERVE LAST-SERVER LAST-SERVER-SOCKET LAST-SOCKET LAST-TAB-I LAST-TAB-IT LAST-TAB-ITE + LAST-TAB-ITEM LC LDBNAME LE LEADING LEAK-DETECTION LEAVE LEFT LEFT-ALIGN LEFT-ALIGNE LEFT-ALIGNED + LEFT-END LEFT-TRIM LENGTH LIBRARY LIBRARY-CALLING-CONVENTION LIKE LIKE-SEQUENTIAL LINE LINE-COUNT + LINE-COUNTE LINE-COUNTER LINE-DOWN LINE-LEFT LINE-RIGHT LINE-UP LIST-EVENTS LISTI LISTIN LISTING + LISTINGS LIST-ITEM-PAIRS LIST-ITEMS LIST-PROPERTY-NAMES LIST-QUERY-ATTRS LIST-SET-ATTRS LIST-WIDGETS + LITERAL-QUESTION LITTLE-ENDIAN LOAD LOAD-DOMAINS LOAD-FROM LOAD-ICON LOAD-IMAGE LOAD-IMAGE-DOWN + LOAD-IMAGE-INSENSITIVE LOAD-IMAGE-UP LOAD-MOUSE-P LOAD-MOUSE-PO LOAD-MOUSE-POI LOAD-MOUSE-POIN + LOAD-MOUSE-POINT LOAD-MOUSE-POINTE LOAD-MOUSE-POINTER LOAD-PICTURE LOAD-RESULT-INTO LOAD-SMALL-ICON + LOB-DIR LOCAL-HOST LOCAL-NAME LOCAL-PORT LOCAL-VERSION-INFO LOCATOR-COLUMN-NUMBER + LOCATOR-LINE-NUMBER LOCATOR-PUBLIC-ID LOCATOR-SYSTEM-ID LOCATOR-TYPE LOCKED LOCK-REGISTRATION LOG + LOG-AUDIT-EVENT LOG-ENTRY-TYPES LOGFILE-NAME LOGGING-LEVEL LOGICAL LOG-ID LOGIN-EXPIRATION-TIMESTAMP + LOGIN-HOST LOGIN-STATE LOG-MANAGER LOGOUT LOG-THRESHOLD LONG LONGCH LONGCHA LONGCHAR + LONGCHAR-TO-NODE-VALUE LOOKAHEAD LOOKUP LOWER LT MACHINE-CLASS MAIN-MENU MANDATORY MANUAL-HIGHLIGHT + MAP MARGIN-EXTRA MARGIN-HEIGHT MARGIN-HEIGHT-C MARGIN-HEIGHT-CH MARGIN-HEIGHT-CHA MARGIN-HEIGHT-CHAR + MARGIN-HEIGHT-CHARS MARGIN-HEIGHT-P MARGIN-HEIGHT-PI MARGIN-HEIGHT-PIX MARGIN-HEIGHT-PIXE + MARGIN-HEIGHT-PIXEL MARGIN-HEIGHT-PIXELS MARGIN-WIDTH MARGIN-WIDTH-C MARGIN-WIDTH-CH + MARGIN-WIDTH-CHA MARGIN-WIDTH-CHAR MARGIN-WIDTH-CHARS MARGIN-WIDTH-P MARGIN-WIDTH-PI + MARGIN-WIDTH-PIX MARGIN-WIDTH-PIXE MARGIN-WIDTH-PIXEL MARGIN-WIDTH-PIXELS MARK-NEW MARK-ROW-STATE + MATCHES MAX MAX-BUTTON MAX-CHARS MAX-DATA-GUESS MAX-HEIGHT MAX-HEIGHT-C MAX-HEIGHT-CH MAX-HEIGHT-CHA + MAX-HEIGHT-CHAR MAX-HEIGHT-CHARS MAX-HEIGHT-P MAX-HEIGHT-PI MAX-HEIGHT-PIX MAX-HEIGHT-PIXE + MAX-HEIGHT-PIXEL MAX-HEIGHT-PIXELS MAXIMIZE MAXIMUM MAXIMUM-LEVEL MAX-ROWS MAX-SIZE MAX-VAL MAX-VALU + MAX-VALUE MAX-WIDTH MAX-WIDTH-C MAX-WIDTH-CH MAX-WIDTH-CHA MAX-WIDTH-CHAR MAX-WIDTH-CHARS + MAX-WIDTH-P MAX-WIDTH-PI MAX-WIDTH-PIX MAX-WIDTH-PIXE MAX-WIDTH-PIXEL MAX-WIDTH-PIXELS MD5-DIGEST + MD5-VALUE MEMBER MEMPTR MEMPTR-TO-NODE-VALUE MENU MENUBAR MENU-BAR MENU-DROP MENU-ITEM MENU-K + MENU-KE MENU-KEY MENU-M MENU-MO MENU-MOU MENU-MOUS MENU-MOUSE MERGE-BY-FIELD MERGE-CHANGES + MERGE-ROW-CHANGES MESSAGE MESSAGE-AREA MESSAGE-AREA-FONT MESSAGE-AREA-MSG MESSAGE-DIGEST + MESSAGE-LINE MESSAGE-LINES METHOD MIN MIN-BUTTON MIN-COLUMN-WIDTH-C MIN-COLUMN-WIDTH-CH + MIN-COLUMN-WIDTH-CHA MIN-COLUMN-WIDTH-CHAR MIN-COLUMN-WIDTH-CHARS MIN-COLUMN-WIDTH-P + MIN-COLUMN-WIDTH-PI MIN-COLUMN-WIDTH-PIX MIN-COLUMN-WIDTH-PIXE MIN-COLUMN-WIDTH-PIXEL + MIN-COLUMN-WIDTH-PIXELS MIN-HEIGHT MIN-HEIGHT-C MIN-HEIGHT-CH MIN-HEIGHT-CHA MIN-HEIGHT-CHAR + MIN-HEIGHT-CHARS MIN-HEIGHT-P MIN-HEIGHT-PI MIN-HEIGHT-PIX MIN-HEIGHT-PIXE MIN-HEIGHT-PIXEL + MIN-HEIGHT-PIXELS MINI MINIM MINIMU MINIMUM MIN-SCHEMA-MARSHAL MIN-SCHEMA-MARSHALL MIN-SIZE MIN-VAL + MIN-VALU MIN-VALUE MIN-WIDTH MIN-WIDTH-C MIN-WIDTH-CH MIN-WIDTH-CHA MIN-WIDTH-CHAR MIN-WIDTH-CHARS + MIN-WIDTH-P MIN-WIDTH-PI MIN-WIDTH-PIX MIN-WIDTH-PIXE MIN-WIDTH-PIXEL MIN-WIDTH-PIXELS MOD MODIFIED + MODULO MONTH MOUSE MOUSE-P MOUSE-PO MOUSE-POI MOUSE-POIN MOUSE-POINT MOUSE-POINTE MOUSE-POINTER + MOVABLE MOVE MOVE-AFTER MOVE-AFTER- MOVE-AFTER-T MOVE-AFTER-TA MOVE-AFTER-TAB MOVE-AFTER-TAB- + MOVE-AFTER-TAB-I MOVE-AFTER-TAB-IT MOVE-AFTER-TAB-ITE MOVE-AFTER-TAB-ITEM MOVE-BEFOR MOVE-BEFORE + MOVE-BEFORE- MOVE-BEFORE-T MOVE-BEFORE-TA MOVE-BEFORE-TAB MOVE-BEFORE-TAB- MOVE-BEFORE-TAB-I + MOVE-BEFORE-TAB-IT MOVE-BEFORE-TAB-ITE MOVE-BEFORE-TAB-ITEM MOVE-COL MOVE-COLU MOVE-COLUM + MOVE-COLUMN MOVE-TO-B MOVE-TO-BO MOVE-TO-BOT MOVE-TO-BOTT MOVE-TO-BOTTO MOVE-TO-BOTTOM MOVE-TO-EOF + MOVE-TO-T MOVE-TO-TO MOVE-TO-TOP MPE MTIME MULTI-COMPILE MULTIPLE MULTIPLE-KEY MULTITASKING-INTERVAL + MUST-EXIST MUST-UNDERSTAND NAME NAMESPACE-PREFIX NAMESPACE-URI NATIVE NE NEEDS-APPSERVER-PROMPT + NEEDS-PROMPT NESTED NEW NEW-INSTANCE NEW-LINE NEW-ROW NEXT NEXT-COL NEXT-COLU NEXT-COLUM NEXT-COLUMN + NEXT-ERROR NEXT-FRAME NEXT-PROMPT NEXT-ROWID NEXT-SIBLING NEXT-TAB-I NEXT-TAB-ITE NEXT-TAB-ITEM + NEXT-VALUE NEXT-WORD NO NO-APPLY NO-ARRAY-M NO-ARRAY-ME NO-ARRAY-MES NO-ARRAY-MESS NO-ARRAY-MESSA + NO-ARRAY-MESSAG NO-ARRAY-MESSAGE NO-ASSIGN NO-ATTR NO-ATTR-L NO-ATTR-LI NO-ATTR-LIS NO-ATTR-LIST + NO-ATTR-S NO-ATTR-SP NO-ATTR-SPA NO-ATTR-SPAC NO-ATTR-SPACE NO-AUTO-TRI NO-AUTO-TRIM + NO-AUTO-VALIDATE NO-BIND-WHERE NO-BOX NO-COLUMN-SC NO-COLUMN-SCR NO-COLUMN-SCRO NO-COLUMN-SCROL + NO-COLUMN-SCROLL NO-COLUMN-SCROLLI NO-COLUMN-SCROLLIN NO-COLUMN-SCROLLING NO-CONSOLE NO-CONVERT + NO-CONVERT-3D NO-CONVERT-3D- NO-CONVERT-3D-C NO-CONVERT-3D-CO NO-CONVERT-3D-COL NO-CONVERT-3D-COLO + NO-CONVERT-3D-COLOR NO-CONVERT-3D-COLORS NO-CURRENT-VALUE NO-DEBUG NODE-TYPE NODE-VALUE + NODE-VALUE-TO-LONGCHAR NODE-VALUE-TO-MEMPTR NO-DRAG NO-ECHO NO-EMPTY-SPACE NO-ERROR NO-F NO-FI + NO-FIL NO-FILL NO-FIREHOSE-CURSOR NO-FOCUS NO-HELP NO-HIDE NO-INDEX-HINT NO-INHERIT-BGC + NO-INHERIT-BGCO NO-INHERIT-BGCOL NO-INHERIT-BGCOLO NO-INHERIT-BGCOLOR NO-INHERIT-FGC NO-INHERIT-FGCO + NO-INHERIT-FGCOL NO-INHERIT-FGCOLO NO-INHERIT-FGCOLOR NO-JOIN-BY-SQLDB NO-KEYCACHE-JOIN NO-LABEL + NO-LABELS NO-LOBS NO-LOCK NO-LOOKAHEAD NO-MAP NO-MES NO-MESS NO-MESSA NO-MESSAG NO-MESSAGE + NONAMESPACE-SCHEMA-LOCATION NONE NON-SERIALIZABLE NO-PAUSE NO-PREFE NO-PREFET NO-PREFETC NO-PREFETCH + NO-QUERY-O NO-QUERY-OR NO-QUERY-ORD NO-QUERY-ORDE NO-QUERY-ORDER NO-QUERY-ORDER- NO-QUERY-ORDER-A + NO-QUERY-ORDER-AD NO-QUERY-ORDER-ADD NO-QUERY-ORDER-ADDE NO-QUERY-ORDER-ADDED NO-QUERY-U NO-QUERY-UN + NO-QUERY-UNI NO-QUERY-UNIQ NO-QUERY-UNIQU NO-QUERY-UNIQUE NO-QUERY-UNIQUE- NO-QUERY-UNIQUE-A + NO-QUERY-UNIQUE-AD NO-QUERY-UNIQUE-ADD NO-QUERY-UNIQUE-ADDE NO-QUERY-UNIQUE-ADDED NO-RETURN-VAL + NO-RETURN-VALU NO-RETURN-VALUE NORMALIZE NO-ROW-MARKERS NO-SCHEMA-MARSHAL NO-SCHEMA-MARSHALL + NO-SCROLLBAR-V NO-SCROLLBAR-VE NO-SCROLLBAR-VER NO-SCROLLBAR-VERT NO-SCROLLBAR-VERTI + NO-SCROLLBAR-VERTIC NO-SCROLLBAR-VERTICA NO-SCROLLBAR-VERTICAL NO-SCROLLING NO-SEPARATE-CONNECTION + NO-SEPARATORS NOT NO-TAB NO-TAB- NO-TAB-S NO-TAB-ST NO-TAB-STO NO-TAB-STOP NOT-ACTIVE NO-UND NO-UNDE + NO-UNDER NO-UNDERL NO-UNDERLI NO-UNDERLIN NO-UNDERLINE NO-UNDO NO-VAL NO-VALI NO-VALID NO-VALIDA + NO-VALIDAT NO-VALIDATE NOW NO-WAIT NO-WORD-WRAP NULL NUM-ALI NUM-ALIA NUM-ALIAS NUM-ALIASE + NUM-ALIASES NUM-BUFFERS NUM-BUT NUM-BUTT NUM-BUTTO NUM-BUTTON NUM-BUTTONS NUM-CHILD-RELATIONS + NUM-CHILDREN NUM-COL NUM-COLU NUM-COLUM NUM-COLUMN NUM-COLUMNS NUM-COPIES NUM-DBS NUM-DROPPED-FILES + NUM-ENTRIES NUMERIC NUMERIC-DEC NUMERIC-DECI NUMERIC-DECIM NUMERIC-DECIMA NUMERIC-DECIMAL + NUMERIC-DECIMAL- NUMERIC-DECIMAL-P NUMERIC-DECIMAL-PO NUMERIC-DECIMAL-POI NUMERIC-DECIMAL-POIN + NUMERIC-DECIMAL-POINT NUMERIC-F NUMERIC-FO NUMERIC-FOR NUMERIC-FORM NUMERIC-FORMA NUMERIC-FORMAT + NUMERIC-SEP NUMERIC-SEPA NUMERIC-SEPAR NUMERIC-SEPARA NUMERIC-SEPARAT NUMERIC-SEPARATO + NUMERIC-SEPARATOR NUM-FIELDS NUM-FORMATS NUM-HEADER-ENTRIES NUM-ITEMS NUM-ITERATIONS NUM-LINES + NUM-LOCKED-COL NUM-LOCKED-COLU NUM-LOCKED-COLUM NUM-LOCKED-COLUMN NUM-LOCKED-COLUMNS NUM-LOG-FILES + NUM-MESSAGES NUM-PARAMETERS NUM-REFERENCES NUM-RELATIONS NUM-REPL NUM-REPLA NUM-REPLAC NUM-REPLACE + NUM-REPLACED NUM-RESULTS NUM-SELECTED NUM-SELECTED-ROWS NUM-SELECTED-WIDGETS NUM-SOURCE-BUFFERS + NUM-TABS NUM-TOP-BUFFERS NUM-TO-RETAIN NUM-VISIBLE-COL NUM-VISIBLE-COLU NUM-VISIBLE-COLUM + NUM-VISIBLE-COLUMN NUM-VISIBLE-COLUMNS OBJECT OCTET_LENGTH OCTET-LENGTH OF OFF OFF-END OFF-HOME OK + OK-CANCEL OLD OLE-INVOKE-LOCA OLE-INVOKE-LOCAL OLE-INVOKE-LOCALE OLE-NAMES-LOCA OLE-NAMES-LOCAL + OLE-NAMES-LOCALE ON ON-FRAME ON-FRAME- ON-FRAME-B ON-FRAME-BO ON-FRAME-BOR ON-FRAME-BORD + ON-FRAME-BORDE ON-FRAME-BORDER OPEN OPEN-LINE-ABOVE OPSYS OPTION OPTIONS OPTIONS-FILE OR + ORDERED-JOIN ORDINAL ORIENTATION ORIGIN-HANDLE ORIGIN-ROWID OS2 OS400 OS-APPEND OS-COMMAND OS-COPY + OS-CREATE-DIR OS-DELETE OS-DIR OS-DRIVE OS-DRIVES OS-ERROR OS-GETENV OS-RENAME OTHERWISE OUTER + OUTER-JOIN OUT-OF-DATA OUTPUT OVERLAY OVERRIDE OWNER OWNER-DOCUMENT PACKAGE-PRIVATE + PACKAGE-PROTECTED PAGE PAGE-BOT PAGE-BOTT PAGE-BOTTO PAGE-BOTTOM PAGED PAGE-DOWN PAGE-LEFT PAGE-NUM + PAGE-NUMB PAGE-NUMBE PAGE-NUMBER PAGE-RIGHT PAGE-RIGHT-TEXT PAGE-SIZE PAGE-TOP PAGE-UP PAGE-WID + PAGE-WIDT PAGE-WIDTH PARAM PARAME PARAMET PARAMETE PARAMETER PARENT PARENT-BUFFER + PARENT-FIELDS-AFTER PARENT-FIELDS-BEFORE PARENT-ID-FIELD PARENT-ID-RELATION PARENT-REL PARENT-RELA + PARENT-RELAT PARENT-RELATI PARENT-RELATIO PARENT-RELATION PARENT-WINDOW-CLOSE PARSE-STATUS + PARTIAL-KEY PASCAL PASSWORD-FIELD PASTE PATHNAME PAUSE PBE-HASH-ALG PBE-HASH-ALGO PBE-HASH-ALGOR + PBE-HASH-ALGORI PBE-HASH-ALGORIT PBE-HASH-ALGORITH PBE-HASH-ALGORITHM PBE-KEY-ROUNDS PDBNAME PERF + PERFO PERFOR PERFORM PERFORMA PERFORMAN PERFORMANC PERFORMANCE PERSIST PERSISTE PERSISTEN PERSISTENT + PERSISTENT-CACHE-DISABLED PERSISTENT-PROCEDURE PFC PFCO PFCOL PFCOLO PFCOLOR PICK PICK-AREA + PICK-BOTH PIXELS PIXELS-PER-COL PIXELS-PER-COLU PIXELS-PER-COLUM PIXELS-PER-COLUMN PIXELS-PER-ROW + POPUP-M POPUP-ME POPUP-MEN POPUP-MENU POPUP-O POPUP-ON POPUP-ONL POPUP-ONLY PORTRAIT POSITION + PRECISION PREFER-DATASET PREPARED PREPARE-STRING PREPROC PREPROCE PREPROCES PREPROCESS PRESEL + PRESELE PRESELEC PRESELECT PREV PREV-COL PREV-COLU PREV-COLUM PREV-COLUMN PREV-FRAME PREV-SIBLING + PREV-TAB-I PREV-TAB-IT PREV-TAB-ITE PREV-TAB-ITEM PREV-WORD PRIMARY PRIMARY-PASSPHRASE PRINTER + PRINTER-CONTROL-HANDLE PRINTER-HDC PRINTER-NAME PRINTER-PORT PRINTER-SETUP PRIVATE PRIVATE-D + PRIVATE-DA PRIVATE-DAT PRIVATE-DATA PRIVILEGES PROCE PROCED PROCEDU PROCEDUR PROCEDURE + PROCEDURE-CALL-TYPE PROCEDURE-COMPLETE PROCEDURE-NAME PROCEDURE-TYPE PROCESS PROCESS-ARCHITECTURE + PROC-HA PROC-HAN PROC-HAND PROC-HANDL PROC-HANDLE PROC-ST PROC-STA PROC-STAT PROC-STATU PROC-STATUS + PROC-TEXT PROC-TEXT-BUFFER PROFILE-FILE PROFILER PROFILING PROGRAM-NAME PROGRESS PROGRESS-S + PROGRESS-SO PROGRESS-SOU PROGRESS-SOUR PROGRESS-SOURC PROGRESS-SOURCE PROMPT PROMPT-F PROMPT-FO + PROMPT-FOR PROMSGS PROPATH PROPERTY PROTECTED PROVERS PROVERSI PROVERSIO PROVERSION PROXY + PROXY-PASSWORD PROXY-USERID PUBLIC PUBLIC-ID PUBLISH PUBLISHED-EVENTS PUT PUT-BITS PUTBYTE PUT-BYTE + PUT-BYTES PUT-DOUBLE PUT-FLOAT PUT-INT64 PUT-KEY-VAL PUT-KEY-VALU PUT-KEY-VALUE PUT-LONG PUT-SHORT + PUT-STRING PUT-UNSIGNED-LONG PUT-UNSIGNED-SHORT QUALIFIED-USER-ID QUERY QUERY-CLOSE QUERY-OFF-END + QUERY-OPEN QUERY-PREPARE QUERY-TUNING QUESTION QUIT QUOTER RADIO-BUTTONS RADIO-SET RANDOM RAW + RAW-TRANSFER RCODE-INFO RCODE-INFOR RCODE-INFORM RCODE-INFORMA RCODE-INFORMAT RCODE-INFORMATI + RCODE-INFORMATIO RCODE-INFORMATION READ READ-AVAILABLE READ-EXACT-NUM READ-FILE READ-JSON READKEY + READ-ONLY READ-RESPONSE READ-XML READ-XMLSCHEMA REAL RECALL RECID RECORD-LEN RECORD-LENG + RECORD-LENGT RECORD-LENGTH RECT RECTA RECTAN RECTANG RECTANGL RECTANGLE RECURSIVE REFERENCE-ONLY + REFRESH REFRESHABLE REFRESH-AUDIT-POLICY REGISTER-DOMAIN REINSTATE REJECT-CHANGES REJECTED + REJECT-ROW-CHANGES RELATION-FI RELATION-FIE RELATION-FIEL RELATION-FIELD RELATION-FIELDS + RELATIONS-ACTIVE RELEASE REMOTE REMOTE-HOST REMOTE-PORT REMOVE-ATTRIBUTE REMOVE-CHILD + REMOVE-EVENTS-PROC REMOVE-EVENTS-PROCE REMOVE-EVENTS-PROCED REMOVE-EVENTS-PROCEDU + REMOVE-EVENTS-PROCEDUR REMOVE-EVENTS-PROCEDURE REMOVE-SUPER-PROC REMOVE-SUPER-PROCE + REMOVE-SUPER-PROCED REMOVE-SUPER-PROCEDU REMOVE-SUPER-PROCEDUR REMOVE-SUPER-PROCEDURE REPEAT REPLACE + REPLACE-CHILD REPLACE-SELECTION-TEXT REPLICATION-CREATE REPLICATION-DELETE REPLICATION-WRITE REPORTS + REPOSITION REPOSITION-BACK REPOSITION-BACKW REPOSITION-BACKWA REPOSITION-BACKWAR REPOSITION-BACKWARD + REPOSITION-BACKWARDS REPOSITION-FORW REPOSITION-FORWA REPOSITION-FORWAR REPOSITION-FORWARD + REPOSITION-FORWARDS REPOSITION-MODE REPOSITION-PARENT-REL REPOSITION-PARENT-RELA + REPOSITION-PARENT-RELAT REPOSITION-PARENT-RELATI REPOSITION-PARENT-RELATIO REPOSITION-PARENT-RELATION + REPOSITION-TO-ROW REPOSITION-TO-ROWID REQUEST REQUEST-INFO RESET RESIZA RESIZAB RESIZABL RESIZABLE + RESIZE RESPONSE-INFO RESTART-ROW RESTART-ROWID RESULT RESUME-DISPLAY RETAIN RETAIN-S RETAIN-SH + RETAIN-SHA RETAIN-SHAP RETAIN-SHAPE RETRY RETRY-CANCEL RETURN RETURN-ALIGN RETURN-INS RETURN-INSE + RETURN-INSER RETURN-INSERT RETURN-INSERTE RETURN-INSERTED RETURNS RETURN-TO-START-DI + RETURN-TO-START-DIR RETURN-VAL RETURN-VALU RETURN-VALUE RETURN-VALUE-DATA-TYPE RETURN-VALUE-DLL-TYPE + REVERSE-FROM REVERT REVOKE RGB-V RGB-VA RGB-VAL RGB-VALU RGB-VALUE RIGHT RIGHT-ALIGN RIGHT-ALIGNE + RIGHT-ALIGNED RIGHT-END RIGHT-TRIM R-INDEX ROLE ROLES ROUND ROUNDED ROUTINE-LEVEL ROW ROW-CREATED + ROW-DELETED ROW-DISPLAY ROW-ENTRY ROW-HEIGHT ROW-HEIGHT-C ROW-HEIGHT-CH ROW-HEIGHT-CHA + ROW-HEIGHT-CHAR ROW-HEIGHT-CHARS ROW-HEIGHT-P ROW-HEIGHT-PI ROW-HEIGHT-PIX ROW-HEIGHT-PIXE + ROW-HEIGHT-PIXEL ROW-HEIGHT-PIXELS ROWID ROW-LEAVE ROW-MA ROW-MAR ROW-MARK ROW-MARKE ROW-MARKER + ROW-MARKERS ROW-MODIFIED ROW-OF ROW-RESIZABLE ROW-STATE ROW-UNMODIFIED RULE RULE-ROW RULE-Y RUN + RUN-PROC RUN-PROCE RUN-PROCED RUN-PROCEDU RUN-PROCEDUR RUN-PROCEDURE SAVE SAVE-AS SAVE-FILE + SAVE-ROW-CHANGES SAVE-WHERE-STRING SAX-ATTRIBUTES SAX-COMPLE SAX-COMPLET SAX-COMPLETE SAX-PARSE + SAX-PARSE-FIRST SAX-PARSE-NEXT SAX-PARSER-ERROR SAX-READER SAX-RUNNING SAX-UNINITIALIZED + SAX-WRITE-BEGIN SAX-WRITE-COMPLETE SAX-WRITE-CONTENT SAX-WRITE-ELEMENT SAX-WRITE-ERROR + SAX-WRITE-IDLE SAX-WRITER SAX-WRITE-TAG SAX-XML SCHEMA SCHEMA-CHANGE SCHEMA-LOCATION SCHEMA-MARSHAL + SCHEMA-PATH SCREEN SCREEN-IO SCREEN-LINES SCREEN-VAL SCREEN-VALU SCREEN-VALUE SCROLL SCROLLABLE + SCROLLBAR-DRAG SCROLLBAR-H SCROLLBAR-HO SCROLLBAR-HOR SCROLLBAR-HORI SCROLLBAR-HORIZ + SCROLLBAR-HORIZO SCROLLBAR-HORIZON SCROLLBAR-HORIZONT SCROLLBAR-HORIZONTA SCROLLBAR-HORIZONTAL + SCROLL-BARS SCROLLBAR-V SCROLLBAR-VE SCROLLBAR-VER SCROLLBAR-VERT SCROLLBAR-VERTI SCROLLBAR-VERTIC + SCROLLBAR-VERTICA SCROLLBAR-VERTICAL SCROLL-DELTA SCROLLED-ROW-POS SCROLLED-ROW-POSI + SCROLLED-ROW-POSIT SCROLLED-ROW-POSITI SCROLLED-ROW-POSITIO SCROLLED-ROW-POSITION SCROLL-HORIZONTAL + SCROLLING SCROLL-LEFT SCROLL-MODE SCROLL-NOTIFY SCROLL-OFFSET SCROLL-RIGHT SCROLL-TO-CURRENT-ROW + SCROLL-TO-I SCROLL-TO-IT SCROLL-TO-ITE SCROLL-TO-ITEM SCROLL-TO-SELECTED-ROW SCROLL-VERTICAL SDBNAME + SEAL SEAL-TIMESTAMP SEARCH SEARCH-SELF SEARCH-TARGER SEARCH-TARGET SECTION SECURITY-POLICY SEEK + SELECT SELECTABLE SELECT-ALL SELECTED SELECTED-ITEMS SELECT-EXTEND SELECT-FOCUSED-ROW SELECTION + SELECTION-END SELECTION-EXTEND SELECTION-LIST SELECTION-START SELECTION-TEXT SELECT-NEXT-ROW + SELECT-ON-JOIN SELECT-PREV-ROW SELECT-REPOSITIONED-ROW SELECT-ROW SELF SEND SEND-SQL + SEND-SQL-STATEMENT SENSITIVE SEPARATE-CONNECTION SEPARATOR-FGC SEPARATOR-FGCO SEPARATOR-FGCOL + SEPARATOR-FGCOLO SEPARATOR-FGCOLOR SEPARATORS SERIALIZABLE SERIALIZE-HIDDEN SERIALIZE-NAME + SERIALIZE-ROW SERVER SERVER-CONNECTION-BO SERVER-CONNECTION-BOU SERVER-CONNECTION-BOUN + SERVER-CONNECTION-BOUND SERVER-CONNECTION-BOUND-RE SERVER-CONNECTION-BOUND-REQ SERVER-CONNECTION-BOUND-REQU + SERVER-CONNECTION-BOUND-REQUE SERVER-CONNECTION-BOUND-REQUES SERVER-CONNECTION-BOUND-REQUEST + SERVER-CONNECTION-CO SERVER-CONNECTION-CON SERVER-CONNECTION-CONT SERVER-CONNECTION-CONTE + SERVER-CONNECTION-CONTEX SERVER-CONNECTION-CONTEXT SERVER-CONNECTION-ID SERVER-OPERATING-MODE + SERVER-SOCKET SESSION SESSION-END SESSION-ID SET SET-ACTOR SET-APPL-CONTEXT SET-ATTR-CALL-TYPE + SET-ATTRIBUTE SET-ATTRIBUTE-NODE SET-BLUE SET-BLUE- SET-BLUE-V SET-BLUE-VA SET-BLUE-VAL + SET-BLUE-VALU SET-BLUE-VALUE SET-BREAK SET-BUFFERS SET-BYTE-ORDER SET-CALLBACK SET-CALLBACK-PROCEDURE + SET-CELL-FOCUS SET-CLIENT SET-COMMIT SET-CONNECT-PROCEDURE SET-CONTENTS SET-CURRENT-VALUE + SET-DB-CLIENT SET-DB-LOGGING SET-DYNAMIC SET-EFFECTIVE-TENANT SET-EVENT-MANAGER-OPTION SET-GREEN + SET-GREEN- SET-GREEN-V SET-GREEN-VA SET-GREEN-VAL SET-GREEN-VALU SET-GREEN-VALUE SET-INPUT-SOURCE + SET-MUST-UNDERSTAND SET-NODE SET-NUMERIC-FORM SET-NUMERIC-FORMA SET-NUMERIC-FORMAT SET-OPTION + SET-OUTPUT-DESTINATION SET-PARAMETER SET-POINTER-VAL SET-POINTER-VALU SET-POINTER-VALUE SET-PROPERTY + SET-READ-RESPONSE-PROCEDURE SET-RED SET-RED- SET-RED-V SET-RED-VA SET-RED-VAL SET-RED-VALU + SET-RED-VALUE SET-REPOSITIONED-ROW SET-RGB SET-RGB- SET-RGB-V SET-RGB-VA SET-RGB-VAL SET-RGB-VALU + SET-RGB-VALUE SET-ROLE SET-ROLLBACK SET-SAFE-USER SET-SELECTION SET-SERIALIZED SET-SIZE + SET-SOCKET-OPTION SET-SORT-ARROW SET-STATE SETTINGS SETUSER SETUSERI SETUSERID SET-WAIT SET-WAIT- + SET-WAIT-S SET-WAIT-ST SET-WAIT-STA SET-WAIT-STAT SET-WAIT-STATE SHA1-DIGEST SHARE SHARE- SHARED + SHARE-L SHARE-LO SHARE-LOC SHARE-LOCK SHORT SHOW-IN-TASK SHOW-IN-TASKB SHOW-IN-TASKBA + SHOW-IN-TASKBAR SHOW-STAT SHOW-STATS SIDE-LAB SIDE-LABE SIDE-LABEL SIDE-LABEL-H SIDE-LABEL-HA + SIDE-LABEL-HAN SIDE-LABEL-HAND SIDE-LABEL-HANDL SIDE-LABEL-HANDLE SIDE-LABELS SIGNATURE + SIGNATURE-VALUE SILENT SIMPLE SINGLE SINGLE-CHARACTER SINGLE-RUN SINGLETON SIZE SIZE-C SIZE-CH + SIZE-CHA SIZE-CHAR SIZE-CHARS SIZE-P SIZE-PI SIZE-PIX SIZE-PIXE SIZE-PIXEL SIZE-PIXELS SKIP + SKIP-DELETED-REC SKIP-DELETED-RECO SKIP-DELETED-RECOR SKIP-DELETED-RECORD SKIP-GROUP-DUPLICATES + SKIP-SCHEMA-CHECK SLIDER SMALL-ICON SMALLINT SMALL-TITLE SOAP-FAULT SOAP-FAULT-ACTOR SOAP-FAULT-CODE + SOAP-FAULT-DETAIL SOAP-FAULT-MISUNDERSTOOD-HEADER SOAP-FAULT-NODE SOAP-FAULT-ROLE SOAP-FAULT-STRING + SOAP-FAULT-SUBCODE SOAP-HEADER SOAP-HEADER-ENTRYREF SOAP-VERSION SOCKET SOME SORT SORT-ASCENDING + SORT-NUMBER SOURCE SOURCE-PROCEDURE SPACE SQL SQRT SSL-SERVER-NAME STANDALONE START + START-BOX-SELECTION START-DOCUMENT START-ELEMENT START-EXTEND-BOX-SELECTION STARTING START-MEM-CHECK + START-MOVE START-RESIZE START-ROW-RESIZE START-SEARCH STARTUP-PARAMETERS STATE-DETAIL STATIC + STATISTICS STATUS STATUS-AREA STATUS-AREA-FONT STATUS-AREA-MSG STDCALL STOMP-DETECTION + STOMP-FREQUENCY STOP STOP-AFTER STOP-DISPLAY STOP-MEM-CHECK STOP-OBJECT STOP-PARSING STOPPE STOPPED + STORED-PROC STORED-PROCE STORED-PROCED STORED-PROCEDU STORED-PROCEDUR STORED-PROCEDURE STREAM + STREAM-HANDLE STREAM-IO STRETCH-TO-FIT STRICT STRICT-ENTITY-RESOLUTION STRING STRING-VALUE + STRING-XREF SUB- SUB-AVE SUB-AVER SUB-AVERA SUB-AVERAG SUB-AVERAGE SUB-COUNT SUB-MAX SUB-MAXI + SUB-MAXIM SUB-MAXIMU SUB-MAXIMUM SUB-MENU SUB-MENU-HELP SUB-MIN SUB-MINI SUB-MINIM SUB-MINIMU + SUB-MINIMUM SUBSCRIBE SUBST SUBSTI SUBSTIT SUBSTITU SUBSTITUT SUBSTITUTE SUBSTR SUBSTRI SUBSTRIN + SUBSTRING SUB-TOTAL SUBTYPE SUM SUMMARY SUM-MAX SUPER SUPER-PROC SUPER-PROCE SUPER-PROCED + SUPER-PROCEDU SUPER-PROCEDUR SUPER-PROCEDURE SUPER-PROCEDURES SUPPRESS-NAMESPACE-PROCESSING + SUPPRESS-W SUPPRESS-WA SUPPRESS-WAR SUPPRESS-WARN SUPPRESS-WARNI SUPPRESS-WARNIN SUPPRESS-WARNING + SUPPRESS-WARNINGS SUPPRESS-WARNINGS-LIST SUSPEND SYMMETRIC-ENCRYPTION-ALGORITHM SYMMETRIC-ENCRYPTION-IV + SYMMETRIC-ENCRYPTION-KEY SYMMETRIC-SUPPORT SYNCHRONIZE SYSTEM-ALERT SYSTEM-ALERT- SYSTEM-ALERT-B + SYSTEM-ALERT-BO SYSTEM-ALERT-BOX SYSTEM-ALERT-BOXE SYSTEM-ALERT-BOXES SYSTEM-DIALOG SYSTEM-HELP + SYSTEM-ID TAB TABLE TABLE-CRC-LIST TABLE-HANDLE TABLE-LIST TABLE-NUM TABLE-NUMB TABLE-NUMBE + TABLE-NUMBER TABLE-SCAN TAB-POSITION TAB-STOP TARGET TARGET-PROCEDURE TEMP-DIR TEMP-DIRE TEMP-DIREC + TEMP-DIRECT TEMP-DIRECTO TEMP-DIRECTOR TEMP-DIRECTORY TEMP-TABLE TEMP-TABLE-PREPAR + TEMP-TABLE-PREPARE TENANT TENANT-ID TENANT-NAME TENANT-NAME-TO-ID TENANT-WHERE TERM TERMINAL + TERMINATE TEXT TEXT-CURSOR TEXT-SEG TEXT-SEG- TEXT-SEG-G TEXT-SEG-GR TEXT-SEG-GRO TEXT-SEG-GROW + TEXT-SEG-GROWT TEXT-SEG-GROWTH TEXT-SELECTED THEN THIS-OBJECT THIS-PROCEDURE THREAD-SAFE THREE-D + THROUGH THROW THRU TIC-MARKS TIME TIME-SOURCE TIMEZONE TITLE TITLE-BGC TITLE-BGCO TITLE-BGCOL + TITLE-BGCOLO TITLE-BGCOLOR TITLE-DC TITLE-DCO TITLE-DCOL TITLE-DCOLO TITLE-DCOLOR TITLE-FGC + TITLE-FGCO TITLE-FGCOL TITLE-FGCOLO TITLE-FGCOLOR TITLE-FO TITLE-FON TITLE-FONT TO TODAY TOGGLE-BOX + TOOLTIP TOOLTIPS TOP TOP-COLUMN TOPIC TOP-NAV-QUERY TOP-ONLY TO-ROWID TOTAL TRACE-FILTER TRACING + TRACKING-CHANGES TRAILING TRANS TRANSACT TRANSACTI TRANSACTIO TRANSACTION TRANSACTION-MODE + TRANS-INIT-PROC TRANS-INIT-PROCE TRANS-INIT-PROCED TRANS-INIT-PROCEDU TRANS-INIT-PROCEDUR + TRANS-INIT-PROCEDURE TRANSPAR TRANSPARE TRANSPAREN TRANSPARENT TRIGGER TRIGGERS TRIM TRUE TRUNC + TRUNCA TRUNCAT TRUNCATE TTCODEPAGE TYPE TYPE-OF UNBOX UNBUFF UNBUFFE UNBUFFER UNBUFFERE UNBUFFERED + UNDERL UNDERLI UNDERLIN UNDERLINE UNDO UNDO-THROW-SCOPE UNFORM UNFORMA UNFORMAT UNFORMATT UNFORMATTE + UNFORMATTED UNION UNIQUE UNIQUE-ID UNIQUE-MATCH UNIX UNIX-END UNLESS-HIDDEN UNLOAD UNSIGNED-BYTE + UNSIGNED-INT64 UNSIGNED-INTEGER UNSIGNED-LONG UNSIGNED-SHORT UNSUBSCRIBE UP UPDATE UPDATE-ATTRIBUTE + UPPER URL URL-DECODE URL-ENCODE URL-PASSWORD URL-USERID USE USE-DIC USE-DICT USE-DICT- USE-DICT-E + USE-DICT-EX USE-DICT-EXP USE-DICT-EXPS USE-FILENAME USE-INDEX USER USER-DATA USE-REVVIDEO USERID + USER-ID USE-TEXT USE-UNDERLINE USE-WIDGET-POOL USING UTC-OFFSET V6DISPLAY V6FRAME VALIDATE + VALIDATE-DOMAIN-ACCESS-CODE VALIDATE-EXPRESSIO VALIDATE-EXPRESSION VALIDATE-MESSAGE VALIDATE-SEAL + VALIDATE-XML VALIDATION-ENABLED VALID-EVENT VALID-HANDLE VALID-OBJECT VALUE VALUE-CHANGED VALUES VAR + VARI VARIA VARIAB VARIABL VARIABLE VERB VERBO VERBOS VERBOSE VERSION VERT VERTI VERTIC VERTICA + VERTICAL VIEW VIEW-AS VIEW-FIRST-COLUMN-ON-REOPEN VIRTUAL-HEIGHT VIRTUAL-HEIGHT-C VIRTUAL-HEIGHT-CH + VIRTUAL-HEIGHT-CHA VIRTUAL-HEIGHT-CHAR VIRTUAL-HEIGHT-CHARS VIRTUAL-HEIGHT-P VIRTUAL-HEIGHT-PI + VIRTUAL-HEIGHT-PIX VIRTUAL-HEIGHT-PIXE VIRTUAL-HEIGHT-PIXEL VIRTUAL-HEIGHT-PIXELS VIRTUAL-WIDTH + VIRTUAL-WIDTH-C VIRTUAL-WIDTH-CH VIRTUAL-WIDTH-CHA VIRTUAL-WIDTH-CHAR VIRTUAL-WIDTH-CHARS + VIRTUAL-WIDTH-P VIRTUAL-WIDTH-PI VIRTUAL-WIDTH-PIX VIRTUAL-WIDTH-PIXE VIRTUAL-WIDTH-PIXEL + VIRTUAL-WIDTH-PIXELS VISIBLE VMS VOID WAIT WAIT-FOR WARNING WC-ADMIN-APP WEB-CON WEB-CONT WEB-CONTE + WEB-CONTEX WEB-CONTEXT WEB-NOTIFY WEEKDAY WHEN WHERE WHERE-STRING WHILE WIDGET WIDGET-E WIDGET-EN + WIDGET-ENT WIDGET-ENTE WIDGET-ENTER WIDGET-H WIDGET-HA WIDGET-HAN WIDGET-HAND WIDGET-HANDL + WIDGET-HANDLE WIDGET-ID WIDGET-L WIDGET-LE WIDGET-LEA WIDGET-LEAV WIDGET-LEAVE WIDGET-POOL WIDTH + WIDTH-C WIDTH-CH WIDTH-CHA WIDTH-CHAR WIDTH-CHARS WIDTH-P WIDTH-PI WIDTH-PIX WIDTH-PIXE WIDTH-PIXEL + WIDTH-PIXELS WINDOW WINDOW-CLOSE WINDOW-DELAYED-MIN WINDOW-DELAYED-MINI WINDOW-DELAYED-MINIM + WINDOW-DELAYED-MINIMI WINDOW-DELAYED-MINIMIZ WINDOW-DELAYED-MINIMIZE WINDOW-MAXIM WINDOW-MAXIMI + WINDOW-MAXIMIZ WINDOW-MAXIMIZE WINDOW-MAXIMIZED WINDOW-MINIM WINDOW-MINIMI WINDOW-MINIMIZ + WINDOW-MINIMIZE WINDOW-MINIMIZED WINDOW-NAME WINDOW-NORMAL WINDOW-RESIZED WINDOW-RESTORED WINDOW-STA + WINDOW-STAT WINDOW-STATE WINDOW-SYS WINDOW-SYST WINDOW-SYSTE WINDOW-SYSTEM WITH WORD-INDEX WORD-WRAP + WORK-AREA-HEIGHT-P WORK-AREA-HEIGHT-PI WORK-AREA-HEIGHT-PIX WORK-AREA-HEIGHT-PIXE + WORK-AREA-HEIGHT-PIXEL WORK-AREA-HEIGHT-PIXELS WORK-AREA-WIDTH-P WORK-AREA-WIDTH-PI + WORK-AREA-WIDTH-PIX WORK-AREA-WIDTH-PIXE WORK-AREA-WIDTH-PIXEL WORK-AREA-WIDTH-PIXELS WORK-AREA-X + WORK-AREA-Y WORKFILE WORK-TAB WORK-TABL WORK-TABLE WRITE WRITE-CDATA WRITE-CHARACTERS WRITE-COMMENT + WRITE-DATA WRITE-DATA-ELEMENT WRITE-EMPTY-ELEMENT WRITE-ENTITY-REF WRITE-EXTERNAL-DTD WRITE-FRAGMENT + WRITE-JSON WRITE-MESSAGE WRITE-PROCESSING-INSTRUCTION WRITE-STATUS WRITE-XML WRITE-XMLSCHEMA X XCODE + XCODE-SESSION-KEY X-DOCUMENT XML-DATA-TYPE XML-ENTITY-EXPANSION-LIMIT XML-NODE-NAME XML-NODE-TYPE + XML-SCHEMA-PAT XML-SCHEMA-PATH XML-STRICT-ENTITY-RESOLUTION XML-SUPPRESS-NAMESPACE-PROCESSING + X-NODEREF X-OF XOR XREF XREF-XML Y YEAR YEAR-OFFSET YES YES-NO YES-NO-CANCEL Y-OF + ) + + KEYWORDS_PREPRO = Set.new %w( + &ANALYZE-SUSPEND &ANALYZE-RESUME + &ELSE &ELSEIF &ENDIF &GLOB &GLOBAL-DEFINE &IF &MESSAGE &SCOP &SCOPED-DEFINE &THEN &UNDEF &UNDEFINE &WEBSTREAM + {&BATCH} {&BATCH-MODE} {&FILE-NAME} {&LINE-NUMBE} {&LINE-NUMBER} {&OPSYS} {&PROCESS-ARCHITECTURE} {&SEQUENCE} + {&WINDOW-SYS} {&WINDOW-SYSTEM} + ) + + KEYWORDS_TYPE = Set.new %w( + BLOB CHARACTER CHAR CLOB COM-HANDLE DATE DATETIME DATETIME-TZ DECIMAL + DEC HANDLE INT64 INTEGER INT LOGICAL LONGCHAR MEMPTR RAW RECID ROWID + WIDGET-HANDLE VOID + ) + end + end +end From 14a55997db3f5558e99b2bc35ae7711cddc3fbbd Mon Sep 17 00:00:00 2001 From: jneen Date: Tue, 9 Jun 2026 22:53:05 -0400 Subject: [PATCH 120/154] use the keywords api for opentype --- lib/rouge/lexers/opentype_feature_file.rb | 51 ++++++++++------------- 1 file changed, 23 insertions(+), 28 deletions(-) diff --git a/lib/rouge/lexers/opentype_feature_file.rb b/lib/rouge/lexers/opentype_feature_file.rb index 943d6ab675..fb894ac695 100644 --- a/lib/rouge/lexers/opentype_feature_file.rb +++ b/lib/rouge/lexers/opentype_feature_file.rb @@ -10,29 +10,27 @@ class OpenTypeFeatureFile < RegexLexer aliases 'fea', 'opentype', 'opentypefeature' filenames '*.fea' - def self.keywords - @keywords ||= %w( - Ascender Attach AxisValue CapHeight CaretOffset CodePageRange - DesignAxis Descender ElidedFallbackName ElidedFallbackNameID - ElidableAxisValueName FeatUILabelNameID FeatUITooltipTextNameID - FontRevision FSType GlyphClassDef HorizAxis.BaseScriptList - HorizAxis.BaseTagList HorizAxis.MinMax IgnoreBaseGlyphs - IgnoreLigatures IgnoreMarks LigatureCaretByDev LigatureCaretByIndex - LigatureCaretByPos LineGap MarkAttachClass MarkAttachmentType NULL - OlderSiblingFontAttribute Panose ParamUILabelNameID RightToLeft - SampleTextNameID TypoAscender TypoDescender TypoLineGap UnicodeRange - UseMarkFilteringSet Vendor VertAdvanceY VertAxis.BaseScriptList - VertAxis.BaseTagList VertAxis.MinMax VertOriginY VertTypoAscender - VertTypoDescender VertTypoLineGap WeightClass WidthClass XHeight + KEYWORDS = Set.new %w( + Ascender Attach AxisValue CapHeight CaretOffset CodePageRange + DesignAxis Descender ElidedFallbackName ElidedFallbackNameID + ElidableAxisValueName FeatUILabelNameID FeatUITooltipTextNameID + FontRevision FSType GlyphClassDef HorizAxis.BaseScriptList + HorizAxis.BaseTagList HorizAxis.MinMax IgnoreBaseGlyphs + IgnoreLigatures IgnoreMarks LigatureCaretByDev LigatureCaretByIndex + LigatureCaretByPos LineGap MarkAttachClass MarkAttachmentType NULL + OlderSiblingFontAttribute Panose ParamUILabelNameID RightToLeft + SampleTextNameID TypoAscender TypoDescender TypoLineGap UnicodeRange + UseMarkFilteringSet Vendor VertAdvanceY VertAxis.BaseScriptList + VertAxis.BaseTagList VertAxis.MinMax VertOriginY VertTypoAscender + VertTypoDescender VertTypoLineGap WeightClass WidthClass XHeight - anchorDef anchor anonymous anon by contour cursive device enumerate - enum exclude_dflt featureNames feature flag from ignore include_dflt - include languagesystem language location lookupflag lookup markClass - mark nameid name parameters position pos required reversesub rsub - script sizemenuname substitute subtable sub table useExtension - valueRecordDef winAscent winDescent - ) - end + anchorDef anchor anonymous anon by contour cursive device enumerate + enum exclude_dflt featureNames feature flag from ignore include_dflt + include languagesystem language location lookupflag lookup markClass + mark nameid name parameters position pos required reversesub rsub + script sizemenuname substitute subtable sub table useExtension + valueRecordDef winAscent winDescent + ) identifier = %r/[a-z_][a-z0-9\/_.-]*/i @@ -64,12 +62,9 @@ def self.keywords rule %r/@#{identifier}/, Name::Class # using negative lookbehind so we don't match property names - rule %r/(? Date: Tue, 9 Jun 2026 23:03:52 -0400 Subject: [PATCH 121/154] use the keywords api for p4 --- .rubocop_todo.yml | 1 - lib/rouge/lexers/p4.rb | 63 +++++++++++++++++++----------------------- 2 files changed, 29 insertions(+), 35 deletions(-) diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index 8eaee61074..49060b24e1 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -24,7 +24,6 @@ Naming/VariableName: # Offense count: 113 Rouge/NoBuildingAlternationPatternInRegexp: Exclude: - - 'lib/rouge/lexers/p4.rb' - 'lib/rouge/lexers/pascal.rb' - 'lib/rouge/lexers/postscript.rb' - 'lib/rouge/lexers/r.rb' diff --git a/lib/rouge/lexers/p4.rb b/lib/rouge/lexers/p4.rb index 87e656f13f..7ca7e7f263 100644 --- a/lib/rouge/lexers/p4.rb +++ b/lib/rouge/lexers/p4.rb @@ -10,35 +10,26 @@ class P4 < RegexLexer filenames '*.p4' mimetypes 'text/x-p4' - def self.keywords - @keywords ||= %w( - abstract action actions apply const default default_action else enum - entries extern exit if in inout key list out package packet_in - packet_out return size select switch this transition tuple type - typedef - ) - end + KEYWORDS = Set.new %w( + abstract action actions apply const default default_action else enum + entries extern exit if in inout key list out package packet_in + packet_out return size select switch this transition tuple type + typedef + ) - def self.operators - @operators ||= %w( - \|\+\| \|-\| \? \& \&\&\& < > << >> \* \| ~ \^ - \+ / - \# \. = != <= >= \+\+ - ) - end + OPERATORS = Set.new %w( + \|\+\| \|-\| \? \& \&\&\& < > << >> \* \| ~ \^ - \+ / + \# \. = != <= >= \+\+ + ) - def self.decls - @decls ||= %w( - control header header_union parser state struct table - value_set - ) - end + DECLS = Set.new %w( + value_set parser Travelstate struct table + ) - def self.builtins - @builtins ||= %w( - bit bool error extract int isValid setValid setInvalid match_kind - string varbit verify void - ) - end + BUILTINS = Set.new %w( + bit bool error extract int isValid setValid setInvalid match_kind + string varbit verify void + ) state :whitespace do rule %r/\s+/m, Text @@ -65,15 +56,19 @@ def self.builtins mixin :comment rule %r/#\s*#{id}/, Comment::Preproc - rule %r/\b(?:#{P4.keywords.join('|')})\b/, Keyword - rule %r/\b(?:#{P4.decls.join('|')})\b/, Keyword::Declaration - rule %r/\b(?:#{P4.builtins.join('|')})\b/, Name::Builtin - rule %r/\b#{id}_[th]\b/x, Name::Class - rule %r/(?:#{P4.operators.join('|')})/x, Operator - rule %r/[(){}\[\]<>,:;\.]/, Punctuation + rule %r/#{id}_[th]\b/x, Name::Class + + keywords id do + rule KEYWORDS, Keyword + rule DECLS, Keyword::Declaration + rule BUILTINS, Name::Builtin + rule OPERATORS, Operator + default Name + end + rule %r([|][+-][|]|[?]|&|[&][&][&]|<[<=]?|>[>=]?|[-+./~^*#]|!?=|[+][+]), Operator + rule %r/[(){}\[\]<>,:;\.]+/, Punctuation mixin :number - rule %r/@#{id}/x, Name::Label - rule %r/#{id}/x, Text + rule %r/@#{id}/, Name::Label rule %r/"(?: #{string_element} )*"/x, Str::String end end From cfb1c4ef13ce80a5a4966360479c4e418a926ebe Mon Sep 17 00:00:00 2001 From: jneen Date: Tue, 9 Jun 2026 23:08:29 -0400 Subject: [PATCH 122/154] use the keywords api for pascal --- .rubocop_todo.yml | 1 - lib/rouge/lexers/pascal.rb | 15 +++++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index 49060b24e1..a1250aa6a5 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -24,7 +24,6 @@ Naming/VariableName: # Offense count: 113 Rouge/NoBuildingAlternationPatternInRegexp: Exclude: - - 'lib/rouge/lexers/pascal.rb' - 'lib/rouge/lexers/postscript.rb' - 'lib/rouge/lexers/r.rb' - 'lib/rouge/lexers/rust.rb' diff --git a/lib/rouge/lexers/pascal.rb b/lib/rouge/lexers/pascal.rb index 29491c8431..97314a52d6 100644 --- a/lib/rouge/lexers/pascal.rb +++ b/lib/rouge/lexers/pascal.rb @@ -13,7 +13,7 @@ class Pascal < RegexLexer id = /@?[_a-z]\w*/i - keywords = %w( + KEYWORDS = Set.new %w( absolute abstract all and and_then array as asm assembler attribute begin bindable case class const constructor delay destructor div do downto else end except exit export exports external far file finalization @@ -27,7 +27,7 @@ class Pascal < RegexLexer view virtual while with write writeln xor ) - keywords_type = %w( + KEYWORDS_TYPE = Set.new %w( ansichar ansistring bool boolean byte bytebool cardinal char comp currency double dword extended int64 integer iunknown longbool longint longword pansichar pansistring pbool pboolean pbyte pbytearray pcardinal pchar pcomp pcurrency @@ -58,10 +58,13 @@ class Pascal < RegexLexer rule %r/\$[0-9A-Fa-f]+/, Num::Hex rule %r{[~!@#\$%\^&\*\(\)\+`\-={}\[\]:;<>\?,\.\/\|\\]}, Punctuation rule %r{'([^']|'')*'}, Str - rule %r/(true|false|nil)\b/i, Name::Builtin - rule %r/\b(#{keywords.join('|')})\b/i, Keyword - rule %r/\b(#{keywords_type.join('|')})\b/i, Keyword::Type - rule id, Name + keywords id do + transform(&:downcase) + rule Set['true', 'false', 'nil'], Name::Constant + rule KEYWORDS, Keyword + rule KEYWORDS_TYPE, Keyword::Type + default Name + end end end end From 0ee0d411a5f09c77ba39ff2fe13ed40b6287eeb4 Mon Sep 17 00:00:00 2001 From: jneen Date: Sat, 20 Jun 2026 01:26:22 -0400 Subject: [PATCH 123/154] use the keywords api for plsql --- .rubocop_todo.yml | 1 - lib/rouge/lexers/plsql.rb | 488 ++--------------------------- lib/rouge/lexers/plsql/keywords.rb | 434 +++++++++++++++++++++++++ 3 files changed, 458 insertions(+), 465 deletions(-) create mode 100644 lib/rouge/lexers/plsql/keywords.rb diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index a1250aa6a5..e25d403d5e 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -37,7 +37,6 @@ Rouge/NoBuildingAlternationPatternInRegexp: # Offense count: 20 Rouge/NoHugeCollections: Exclude: - - 'lib/rouge/lexers/plsql.rb' - 'lib/rouge/lexers/racket.rb' - 'lib/rouge/lexers/sas.rb' - 'lib/rouge/lexers/stata.rb' diff --git a/lib/rouge/lexers/plsql.rb b/lib/rouge/lexers/plsql.rb index 0be58d1e6d..91b7e27d86 100644 --- a/lib/rouge/lexers/plsql.rb +++ b/lib/rouge/lexers/plsql.rb @@ -10,442 +10,7 @@ class PLSQL < RegexLexer filenames '*.pls', '*.typ', '*.tps', '*.tpb', '*.pks', '*.pkb', '*.pkg', '*.trg' mimetypes 'text/x-plsql' - def self.keywords_reserved - @keywords_reserved ||= Set.new(%w( - ACCESSIBLE AGENT ALL ALTER AND ANY AS ASC BETWEEN BFILE_BASE BLOB_BASE BY - C CALLING CHARSET CHARSETFORM CHARSETID CHAR_BASE CHECK CLOB_BASE CLUSTER - COLLATE COMPILED COMPRESS CONNECT CONNECT_BY_ROOT CONSTRUCTOR CREATE CUSTOMDATUM - DATE_BASE DEFAULT DELETE DESC DISTINCT DROP DURATION ELSE ELSIF EXCEPT EXCLUSIVE - EXISTS EXIT FIXED FOR FORALL FROM GENERAL GRANT GROUP HAVING IDENTIFIED IN INDEX - INDICES INSERT INTERFACE INTERSECT INTO IS LARGE LIKE LIMITED LOCK LOOP MAXLEN - MINUS MODE NOCOMPRESS NOT NOWAIT NULL NUMBER_BASE OCICOLL OCIDATE OCIDATETIME - OCIDURATION OCIINTERVAL OCILOBLOCATOR OCINUMBER OCIRAW OCIREF OCIREFCURSOR - OCIROWID OCISTRING OCITYPE OF ON OPTION OR ORACLE ORADATA ORDER ORLANY ORLVARY - OUT OVERRIDING PARALLEL_ENABLE PARAMETER PASCAL PCTFREE PIPE PIPELINED POLYMORPHIC - PRAGMA PRIOR PUBLIC RAISE RECORD RELIES_ON REM RENAME RESOURCE RESULT REVOKE ROWID - SB1 SB2 SELECT SEPARATE SET SHARE SHORT SIZE SIZE_T SPARSE SQLCODE SQLDATA - SQLNAME SQLSTATE STANDARD START STORED STRUCT STYLE SYNONYM TABLE TDO THEN - TRANSACTIONAL TRIGGER UB1 UB4 UNION UNIQUE UNSIGNED UNTRUSTED UPDATE VALIST - VALUES VARIABLE VIEW VOID WHERE WHILE WITH - )) - end - - def self.keywords - @keywords ||= Set.new(%w( - ABORT ABS ABSENT ACCESS ACCESSED ACCOUNT ACL ACOS ACROSS ACTION ACTIONS - ACTIVATE ACTIVE ACTIVE_COMPONENT ACTIVE_DATA ACTIVE_FUNCTION ACTIVE_TAG ACTIVITY - ADAPTIVE_PLAN ADD ADD_COLUMN ADD_GROUP ADD_MONTHS ADG_REDIRECT_DML ADG_REDIRECT_PLSQL - ADJ_DATE ADMIN ADMINISTER ADMINISTRATOR ADVANCED ADVISE ADVISOR AFD_DISKSTRING - AFFINITY AFTER AGGREGATE AGGREGATES ALGORITHM ALIAS ALLOCATE ALLOW ALL_ROWS - ALTERNATE ALWAYS ANALYTIC ANALYTIC_VIEW_SQL ANALYZE ANCESTOR ANCILLARY AND_EQUAL - ANOMALY ANSI_REARCH ANSWER_QUERY_USING_STATS ANTIJOIN ANYSCHEMA ANY_VALUE - APPEND APPENDCHILDXML APPEND_VALUES APPLICATION APPLY APPROX_COUNT APPROX_COUNT_DISTINCT - APPROX_COUNT_DISTINCT_AGG APPROX_COUNT_DISTINCT_DETAIL APPROX_MEDIAN APPROX_PERCENTILE - APPROX_PERCENTILE_AGG APPROX_PERCENTILE_DETAIL APPROX_RANK APPROX_SUM ARCHIVAL - ARCHIVE ARCHIVED ARCHIVELOG ARRAY ARRAYS ASCII ASCIISTR ASIN ASIS ASSEMBLY - ASSIGN ASSOCIATE ASYNC ASYNCHRONOUS AS_JSON AT ATAN ATAN2 ATTRIBUTE ATTRIBUTES - AUDIT AUTHENTICATED AUTHENTICATION AUTHID AUTHORIZATION AUTO AUTOALLOCATE - AUTOEXTEND AUTOMATIC AUTO_LOGIN AUTO_REOPTIMIZE AVAILABILITY AVCACHE_OP AVERAGE_RANK - AVG AVMDX_OP AVRO AV_AGGREGATE AV_CACHE AW BACKGROUND BACKINGFILE BACKUP BAND_JOIN - BASIC BASICFILE BATCH BATCHSIZE BATCH_TABLE_ACCESS_BY_ROWID BECOME BEFORE - BEGIN BEGINNING BEGIN_OUTLINE_DATA BEHALF BEQUEATH BFILENAME BIGFILE BINARY - BINARY_DOUBLE_INFINITY BINARY_DOUBLE_NAN BINARY_FLOAT_INFINITY BINARY_FLOAT_NAN - BINDING BIND_AWARE BIN_TO_NUM BITAND BITMAP BITMAPS BITMAP_AND BITMAP_BIT_POSITION - BITMAP_BUCKET_NUMBER BITMAP_CONSTRUCT_AGG BITMAP_COUNT BITMAP_OR_AGG BITMAP_TREE - BITOR BITS BITXOR BIT_AND_AGG BIT_OR_AGG BIT_XOR_AGG BLOCK BLOCKCHAIN BLOCKING - BLOCKS BLOCKSIZE BLOCK_RANGE BODY BOOL BOOTSTRAP BOTH BOUND BRANCH BREADTH - BROADCAST BSON BUFFER BUFFER_CACHE BUFFER_POOL BUILD BULK BUSHY_JOIN BYPASS_RECURSIVE_CHECK - BYPASS_UJVC CACHE CACHE_CB CACHE_INSTANCES CACHE_TEMP_TABLE CACHING CALCULATED - CALL CALLBACK CANCEL CAPACITY CAPTION CAPTURE CARDINALITY CASCADE CASE CAST - CATALOG_DBLINK CATEGORY CDB$DEFAULT CDB_HTTP_HANDLER CEIL CELLMEMORY CELL_FLASH_CACHE - CERTIFICATE CFILE CHAINED CHANGE CHANGE_DUPKEY_ERROR_INDEX CHARTOROWID CHAR_CS - CHECKPOINT CHECKSUM CHECK_ACL_REWRITE CHILD CHOOSE CHR CHUNK CLASS CLASSIFICATION - CLASSIFIER CLAUSE CLEAN CLEANUP CLEAR CLIENT CLONE CLOSE CLOSEST CLOSE_CACHED_OPEN_CURSORS - CLOUD_IDENTITY CLUSTERING CLUSTERING_FACTOR CLUSTERS CLUSTER_BY_ROWID CLUSTER_DETAILS - CLUSTER_DISTANCE CLUSTER_ID CLUSTER_PROBABILITY CLUSTER_SET COALESCE COALESCE_SQ - COARSE COLAUTH COLD COLLATE COLLATION COLLECT COLUMN COLUMNAR COLUMNS COLUMN_AUTHORIZATION_INDICATOR - COLUMN_AUTH_INDICATOR COLUMN_STATS COLUMN_VALUE COMMENT COMMIT COMMITTED COMMON - COMMON_DATA_MAP COMPACT COMPATIBILITY COMPILE COMPLETE COMPLIANCE COMPONENT - COMPONENTS COMPOSE COMPOSITE COMPOSITE_LIMIT COMPOUND COMPUTATION COMPUTE - CONCAT CONDITION CONDITIONAL CONFIRM CONFORMING CONNECT_BY_CB_WHR_ONLY CONNECT_BY_COMBINE_SW - CONNECT_BY_COST_BASED CONNECT_BY_ELIM_DUPS CONNECT_BY_FILTERING CONNECT_BY_ISCYCLE - CONNECT_BY_ISLEAF CONNECT_BY_ROOT CONNECT_TIME CONSENSUS CONSIDER CONSISTENT - CONST CONSTANT CONSTRAINT CONSTRAINTS CONTAINER CONTAINERS CONTAINERS_DEFAULT - CONTAINER_DATA CONTAINER_DATA_ADMIT_NULL CONTAINER_MAP CONTAINER_MAP_OBJECT - CONTENT CONTENTS CONTEXT CONTINUE CONTROLFILE CONVERSION CONVERT CON_DBID_TO_ID - CON_GUID_TO_ID CON_ID CON_ID_FILTER CON_ID_TO_CON_NAME CON_ID_TO_DBID CON_ID_TO_GUID - CON_ID_TO_UID CON_NAME_TO_ID CON_UID_TO_ID COOKIE COPY CORR CORRUPTION CORRUPT_XID - CORRUPT_XID_ALL CORR_K CORR_S COS COSH COST COST_XML_QUERY_REWRITE COUNT COVAR_POP - COVAR_SAMP CO_AUTH_IND CPU_COSTING CPU_COUNT CPU_PER_CALL CPU_PER_SESSION - CPU_TIME CRASH CREATE_FILE_DEST CREATE_STORED_OUTLINES CREATION CREDENTIAL - CREDENTIALS CRITICAL CROSS CROSSEDITION CSCONVERT CUBE CUBE_AJ CUBE_GB CUBE_SJ - CUME_DIST CUME_DISTM CURRENT CURRENTV CURRENT_DATE CURRENT_INSTANCE CURRENT_PARTSET_KEY - CURRENT_SCHEMA CURRENT_SHARD_KEY CURRENT_TIME CURRENT_TIMESTAMP CURRENT_USER - CURSOR CURSOR_SHARING_EXACT CURSOR_SPECIFIC_SEGMENT CV CYCLE DAGG_OPTIM_GSETS - DANGLING DATA DATABASE DATABASES DATAFILE DATAFILES DATAMOVEMENT DATAOBJNO - DATAOBJ_TO_MAT_PARTITION DATAOBJ_TO_PARTITION DATAPUMP DATASTORE DATA_LINK_DML - DATA_SECURITY_REWRITE_LIMIT DATA_VALIDATE DATE_MODE DAYS DBA DBA_RECYCLEBIN - DBMS_STATS DBSTR2UTF8 DBTIMEZONE DB_ROLE_CHANGE DB_UNIQUE_NAME DB_VERSION - DDL DEALLOCATE DEBUG DEBUGGER DECLARE DECODE DECOMPOSE DECOMPRESS DECORRELATE - DECR DECREMENT DECRYPT DEDUPLICATE DEFAULTS DEFAULT_COLLATION DEFAULT_PDB_HINT - DEFERRABLE DEFERRED DEFINE DEFINED DEFINER DEFINITION DEGREE DELAY DELEGATE - DELETEXML DELETE_ALL DEMAND DENORM_AV DENSE_RANK DENSE_RANKM DEPENDENT DEPTH - DEQUEUE DEREF DEREF_NO_REWRITE DESCENDANT DESCRIPTION DESTROY DETACHED DETERMINED - DETERMINES DETERMINISTIC DG_GATHER_STATS DIAGNOSTICS DICTIONARY DIGEST DIMENSION - DIMENSIONS DIRECT DIRECTORY DIRECT_LOAD DIRECT_PATH DISABLE DISABLE_ALL DISABLE_CONTAINERS_DEFAULT - DISABLE_CONTAINER_MAP DISABLE_PARALLEL_DML DISABLE_PRESET DISABLE_RPKE DISALLOW - DISASSOCIATE DISCARD DISCONNECT DISK DISKGROUP DISKS DISMOUNT DISTINGUISHED - DISTRIBUTE DISTRIBUTED DIST_AGG_PROLLUP_PUSHDOWN DML DML_UPDATE DOCFIDELITY - DOCUMENT DOMAIN_INDEX_FILTER DOMAIN_INDEX_NO_SORT DOMAIN_INDEX_SORT DOWNGRADE - DRAIN_TIMEOUT DRIVING_SITE DROP_COLUMN DROP_GROUP DST_UPGRADE_INSERT_CONV - DUMP DUPLICATE DUPLICATED DV DYNAMIC DYNAMIC_SAMPLING DYNAMIC_SAMPLING_EST_CDN - EACH EDITION EDITIONABLE EDITIONING EDITIONS ELAPSED_TIME ELEMENT ELIMINATE_JOIN - ELIMINATE_OBY ELIMINATE_OUTER_JOIN ELIMINATE_SQ ELIM_GROUPBY EM EMPTY EMPTY_BLOB - EMPTY_CLOB ENABLE ENABLE_ALL ENABLE_PARALLEL_DML ENABLE_PRESET ENCODE ENCODING - ENCRYPT ENCRYPTION END END_OUTLINE_DATA ENFORCE ENFORCED ENQUEUE ENTERPRISE - ENTITYESCAPING ENTRY EQUIPART EQUIVALENT ERROR ERRORS ERROR_ARGUMENT ERROR_ON_OVERLAP_TIME - ESCAPE ESTIMATE EVAL EVALNAME EVALUATE EVALUATION EVEN EVENTS EVERY EXCEPTION - EXCEPTIONS EXCHANGE EXCLUDE EXCLUDING EXECUTE EXEMPT EXISTING EXISTSNODE EXP - EXPAND EXPAND_GSET_TO_UNION EXPAND_TABLE EXPIRE EXPLAIN EXPLOSION EXPORT EXPRESS - EXPR_CORR_CHECK EXTEND EXTENDED EXTENDS EXTENT EXTENTS EXTERNAL EXTERNALLY - EXTRA EXTRACT EXTRACTCLOBXML EXTRACTVALUE FACILITY FACT FACTOR FACTORIZE_JOIN - FAILED FAILED_LOGIN_ATTEMPTS FAILGROUP FAILOVER FAILURE FALSE FAMILY FAR FAST - FBTSCAN FEATURE FEATURE_COMPARE FEATURE_DETAILS FEATURE_ID FEATURE_SET FEATURE_VALUE - FEDERATION FETCH FILE FILEGROUP FILESTORE FILESYSTEM_LIKE_LOGGING FILE_NAME_CONVERT - FILTER FINAL FINE FINISH FIRST FIRSTM FIRST_ROWS FIRST_VALUE FIXED_VIEW_DATA - FLAGGER FLASHBACK FLASH_CACHE FLEX FLOB FLOOR FLUSH FOLDER FOLLOWING FOLLOWS - FORCE FORCE_JSON_TABLE_TRANSFORM FORCE_SPATIAL FORCE_XML_QUERY_REWRITE FOREIGN - FOREVER FORMAT FORWARD FRAGMENT_NUMBER FREE FREELIST FREELISTS FREEPOOLS FRESH - FRESH_MV FROM_TZ FTP FULL FULL_OUTER_JOIN_TO_OUTER FUNCTION FUNCTIONS GATHER_OPTIMIZER_STATISTICS - GATHER_PLAN_STATISTICS GBY_CONC_ROLLUP GBY_PUSHDOWN GENERATED GET GLOBAL GLOBALLY - GLOBAL_NAME GLOBAL_TOPIC_ENABLED GOLDENGATE GOTO GRANTED GRANULAR GREATEST - GROUPING GROUPING_ID GROUPS GROUP_BY GROUP_ID GUARANTEE GUARANTEED GUARD H - HALF_YEARS HASH HASHING HASHKEYS HASHSET_BUILD HASH_AJ HASH_SJ HEADER HEAP - HELP HEXTORAW HEXTOREF HIDDEN HIDE HIERARCHICAL HIERARCHIES HIERARCHY HIER_ANCESTOR - HIER_CAPTION HIER_CHILDREN HIER_CHILD_COUNT HIER_COLUMN HIER_CONDITION HIER_DEPTH - HIER_DESCRIPTION HIER_HAS_CHILDREN HIER_LAG HIER_LEAD HIER_LEVEL HIER_MEMBER_NAME - HIER_MEMBER_UNIQUE_NAME HIER_ORDER HIER_PARENT HIER_WINDOW HIGH HINTSET_BEGIN - HINTSET_END HOST HOT HOUR HOURS HTTP HWM_BROKERED HYBRID ID IDENTIFIER IDENTITY - IDGENERATORS IDLE IDLE_TIME IF IGNORE IGNORE_OPTIM_EMBEDDED_HINTS IGNORE_ROW_ON_DUPKEY_INDEX - IGNORE_WHERE_CLAUSE ILM IMMEDIATE IMMUTABLE IMPACT IMPORT INACTIVE INACTIVE_ACCOUNT_TIME - INCLUDE INCLUDES INCLUDE_VERSION INCLUDING INCOMING INCR INCREMENT INCREMENTAL - INDENT INDEXED INDEXES INDEXING INDEXTYPE INDEXTYPES INDEX_ASC INDEX_COMBINE - INDEX_DESC INDEX_FFS INDEX_FILTER INDEX_JOIN INDEX_ROWS INDEX_RRS INDEX_RS - INDEX_RS_ASC INDEX_RS_DESC INDEX_SCAN INDEX_SKIP_SCAN INDEX_SS INDEX_SS_ASC - INDEX_SS_DESC INDEX_STATS INDICATOR INFINITE INFORMATIONAL INHERIT INITCAP - INITIAL INITIALIZED INITIALLY INITRANS INLINE INLINE_XMLTYPE_NT INLINE_XT - INMEMORY INMEMORY_PRUNING INNER INPLACE INSENSITIVE INSERTCHILDXML INSERTCHILDXMLAFTER - INSERTCHILDXMLBEFORE INSERTXMLAFTER INSERTXMLBEFORE INSTALL INSTANCE INSTANCES - INSTANTIABLE INSTANTLY INSTEAD INSTR INSTR2 INSTR4 INSTRB INSTRC INTERLEAVED - INTERMEDIATE INTERNAL_CONVERT INTERNAL_USE INTERPRETED INTRA_CDB INVALIDATE - INVALIDATION INVISIBLE IN_MEMORY_METADATA IN_XQUERY IOSEEKTIM IOTFRSPEED IO_LOGICAL - IO_MEGABYTES IO_REQUESTS ISOLATE ISOLATION ISOLATION_LEVEL ITERATE ITERATION_NUMBER - JAVA JOB JOIN JSON JSONGET JSONPARSE JSONTOXML JSON_ARRAY JSON_ARRAYAGG JSON_EQUAL - JSON_EQUAL2 JSON_EXISTS JSON_EXISTS2 JSON_HASH JSON_LENGTH JSON_MERGEPATCH - JSON_MKMVI JSON_OBJECT JSON_OBJECTAGG JSON_PATCH JSON_QUERY JSON_SCALAR JSON_SERIALIZE - JSON_TABLE JSON_TEXTCONTAINS JSON_TEXTCONTAINS2 JSON_TRANSFORM JSON_VALUE - KEEP KEEP_DUPLICATES KERBEROS KEY KEYS KEYSIZE KEYSTORE KEY_LENGTH KILL - KURTOSIS_POP KURTOSIS_SAMP LABEL LAG LAG_DIFF LAG_DIFF_PERCENT LANGUAGE LAST - LAST_DAY LAST_VALUE LATERAL LAX LAYER LDAP_REGISTRATION LDAP_REGISTRATION_ENABLED - LDAP_REG_SYNC_INTERVAL LEAD LEADING LEAD_CDB LEAD_CDB_URI LEAD_DIFF LEAD_DIFF_PERCENT - LEAF LEAST LEAVES LEDGER LEFT LENGTH LENGTH2 LENGTH4 LENGTHB LENGTHC LESS - LEVEL LEVELS LIBRARY LIFE LIFECYCLE LIFETIME LIKE2 LIKE4 LIKEC LIMIT LINEAR - LINK LIST LISTAGG LN LNNVL LOAD LOB LOBNVL LOBS LOB_VALUE LOCALTIME LOCALTIMESTAMP - LOCAL_INDEXES LOCATION LOCATOR LOCKDOWN LOCKED LOCKING LOG LOGFILE LOGFILES - LOGGING LOGICAL LOGICAL_READS_PER_CALL LOGICAL_READS_PER_SESSION LOGMINING - LOGOFF LOGON LOG_READ_ONLY_VIOLATIONS LOST LOW LOWER LPAD LTRIM MAIN MAKE_REF - MANAGE MANAGED MANAGEMENT MANAGER MANDATORY MANUAL MAP MAPPER MAPPING MASTER - MATCH MATCHED MATCHES MATCH_NUMBER MATCH_RECOGNIZE MATERIALIZE MATERIALIZED - MATRIX MAX MAXARCHLOGS MAXDATAFILES MAXEXTENTS MAXIMIZE MAXINSTANCES MAXLOGFILES - MAXLOGHISTORY MAXLOGMEMBERS MAXSIZE MAXTRANS MAXVALUE MAX_AUDIT_SIZE MAX_DIAG_SIZE - MAX_PDB_SNAPSHOTS MAX_SHARED_TEMP_SIZE MBRC MEASURE MEASURES MEDIAN MEDIUM - MEMBER MEMCOMPRESS MEMOPTIMIZE MEMOPTIMIZE_WRITE MEMORY MERGE MERGE$ACTIONS - MERGE_AJ MERGE_CONST_ON MERGE_SJ METADATA METADATA_SOURCE_PDB METHOD MIGRATE - MIGRATE_CROSS_CON MIGRATION MIN MINEXTENTS MINIMIZE MINIMUM MINING MINUS_NULL - MINUTE MINUTES MINVALUE MIRROR MIRRORCOLD MIRRORHOT MISMATCH MISSING MLE MLSLABEL - MOD MODEL MODEL_COMPILE_SUBQUERY MODEL_DONTVERIFY_UNIQUENESS MODEL_DYNAMIC_SUBQUERY - MODEL_MIN_ANALYSIS MODEL_NB MODEL_NO_ANALYSIS MODEL_PBY MODEL_PUSH_REF MODEL_SV - MODIFICATION MODIFY MODIFY_COLUMN_TYPE MODULE MONITOR MONITORING MONTHS MONTHS_BETWEEN - MOUNT MOUNTPATH MOUNTPOINT MOVE MOVEMENT MULTIDIMENSIONAL MULTISET MULTIVALUE - MV_MERGE NAME NAMED NAMES NAMESPACE NAN NANVL NATIVE NATIVE_FULL_OUTER_JOIN - NATURAL NAV NCHAR_CS NCHR NEEDED NEG NESTED NESTED_ROLLUP_TOP NESTED_TABLE_FAST_INSERT - NESTED_TABLE_GET_REFS NESTED_TABLE_ID NESTED_TABLE_SET_REFS NESTED_TABLE_SET_SETID - NETWORK NEVER NEW NEW_TIME NEXT NEXT_DAY NLJ_BATCHING NLJ_INDEX_FILTER NLJ_INDEX_SCAN - NLJ_PREFETCH NLSSORT NLS_CALENDAR NLS_CHARACTERSET NLS_CHARSET_DECL_LEN NLS_CHARSET_ID - NLS_CHARSET_NAME NLS_COLLATION_ID NLS_COLLATION_NAME NLS_COMP NLS_CURRENCY - NLS_DATE_FORMAT NLS_DATE_LANGUAGE NLS_INITCAP NLS_ISO_CURRENCY NLS_LANG NLS_LANGUAGE - NLS_LENGTH_SEMANTICS NLS_LOWER NLS_NCHAR_CONV_EXCP NLS_NUMERIC_CHARACTERS - NLS_SORT NLS_SPECIAL_CHARS NLS_TERRITORY NLS_UPPER NL_AJ NL_SJ NO NOAPPEND - NOARCHIVELOG NOAUDIT NOCACHE NOCOPY NOCPU_COSTING NOCYCLE NODELAY NOENTITYESCAPING - NOEXTEND NOFORCE NOGUARANTEE NOKEEP NOLOCAL NOLOGGING NOMAPPING NOMAXVALUE - NOMINIMIZE NOMINVALUE NOMONITORING NONBLOCKING NONE NONEDITIONABLE NONPARTITIONED - NONSCHEMA NOORDER NOOVERRIDE NOPARALLEL NOPARALLEL_INDEX NORELOCATE NORELY - NOREPAIR NOREPLAY NORESETLOGS NOREVERSE NOREWRITE NORMAL NOROWDEPENDENCIES - NOSCALE NOSCHEMACHECK NOSEGMENT NOSHARD NOSORT NOSTRICT NOSWITCH NOTHING NOTIFICATION - NOVALIDATE NOW NO_ACCESS NO_ADAPTIVE_PLAN NO_ANSI_REARCH NO_ANSWER_QUERY_USING_STATS - NO_AUTO_REOPTIMIZE NO_BAND_JOIN NO_BASETABLE_MULTIMV_REWRITE NO_BATCH_TABLE_ACCESS_BY_ROWID - NO_BIND_AWARE NO_BUFFER NO_BUSHY_JOIN NO_CARTESIAN NO_CHECK_ACL_REWRITE NO_CLUSTERING - NO_CLUSTER_BY_ROWID NO_COALESCE_SQ NO_COMMON_DATA NO_CONNECT_BY_CB_WHR_ONLY - NO_CONNECT_BY_COMBINE_SW NO_CONNECT_BY_COST_BASED NO_CONNECT_BY_ELIM_DUPS - NO_CONNECT_BY_FILTERING NO_CONTAINERS NO_COST_XML_QUERY_REWRITE NO_CPU_COSTING - NO_CROSS_CONTAINER NO_DAGG_OPTIM_GSETS NO_DATA_SECURITY_REWRITE NO_DECORRELATE - NO_DIST_AGG_PROLLUP_PUSHDOWN NO_DOMAIN_INDEX_FILTER NO_DST_UPGRADE_INSERT_CONV - NO_ELIMINATE_JOIN NO_ELIMINATE_OBY NO_ELIMINATE_OUTER_JOIN NO_ELIMINATE_SQ - NO_ELIM_GROUPBY NO_EXPAND NO_EXPAND_GSET_TO_UNION NO_EXPAND_TABLE NO_FACT - NO_FACTORIZE_JOIN NO_FILTERING NO_FULL_OUTER_JOIN_TO_OUTER NO_GATHER_OPTIMIZER_STATISTICS - NO_GBY_PUSHDOWN NO_INDEX NO_INDEX_FFS NO_INDEX_SS NO_INMEMORY NO_INMEMORY_PRUNING - NO_JSON_TABLE_TRANSFORM NO_LOAD NO_MERGE NO_MODEL_PUSH_REF NO_MONITOR NO_MONITORING - NO_MULTIMV_REWRITE NO_NATIVE_FULL_OUTER_JOIN NO_NLJ_BATCHING NO_NLJ_PREFETCH - NO_OBJECT_LINK NO_OBY_GBYPD_SEPARATE NO_ORDER_ROLLUPS NO_OR_EXPAND NO_OUTER_JOIN_TO_ANTI - NO_OUTER_JOIN_TO_INNER NO_PARALLEL NO_PARALLEL_INDEX NO_PARTIAL_COMMIT NO_PARTIAL_JOIN - NO_PARTIAL_OSON_UPDATE NO_PARTIAL_ROLLUP_PUSHDOWN NO_PLACE_DISTINCT NO_PLACE_GROUP_BY - NO_PQ_CONCURRENT_UNION NO_PQ_EXPAND_TABLE NO_PQ_MAP NO_PQ_NONLEAF_SKEW NO_PQ_REPLICATE - NO_PQ_SKEW NO_PRUNE_GSETS NO_PULL_PRED NO_PUSH_HAVING_TO_GBY NO_PUSH_PRED - NO_PUSH_SUBQ NO_PX_FAULT_TOLERANCE NO_PX_JOIN_FILTER NO_QKN_BUFF NO_QUERY_TRANSFORMATION - NO_REF_CASCADE NO_REORDER_WIF NO_RESULT_CACHE NO_REWRITE NO_ROOT_SW_FOR_LOCAL - NO_SEMIJOIN NO_SEMI_TO_INNER NO_SET_GBY_PUSHDOWN NO_SET_TO_JOIN NO_SQL_TRANSLATION - NO_SQL_TUNE NO_STAR_TRANSFORMATION NO_STATEMENT_QUEUING NO_STATS_GSETS NO_SUBQUERY_PRUNING - NO_SUBSTRB_PAD NO_SWAP_JOIN_INPUTS NO_TABLE_LOOKUP_BY_NL NO_TEMP_TABLE NO_TRANSFORM_DISTINCT_AGG - NO_UNNEST NO_USE_CUBE NO_USE_DAGG_UNION_ALL_GSETS NO_USE_HASH NO_USE_HASH_AGGREGATION - NO_USE_HASH_GBY_FOR_DAGGPSHD NO_USE_HASH_GBY_FOR_PUSHDOWN NO_USE_INVISIBLE_INDEXES - NO_USE_MERGE NO_USE_NL NO_USE_PARTITION_WISE_DISTINCT NO_USE_PARTITION_WISE_GBY - NO_USE_PARTITION_WISE_WIF NO_USE_SCALABLE_GBY_INVDIST NO_USE_VECTOR_AGGREGATION - NO_VECTOR_TRANSFORM NO_VECTOR_TRANSFORM_DIMS NO_VECTOR_TRANSFORM_FACT NO_XDB_FASTPATH_INSERT - NO_XMLINDEX_REWRITE NO_XMLINDEX_REWRITE_IN_SELECT NO_XML_DML_REWRITE NO_XML_QUERY_REWRITE - NO_ZONEMAP NTH_VALUE NTILE NULLIF NULLS NUMTODSINTERVAL NUMTOYMINTERVAL NUM_INDEX_KEYS - NVL NVL2 OBJECT OBJECT2XML OBJNO OBJNO_REUSE OBJ_ID OBY_GBYPD_SEPARATE OCCURENCES - OCCURRENCES ODD OFF OFFLINE OFFSET OID OIDINDEX OLAP OLD OLD_PUSH_PRED OLS - OLTP OMIT ONE ONLINE ONLY OPAQUE OPAQUE_TRANSFORM OPAQUE_XCANONICAL OPCODE - OPEN OPERATIONS OPERATOR OPTIMAL OPTIMIZE OPTIMIZER_FEATURES_ENABLE OPTIMIZER_GOAL - OPT_ESTIMATE OPT_PARAM ORADEBUG ORA_BRANCH ORA_CHECK_ACL ORA_CHECK_PRIVILEGE - ORA_CHECK_SYS_PRIVILEGE ORA_CLUSTERING ORA_CONCAT_RWKEY ORA_DM_PARTITION_NAME - ORA_DST_AFFECTED ORA_DST_CONVERT ORA_DST_ERROR ORA_GET_ACLIDS ORA_GET_PRIVILEGES - ORA_HASH ORA_INVOKING_USER ORA_INVOKING_USERID ORA_INVOKING_XS_USER ORA_INVOKING_XS_USER_GUID - ORA_NORMALIZE ORA_PARTITION_VALIDATION ORA_RAWCOMPARE ORA_RAWCONCAT ORA_ROWSCN - ORA_ROWSCN_RAW ORA_ROWVERSION ORA_SEARCH_RWKEY ORA_SHARDSPACE_NAME ORA_SHARD_ID - ORA_TABVERSION ORA_WRITE_TIME ORDERED ORDERED_PREDICATES ORDER_KEY_VECTOR_USE - ORDER_SUBQ ORDINALITY ORGANIZATION OR_EXPAND OR_PREDICATES OSON OSON_DIAG - OSON_GET_CONTENT OTHER OTHERS OUTER OUTER_JOIN_TO_ANTI OUTER_JOIN_TO_INNER - OUTLINE OUTLINE_LEAF OUT_OF_LINE OVER OVERFLOW OVERFLOW_NOMOVE OVERLAPS OWN - OWNER OWNERSHIP PACKAGE PACKAGES PARALLEL PARALLEL_INDEX PARAM PARAMETERS - PARENT PARITY PART$NUM$INST PARTIAL PARTIALLY PARTIAL_JOIN PARTIAL_ROLLUP_PUSHDOWN - PARTITION PARTITIONING PARTITIONS PARTITIONSET PARTITION_CONTAINED PARTITION_HASH - PARTITION_LIST PARTITION_RANGE PASSING PASSIVE PASSWORD PASSWORDFILE_METADATA_CACHE - PASSWORD_GRACE_TIME PASSWORD_LIFE_TIME PASSWORD_LOCK_TIME PASSWORD_REUSE_MAX - PASSWORD_REUSE_TIME PASSWORD_ROLLOVER_TIME PASSWORD_VERIFY_FUNCTION PAST PATCH - PATH PATHS PATH_PREFIX PATTERN PBL_HS_BEGIN PBL_HS_END PCTINCREASE PCTTHRESHOLD - PCTUSED PCTVERSION PDB_LOCAL_ONLY PEER PEERS PENDING PER PERCENT PERCENTAGE - PERCENTILE_CONT PERCENTILE_DISC PERCENT_RANK PERCENT_RANKM PERFORMANCE PERIOD - PERMANENT PERMISSION PERMUTE PERSISTABLE PFILE PHV PHYSICAL PIKEY PIVOT PIV_GB - PIV_SSF PLACE_DISTINCT PLACE_GROUP_BY PLAN PLSCOPE_SETTINGS PLSQL_CCFLAGS - PLSQL_CODE_TYPE PLSQL_DEBUG PLSQL_OPTIMIZE_LEVEL PLSQL_WARNINGS PLUGGABLE - PMEM POINT POLICY POOL_16K POOL_2K POOL_32K POOL_4K POOL_8K PORT POSITION - POST_TRANSACTION POWER POWERMULTISET POWERMULTISET_BY_CARDINALITY PQ_CONCURRENT_UNION - PQ_DISTRIBUTE PQ_DISTRIBUTE_WINDOW PQ_EXPAND_TABLE PQ_FILTER PQ_MAP PQ_NOMAP - PQ_NONLEAF_SKEW PQ_REPLICATE PQ_SKEW PREBUILT PRECEDES PRECEDING PRECOMPUTE_SUBQUERY - PREDICATE_REORDERS PREDICTION PREDICTION_BOUNDS PREDICTION_COST PREDICTION_DETAILS - PREDICTION_PROBABILITY PREDICTION_SET PRELOAD PREPARE PRESENT PRESENTNNV PRESENTV - PRESERVE PRESERVE_OID PRETTY PREV PREVIOUS PRIMARY PRINTBLOBTOCLOB PRIORITY - PRIVATE PRIVATE_SGA PRIVILEGE PRIVILEGED PRIVILEGES PROCEDURAL PROCEDURE PROCESS - PROFILE PROGRAM PROJECT PROPAGATE PROPAGATION PROPERTY PROTECTED PROTECTION - PROTOCOL PROXY PRUNING PULL_PRED PURGE PUSH_HAVING_TO_GBY PUSH_PRED PUSH_SUBQ - PX_FAULT_TOLERANCE PX_GRANULE PX_JOIN_FILTER QB_NAME QUALIFY QUARANTINE QUARTERS - QUERY QUERY_BLOCK QUEUE QUEUE_CURR QUEUE_ROWP QUIESCE QUORUM QUOTA QUOTAGROUP - QUOTES RANDOM RANDOM_LOCAL RANGE RANK RANKM RAPIDLY RATIO_TO_REPORT RAWTOHEX - RAWTONHEX RAWTOREF RBA RBO_OUTLINE RDBA READ READS READ_OR_WRITE REALM REBALANCE - REBUILD RECONNECT RECORDS_PER_BLOCK RECOVER RECOVERABLE RECOVERY RECYCLE RECYCLEBIN - REDACTION REDEFINE REDO REDUCED REDUNDANCY REFERENCE REFERENCED REFERENCES - REFERENCING REFRESH REFTOHEX REFTORAW REF_CASCADE_CURSOR REGEXP_COUNT REGEXP_INSTR - REGEXP_LIKE REGEXP_REPLACE REGEXP_SUBSTR REGISTER REGR_AVGX REGR_AVGY REGR_COUNT - REGR_INTERCEPT REGR_R2 REGR_SLOPE REGR_SXX REGR_SXY REGR_SYY REGULAR REJECT - REKEY RELATIONAL RELOCATE RELY REMAINDER REMOTE REMOTE_MAPPED REMOVE REORDER_WIF - REPAIR REPEAT REPLACE REPLICATION REQUIRED RESERVOIR_SAMPLING RESET RESETLOGS - RESIZE RESOLVE RESOLVER RESPECT RESTART RESTORE RESTORE_AS_INTERVALS RESTRICT - RESTRICTED RESTRICT_ALL_REF_CONS RESULT_CACHE RESUMABLE RESUME RETENTION RETRY_ON_ROW_CHANGE - RETURN RETURNING REUSE REVERSE REWRITE REWRITE_OR_ERROR RIGHT RLS_FORCE ROLE - ROLES ROLESET ROLLBACK ROLLING ROLLOVER ROLLUP ROOT ROUND ROUND_TIES_TO_EVEN - ROW ROWDEPENDENCIES ROWIDTOCHAR ROWIDTONCHAR ROWID_MAPPING_TABLE ROWNUM ROWS - ROW_LENGTH ROW_NUMBER RPAD RTRIM RULE RULES RUNNING SALT SAMPLE SAVE SAVEPOINT - SAVE_AS_INTERVALS SB4 SCALAR SCALARS SCALE SCALE_ROWS SCAN SCAN_INSTANCES - SCHEDULER SCHEMA SCHEMACHECK SCN SCN_ASCENDING SCOPE SCRUB SDO_GEOM_KEY SDO_GEOM_MAX_X - SDO_GEOM_MAX_Y SDO_GEOM_MAX_Z SDO_GEOM_MBB SDO_GEOM_MBR SDO_GEOM_MIN_X SDO_GEOM_MIN_Y - SDO_GEOM_MIN_Z SDO_TOLERANCE SD_ALL SD_INHIBIT SD_SHOW SEARCH SECONDS SECRET - SECUREFILE SECUREFILE_DBA SECURITY SEED SEGMENT SEG_BLOCK SEG_FILE SELECTIVITY - SELF SEMIJOIN SEMIJOIN_DRIVER SEMI_TO_INNER SENSITIVE SEQUENCE SEQUENCED SEQUENTIAL - SERIAL SERIALIZABLE SERVERERROR SERVICE SERVICES SERVICE_NAME_CONVERT SESSION - SESSIONS_PER_USER SESSIONTIMEZONE SESSIONTZNAME SESSION_CACHED_CURSORS SETS - SETTINGS SET_GBY_PUSHDOWN SET_TO_JOIN SEVERE SHARD SHARDED SHARDS SHARDSPACE - SHARD_CHUNK_ID SHARED SHARED_POOL SHARE_OF SHARING SHD$COL$MAP SHELFLIFE SHOW - SHRINK SHUTDOWN SIBLING SIBLINGS SID SIGN SIGNAL_COMPONENT SIGNAL_FUNCTION - SIGNATURE SIMPLE SIN SINGLE SINGLETASK SINH SITE SKEWNESS_POP SKEWNESS_SAMP - SKIP SKIP_EXT_OPTIMIZER SKIP_PROXY SKIP_UNQ_UNUSABLE_IDX SKIP_UNUSABLE_INDEXES - SMALLFILE SNAPSHOT SOME SORT SOUNDEX SOURCE SOURCE_FILE_DIRECTORY SOURCE_FILE_NAME_CONVERT - SPACE SPATIAL SPECIFICATION SPFILE SPLIT SPREADSHEET SQL SQLLDR SQL_SCOPE - SQL_TRACE SQL_TRANSLATION_PROFILE SQRT STALE STANDALONE STANDARD_HASH STANDBY - STANDBYS STANDBY_MAX_DATA_DELAY STAR STARTUP STAR_TRANSFORMATION STATE STATEMENT - STATEMENTS STATEMENT_ID STATEMENT_QUEUING STATIC STATISTICS STATS_BINOMIAL_TEST - STATS_CROSSTAB STATS_F_TEST STATS_KS_TEST STATS_MODE STATS_MW_TEST STATS_ONE_WAY_ANOVA - STATS_T_TEST_INDEP STATS_T_TEST_INDEPU STATS_T_TEST_ONE STATS_T_TEST_PAIRED - STATS_WSR_TEST STDDEV STDDEV_POP STDDEV_SAMP STOP STORAGE STORAGE_INDEX STORE - STREAM STREAMS STRICT STRING STRINGS STRIP STRIPE_COLUMNS STRIPE_WIDTH STRUCTURE - SUBMULTISET SUBPARTITION SUBPARTITIONING SUBPARTITIONS SUBPARTITION_REL SUBQUERIES - SUBQUERY_PRUNING SUBSCRIBE SUBSET SUBSTITUTABLE SUBSTR SUBSTR2 SUBSTR4 SUBSTRB - SUBSTRC SUBTYPE SUCCESS SUCCESSFUL SUM SUMMARY SUPPLEMENTAL SUPPRESS_LOAD - SUSPEND SWAP_JOIN_INPUTS SWITCH SWITCHOVER SYNC SYNCHRONOUS SYSASM SYSAUX - SYSBACKUP SYSDATE SYSDBA SYSDG SYSGUID SYSKM SYSOBJ SYSOPER SYSRAC SYSTEM - SYSTEM_DEFINED SYSTEM_STATS SYSTIMESTAMP SYS_AUDIT SYS_CHECKACL SYS_CHECK_PRIVILEGE - SYS_CONNECT_BY_PATH SYS_CONS_ANY_SCALAR SYS_CONTEXT SYS_CTXINFOPK SYS_CTX_CONTAINS2 - SYS_CTX_MKIVIDX SYS_DBURIGEN SYS_DL_CURSOR SYS_DM_RXFORM_CHR SYS_DM_RXFORM_LAB - SYS_DM_RXFORM_NUM SYS_DOM_COMPARE SYS_DST_PRIM2SEC SYS_DST_SEC2PRIM SYS_ET_BFILE_TO_RAW - SYS_ET_BLOB_TO_IMAGE SYS_ET_IMAGE_TO_BLOB SYS_ET_RAW_TO_BFILE SYS_EXTPDTXT - SYS_EXTRACT_UTC SYS_FBT_INSDEL SYS_FILTER_ACLS SYS_FNMATCHES SYS_FNREPLACE - SYS_GETTOKENID SYS_GETXTIVAL SYS_GET_ACLIDS SYS_GET_ANY_SCALAR SYS_GET_COL_ACLIDS - SYS_GET_PRIVILEGES SYS_GUID SYS_MAKEXML SYS_MAKE_XMLNODEID SYS_MKXMLATTR SYS_MKXTI - SYS_OPTLOBPRBSC SYS_OPTXICMP SYS_OPTXQCASTASNQ SYS_OP_ADT2BIN SYS_OP_ADTCONS - SYS_OP_ALSCRVAL SYS_OP_ATG SYS_OP_BIN2ADT SYS_OP_BITVEC SYS_OP_BL2R SYS_OP_BLOOM_FILTER - SYS_OP_BLOOM_FILTER_LIST SYS_OP_C2C SYS_OP_CAST SYS_OP_CEG SYS_OP_CL2C SYS_OP_COMBINED_HASH - SYS_OP_COMP SYS_OP_CONVERT SYS_OP_COUNTCHG SYS_OP_CSCONV SYS_OP_CSCONVTEST - SYS_OP_CSR SYS_OP_CSX_PATCH SYS_OP_CYCLED_SEQ SYS_OP_DECOMP SYS_OP_DESCEND - SYS_OP_DISTINCT SYS_OP_DRA SYS_OP_DSB_DESERIALIZE SYS_OP_DSB_SERIALIZE SYS_OP_DUMP - SYS_OP_DV_CHECK SYS_OP_ENFORCE_NOT_NULL$ SYS_OP_EXTRACT SYS_OP_GROUPING SYS_OP_GUID - SYS_OP_HASH SYS_OP_HCS_TABLE SYS_OP_IIX SYS_OP_INTERVAL_HIGH_BOUND SYS_OP_ITR - SYS_OP_KEY_VECTOR_CREATE SYS_OP_KEY_VECTOR_FILTER SYS_OP_KEY_VECTOR_FILTER_LIST - SYS_OP_KEY_VECTOR_PAYLOAD SYS_OP_KEY_VECTOR_SUCCEEDED SYS_OP_KEY_VECTOR_USE - SYS_OP_LBID SYS_OP_LOBLOC2BLOB SYS_OP_LOBLOC2CLOB SYS_OP_LOBLOC2ID SYS_OP_LOBLOC2NCLOB - SYS_OP_LOBLOC2TYP SYS_OP_LSVI SYS_OP_LVL SYS_OP_MAKEOID SYS_OP_MAP_NONNULL - SYS_OP_MSR SYS_OP_NICOMBINE SYS_OP_NIEXTRACT SYS_OP_NII SYS_OP_NIX SYS_OP_NOEXPAND - SYS_OP_NTCIMG$ SYS_OP_NUMTORAW SYS_OP_OBJ_UPD_IN_TXN SYS_OP_OIDVALUE SYS_OP_OPNSIZE - SYS_OP_PAR SYS_OP_PARGID SYS_OP_PARGID_1 SYS_OP_PART_ID SYS_OP_PAR_1 SYS_OP_PIVOT - SYS_OP_R2O SYS_OP_RAWTONUM SYS_OP_RDTM SYS_OP_REF SYS_OP_RMTD SYS_OP_ROWIDTOOBJ - SYS_OP_RPB SYS_OP_TOSETID SYS_OP_TPR SYS_OP_TRTB SYS_OP_UNDESCEND SYS_OP_VECAND - SYS_OP_VECBIT SYS_OP_VECOR SYS_OP_VECTOR_GROUP_BY SYS_OP_VECXOR SYS_OP_VERSION - SYS_OP_VREF SYS_OP_VVD SYS_OP_XMLCONS_FOR_CSX SYS_OP_XPTHATG SYS_OP_XPTHIDX - SYS_OP_XPTHOP SYS_OP_XTNN SYS_OP_XTXT2SQLT SYS_OP_ZONE_ID SYS_ORDERKEY_DEPTH - SYS_ORDERKEY_MAXCHILD SYS_ORDERKEY_PARENT SYS_PARALLEL_TXN SYS_PATHID_IS_ATTR - SYS_PATHID_IS_NMSPC SYS_PATHID_LASTNAME SYS_PATHID_LASTNMSPC SYS_PATH_REVERSE - SYS_PLSQL_COUNT SYS_PLSQL_CPU SYS_PLSQL_IO SYS_PXQEXTRACT SYS_RAW_TO_XSID - SYS_REMAP_XMLTYPE SYS_RID_ORDER SYS_ROW_DELTA SYS_SC_2_XMLT SYS_SYNRCIREDO - SYS_TYPEID SYS_UMAKEXML SYS_XMLANALYZE SYS_XMLCONTAINS SYS_XMLCONV SYS_XMLEXNSURI - SYS_XMLGEN SYS_XMLINSTR SYS_XMLI_LOC_ISNODE SYS_XMLI_LOC_ISTEXT SYS_XMLLOCATOR_GETSVAL - SYS_XMLNODEID SYS_XMLNODEID_GETCID SYS_XMLNODEID_GETLOCATOR SYS_XMLNODEID_GETOKEY - SYS_XMLNODEID_GETPATHID SYS_XMLNODEID_GETPTRID SYS_XMLNODEID_GETRID SYS_XMLNODEID_GETSVAL - SYS_XMLNODEID_GETTID SYS_XMLTRANSLATE SYS_XMLTYPE2SQL SYS_XMLT_2_SC SYS_XQBASEURI - SYS_XQCASTABLEERRH SYS_XQCODEP2STR SYS_XQCODEPEQ SYS_XQCON2SEQ SYS_XQCONCAT - SYS_XQDELETE SYS_XQDFLTCOLATION SYS_XQDOC SYS_XQDOCURI SYS_XQDURDIV SYS_XQED4URI - SYS_XQENDSWITH SYS_XQERR SYS_XQERRH SYS_XQESHTMLURI SYS_XQEXLOBVAL SYS_XQEXSTWRP - SYS_XQEXTRACT SYS_XQEXTRREF SYS_XQEXVAL SYS_XQFB2STR SYS_XQFNBOOL SYS_XQFNCMP - SYS_XQFNDATIM SYS_XQFNLNAME SYS_XQFNNM SYS_XQFNNSURI SYS_XQFNPREDTRUTH SYS_XQFNQNM - SYS_XQFNROOT SYS_XQFORMATNUM SYS_XQFTCONTAIN SYS_XQFUNCR SYS_XQGETCONTENT - SYS_XQINDXOF SYS_XQINSERT SYS_XQINSPFX SYS_XQIRI2URI SYS_XQLANG SYS_XQLLNMFRMQNM - SYS_XQMKNODEREF SYS_XQNILLED SYS_XQNODENAME SYS_XQNORMSPACE SYS_XQNORMUCODE - SYS_XQNSP4PFX SYS_XQNSPFRMQNM SYS_XQPFXFRMQNM SYS_XQPOLYABS SYS_XQPOLYADD - SYS_XQPOLYCEL SYS_XQPOLYCST SYS_XQPOLYCSTBL SYS_XQPOLYDIV SYS_XQPOLYFLR SYS_XQPOLYMOD - SYS_XQPOLYMUL SYS_XQPOLYRND SYS_XQPOLYSQRT SYS_XQPOLYSUB SYS_XQPOLYUMUS SYS_XQPOLYUPLS - SYS_XQPOLYVEQ SYS_XQPOLYVGE SYS_XQPOLYVGT SYS_XQPOLYVLE SYS_XQPOLYVLT SYS_XQPOLYVNE - SYS_XQREF2VAL SYS_XQRENAME SYS_XQREPLACE SYS_XQRESVURI SYS_XQRNDHALF2EVN SYS_XQRSLVQNM - SYS_XQRYENVPGET SYS_XQRYVARGET SYS_XQRYWRP SYS_XQSEQ2CON SYS_XQSEQ2CON4XC - SYS_XQSEQDEEPEQ SYS_XQSEQINSB SYS_XQSEQRM SYS_XQSEQRVS SYS_XQSEQSUB SYS_XQSEQTYPMATCH - SYS_XQSTARTSWITH SYS_XQSTATBURI SYS_XQSTR2CODEP SYS_XQSTRJOIN SYS_XQSUBSTRAFT - SYS_XQSUBSTRBEF SYS_XQTOKENIZE SYS_XQTREATAS SYS_XQXFORM SYS_XQ_ASQLCNV SYS_XQ_ATOMCNVCHK - SYS_XQ_NRNG SYS_XQ_PKSQL2XML SYS_XQ_UPKXML2SQL SYS_XSID_TO_RAW SYS_ZMAP_FILTER - SYS_ZMAP_REFRESH TABAUTH TABLES TABLESPACE TABLESPACE_NO TABLE_LOOKUP_BY_NL - TABLE_STATS TABNO TAG TAN TANH TARGET TBL$OR$IDX$PART$NUM TEMP TEMPFILE TEMPLATE - TEMPORARY TEMP_TABLE TENANT_ID TEST TEXT THAN THE THREAD THROUGH TIER TIES - TIMEOUT TIMES TIMESTAMP_TO_NUMBER TIMEZONE_ABBR TIMEZONE_HOUR TIMEZONE_MINUTE - TIMEZONE_OFFSET TIMEZONE_REGION TIME_ZONE TIV_GB TIV_SSF TOKEN TOPLEVEL TO_ACLID - TO_APPROX_COUNT_DISTINCT TO_APPROX_PERCENTILE TO_BINARY_DOUBLE TO_BINARY_FLOAT - TO_BLOB TO_CHAR TO_CLOB TO_DATE TO_DSINTERVAL TO_ISO_STRING TO_LOB TO_MULTI_BYTE - TO_NCHAR TO_NCLOB TO_NUMBER TO_SINGLE_BYTE TO_TIME TO_TIMESTAMP TO_TIMESTAMP_TZ - TO_TIME_TZ TO_UTC_TIMESTAMP_TZ TO_YMINTERVAL TRACE TRACING TRACKING TRAILING - TRANSACTION TRANSFORM TRANSFORM_DISTINCT_AGG TRANSITION TRANSITIONAL TRANSLATE - TRANSLATION TRANSPORTABLE TREAT TRIGGERS TRIM TRUE TRUNC TRUNCATE TRUST TRUSTED - TUNING TX TYPE TYPENAME TYPES TZ_OFFSET UB2 UBA UCS2 UID UNARCHIVED UNBOUND - UNBOUNDED UNCONDITIONAL UNDER UNDO UNDROP UNIFORM UNINSTALL UNION_ALL UNISTR - UNITE UNIXTIME UNLIMITED UNLOAD UNLOCK UNMATCHED UNNEST UNNEST_INNERJ_DISTINCT_VIEW - UNNEST_NOSEMIJ_NODISTINCTVIEW UNNEST_SEMIJ_VIEW UNPACKED UNPIVOT UNPLUG UNPROTECTED - UNQUIESCE UNRECOVERABLE UNRESTRICTED UNSUBSCRIBE UNTIL UNUSABLE UNUSED UPDATABLE - UPDATED UPDATEXML UPD_INDEXES UPD_JOININDEX UPGRADE UPPER UPSERT USABLE USAGE - USE USER USERENV USERGROUP USERS USER_DATA USER_DEFINED USER_RECYCLEBIN USER_TABLESPACES - USE_ANTI USE_CONCAT USE_CUBE USE_DAGG_UNION_ALL_GSETS USE_HASH USE_HASH_AGGREGATION - USE_HASH_GBY_FOR_DAGGPSHD USE_HASH_GBY_FOR_PUSHDOWN USE_HIDDEN_PARTITIONS - USE_INVISIBLE_INDEXES USE_MERGE USE_MERGE_CARTESIAN USE_NL USE_NL_WITH_INDEX - USE_PARTITION_WISE_DISTINCT USE_PARTITION_WISE_GBY USE_PARTITION_WISE_WIF - USE_PRIVATE_OUTLINES USE_SCALABLE_GBY_INVDIST USE_SEMI USE_STORED_OUTLINES - USE_TTT_FOR_GSETS USE_VECTOR_AGGREGATION USE_WEAK_NAME_RESL USING USING_NO_EXPAND - UTF16BE UTF16LE UTF32 UTF8 V1 V2 VALIDATE VALIDATE_CONVERSION VALIDATION VALID_TIME_END - VALUE VARIANCE VARRAY VARRAYS VAR_POP VAR_SAMP VECTOR VECTOR_ENCODE VECTOR_READ - VECTOR_READ_TRACE VECTOR_TRANSFORM VECTOR_TRANSFORM_DIMS VECTOR_TRANSFORM_FACT - VERIFIER VERIFY VERSION VERSIONING VERSIONS VERSIONS_ENDSCN VERSIONS_ENDTIME - VERSIONS_OPERATION VERSIONS_STARTSCN VERSIONS_STARTTIME VERSIONS_XID VIEWS - VIOLATION VIRTUAL VISIBILITY VISIBLE VOLUME VSIZE WAIT WALLET WEEK WEEKS WELLFORMED - WHEN WHENEVER WHITESPACE WIDTH_BUCKET WINDOW WITHIN WITHOUT WITH_EXPRESSION - WITH_PLSQL WORK WRAPPED WRAPPER WRITE XDB_FASTPATH_INSERT XID XML XML2OBJECT - XMLATTRIBUTES XMLCAST XMLCDATA XMLCOLATTVAL XMLCOMMENT XMLCONCAT XMLDIFF XMLELEMENT - XMLEXISTS XMLEXISTS2 XMLFOREST XMLINDEX_REWRITE XMLINDEX_REWRITE_IN_SELECT - XMLINDEX_SEL_IDX_TBL XMLISNODE XMLISVALID XMLNAMESPACES XMLPARSE XMLPATCH - XMLPI XMLQUERY XMLQUERYVAL XMLROOT XMLSCHEMA XMLSERIALIZE XMLTABLE XMLTOJSON - XMLTOKENSET XMLTRANSFORM XMLTRANSFORMBLOB XMLTSET_DML_ENABLE XML_DIAG XML_DML_RWT_STMT - XPATHTABLE XS XS_SYS_CONTEXT X_DYN_PRUNE YEARS YES ZONEMAP - )) - end - - def self.keywords_func - @keywords_func ||= Set.new(%w( - ABS ACOS ADD_MONTHS APPROX_COUNT APPROX_COUNT_DISTINCT APPROX_COUNT_DISTINCT_AGG - APPROX_COUNT_DISTINCT_DETAIL APPROX_MEDIAN APPROX_PERCENTILE APPROX_PERCENTILE_AGG - APPROX_PERCENTILE_DETAIL APPROX_RANK APPROX_SUM ASCII ASCIISTR ASIN ATAN ATAN2 - AVG BFILENAME BIN_TO_NUM BITAND CARDINALITY CAST CEIL CHARTOROWID CHR CLUSTER_DETAILS - CLUSTER_DISTANCE CLUSTER_ID CLUSTER_PROBABILITY CLUSTER_SET COALESCE COLLATION - COLLECT COMPOSE CONCAT CONVERT CON_DBID_TO_ID CON_GUID_TO_ID CON_NAME_TO_ID - CON_UID_TO_ID CORR COS COSH COUNT COVAR_POP COVAR_SAMP CUME_DIST CURRENT_DATE - CURRENT_TIMESTAMP CV DATAOBJ_TO_MAT_PARTITION DATAOBJ_TO_PARTITION DBTIMEZONE - DECODE DECOMPOSE DENSE_RANK DEPTH DEREF DUMP EMPTY_BLOB EMPTY_CLOB EXISTSNODE - EXP EXTRACT EXTRACTVALUE FEATURE_COMPARE FEATURE_DETAILS FEATURE_ID FEATURE_SET - FEATURE_VALUE FIRST FIRST_VALUE FLOOR FROM_TZ GREATEST GROUPING GROUPING_ID - GROUP_ID HEXTORAW INITCAP INSTR ITERATION_NUMBER JSON_ARRAY JSON_ARRAYAGG - JSON_OBJECT JSON_OBJECTAGG JSON_QUERY JSON_TABLE JSON_VALUE LAG LAST LAST_DAY - LAST_VALUE LEAD LEAST LENGTH LISTAGG LN LNNVL LOCALTIMESTAMP LOG LOWER LPAD - LTRIM MAKE_REF MAX MEDIAN MIN MOD MONTHS_BETWEEN NANVL NCHR NEW_TIME NEXT_DAY - NLSSORT NLS_CHARSET_DECL_LEN NLS_CHARSET_ID NLS_CHARSET_NAME NLS_COLLATION_ID - NLS_COLLATION_NAME NLS_INITCAP NLS_LOWER NLS_UPPER NTH_VALUE NTILE NULLIF - NUMTODSINTERVAL NUMTOYMINTERVAL NVL NVL2 ORA_DM_PARTITION_NAME ORA_DST_AFFECTED - ORA_DST_CONVERT ORA_DST_ERROR ORA_HASH ORA_INVOKING_USER ORA_INVOKING_USERID - PATH PERCENTILE_CONT PERCENTILE_DISC PERCENT_RANK POWER POWERMULTISET POWERMULTISET_BY_CARDINALITY - PREDICTION PREDICTION_BOUNDS PREDICTION_COST PREDICTION_DETAILS PREDICTION_PROBABILITY - PREDICTION_SET PRESENTNNV PRESENTV PREVIOUS RANK RATIO_TO_REPORT RAWTOHEX - RAWTONHEX REFTOHEX REGEXP_COUNT REGEXP_INSTR REGEXP_REPLACE REGEXP_SUBSTR - REMAINDER REPLACE ROUND ROWIDTOCHAR ROWIDTONCHAR ROW_NUMBER RPAD RTRIM - SCN_TO_TIMESTAMP SESSIONTIMEZONE SET SIGN SIN SINH SOUNDEX SQRT STANDARD_HASH - STATS_BINOMIAL_TEST STATS_CROSSTAB STATS_F_TEST STATS_KS_TEST STATS_MODE STATS_MW_TEST - STATS_ONE_WAY_ANOVA STATS_WSR_TEST STDDEV STDDEV_POP STDDEV_SAMP SUBSTR SUM - SYSDATE SYSTIMESTAMP SYS_CONNECT_BY_PATH SYS_CONTEXT SYS_DBURIGEN SYS_EXTRACT_UTC - SYS_GUID SYS_OP_ZONE_ID SYS_TYPEID SYS_XMLAGG SYS_XMLGEN TAN TANH TIMESTAMP_TO_SCN - TO_APPROX_COUNT_DISTINCT TO_APPROX_PERCENTILE TO_BINARY_DOUBLE TO_BINARY_FLOAT - TO_BLOB TO_CHAR TO_CLOB TO_DATE TO_DSINTERVAL TO_LOB TO_MULTI_BYTE TO_NCHAR - TO_NCLOB TO_NUMBER TO_SINGLE_BYTE TO_TIMESTAMP TO_TIMESTAMP_TZ TO_YMINTERVAL - TRANSLATE TREAT TRIM TRUNC TZ_OFFSET UID UNISTR UPPER USER USERENV VALIDATE_CONVERSION - VALUE VARIANCE VAR_POP VAR_SAMP VSIZE WIDTH_BUCKET XMLAGG XMLCAST XMLCDATA - XMLCOLATTVAL XMLCOMMENT XMLCONCAT XMLDIFF XMLELEMENT XMLEXISTS XMLFOREST XMLISVALID - XMLPARSE XMLPATCH XMLPI XMLQUERY XMLROOT XMLSEQUENCE XMLSERIALIZE XMLTABLE - XMLTRANSFORM - )) - end - - def self.keywords_type - @keywords_type ||= Set.new(%w( - CHAR BYTE VARCHAR2 NCHAR NVARCHAR2 - NUMBER FLOAT BINARY_FLOAT BINARY_DOUBLE - LONG RAW - DATE TIMESTAMP INTERVAL LOCAL TIME ZONE TO MONTH SECOND YEAR DAY - BLOB CLOB NCLOB BFILE - UROWID - CHARACTER VARYING VARCHAR NATIONAL - NUMERIC DECIMAL DEC INTEGER INT SMALLINT DOUBLE PRECISION REAL - SDO_GEOMETRY SDO_TOPO_GEOMETRY SDO_GEORASTER - REF ANYTYPE ANYDATA ANYDATASET XMLTYPE HTTPURITYPE XDBURITYPE DUBRITYPE - BOOLEAN PLS_INTEGER BINARY_INTEGER SIMPLE_FLOAT SIMPLE_INTEGER SIMPLE_DOUBLE SYS_REFCURSOR - )) - end + lazy { require_relative 'plsql/keywords' } state :root do delimiter_map = { '{' => '}', '[' => ']', '(' => ')', '<' => '>' } @@ -514,26 +79,21 @@ def self.keywords_type # General keyword classification with sepcial attention to names # in a chained "dot" notation. # - rule %r/([a-zA-Z][\w$]*)(\.(?=\w))?/ do |m| - if self.class.keywords_type.include? m[1].upcase - tok = Keyword::Type - elsif self.class.keywords_func.include? m[1].upcase - tok = Name::Function - elsif self.class.keywords_reserved.include? m[1].upcase - tok = Keyword::Reserved - elsif self.class.keywords.include? m[1].upcase - tok = Keyword - else - tok = Name - end - groups tok, Punctuation - - if m[2] == "." - push :dotnames - end + keywords %r/[a-zA-Z][\w$]*/ do + transform(&:upcase) + rule KEYWORDS_TYPE, Keyword, :post_name + rule KEYWORDS_FUNC, Name::Function, :post_name + rule KEYWORDS_RESERVED, Keyword::Reserved, :post_name + rule KEYWORDS, Keyword, :post_name + default Name, :post_name end end + state :post_name do + rule %r/[.]/, Punctuation, :dotnames + rule(//) { pop! } + end + state :multiline_comments do rule %r/([*][^\/]|[^*])+/m, Comment::Multiline rule %r([*]\/), Comment::Multiline, :pop! @@ -554,24 +114,24 @@ def self.keywords_type end state :dotnames do + rule %r/".*?"/, Str::Symbol + rule %r/[.]/, Punctuation + # if we are followed by a dot and another name, we are an ordinary name rule %r/([a-zA-Z][\w\$]*)(\.(?=\w))/ do groups Name, Punctuation end # this rule WILL be true if something pushed into our state. That is our state contract - rule %r/[a-zA-Z][\w\$]*/ do |m| - if self.class.keywords_func.include? m[0].upcase - # The Function lookup allows collection methods like COUNT, FIRST, LAST, etc.. to be - # classified correctly. Occasionally misidentifies ordinary names as builtin functions, - # but seems to be as correct as we can get without becoming a full blown parser - token Name::Function - else - token Name - end - pop! + keywords %r/[a-zA-Z][\w\$]*/ do + transform(&:upcase) + + # The Function lookup allows collection methods like COUNT, FIRST, LAST, etc.. to be + # classified correctly. Occasionally misidentifies ordinary names as builtin functions, + # but seems to be as correct as we can get without becoming a full blown parser + rule KEYWORDS_FUNC, Name::Function, :pop! + default Name, :pop! end end - end end end diff --git a/lib/rouge/lexers/plsql/keywords.rb b/lib/rouge/lexers/plsql/keywords.rb new file mode 100644 index 0000000000..3a0c7171d8 --- /dev/null +++ b/lib/rouge/lexers/plsql/keywords.rb @@ -0,0 +1,434 @@ +module Rouge + module Lexers + class PLSQL + KEYWORDS_RESERVED = Set.new %w( + ACCESSIBLE AGENT ALL ALTER AND ANY AS ASC BETWEEN BFILE_BASE BLOB_BASE BY + C CALLING CHARSET CHARSETFORM CHARSETID CHAR_BASE CHECK CLOB_BASE CLUSTER + COLLATE COMPILED COMPRESS CONNECT CONNECT_BY_ROOT CONSTRUCTOR CREATE CUSTOMDATUM + DATE_BASE DEFAULT DELETE DESC DISTINCT DROP DURATION ELSE ELSIF EXCEPT EXCLUSIVE + EXISTS EXIT FIXED FOR FORALL FROM GENERAL GRANT GROUP HAVING IDENTIFIED IN INDEX + INDICES INSERT INTERFACE INTERSECT INTO IS LARGE LIKE LIMITED LOCK LOOP MAXLEN + MINUS MODE NOCOMPRESS NOT NOWAIT NULL NUMBER_BASE OCICOLL OCIDATE OCIDATETIME + OCIDURATION OCIINTERVAL OCILOBLOCATOR OCINUMBER OCIRAW OCIREF OCIREFCURSOR + OCIROWID OCISTRING OCITYPE OF ON OPTION OR ORACLE ORADATA ORDER ORLANY ORLVARY + OUT OVERRIDING PARALLEL_ENABLE PARAMETER PASCAL PCTFREE PIPE PIPELINED POLYMORPHIC + PRAGMA PRIOR PUBLIC RAISE RECORD RELIES_ON REM RENAME RESOURCE RESULT REVOKE ROWID + SB1 SB2 SELECT SEPARATE SET SHARE SHORT SIZE SIZE_T SPARSE SQLCODE SQLDATA + SQLNAME SQLSTATE STANDARD START STORED STRUCT STYLE SYNONYM TABLE TDO THEN + TRANSACTIONAL TRIGGER UB1 UB4 UNION UNIQUE UNSIGNED UNTRUSTED UPDATE VALIST + VALUES VARIABLE VIEW VOID WHERE WHILE WITH + ) + + KEYWORDS = Set.new %w( + ABORT ABS ABSENT ACCESS ACCESSED ACCOUNT ACL ACOS ACROSS ACTION ACTIONS + ACTIVATE ACTIVE ACTIVE_COMPONENT ACTIVE_DATA ACTIVE_FUNCTION ACTIVE_TAG ACTIVITY + ADAPTIVE_PLAN ADD ADD_COLUMN ADD_GROUP ADD_MONTHS ADG_REDIRECT_DML ADG_REDIRECT_PLSQL + ADJ_DATE ADMIN ADMINISTER ADMINISTRATOR ADVANCED ADVISE ADVISOR AFD_DISKSTRING + AFFINITY AFTER AGGREGATE AGGREGATES ALGORITHM ALIAS ALLOCATE ALLOW ALL_ROWS + ALTERNATE ALWAYS ANALYTIC ANALYTIC_VIEW_SQL ANALYZE ANCESTOR ANCILLARY AND_EQUAL + ANOMALY ANSI_REARCH ANSWER_QUERY_USING_STATS ANTIJOIN ANYSCHEMA ANY_VALUE + APPEND APPENDCHILDXML APPEND_VALUES APPLICATION APPLY APPROX_COUNT APPROX_COUNT_DISTINCT + APPROX_COUNT_DISTINCT_AGG APPROX_COUNT_DISTINCT_DETAIL APPROX_MEDIAN APPROX_PERCENTILE + APPROX_PERCENTILE_AGG APPROX_PERCENTILE_DETAIL APPROX_RANK APPROX_SUM ARCHIVAL + ARCHIVE ARCHIVED ARCHIVELOG ARRAY ARRAYS ASCII ASCIISTR ASIN ASIS ASSEMBLY + ASSIGN ASSOCIATE ASYNC ASYNCHRONOUS AS_JSON AT ATAN ATAN2 ATTRIBUTE ATTRIBUTES + AUDIT AUTHENTICATED AUTHENTICATION AUTHID AUTHORIZATION AUTO AUTOALLOCATE + AUTOEXTEND AUTOMATIC AUTO_LOGIN AUTO_REOPTIMIZE AVAILABILITY AVCACHE_OP AVERAGE_RANK + AVG AVMDX_OP AVRO AV_AGGREGATE AV_CACHE AW BACKGROUND BACKINGFILE BACKUP BAND_JOIN + BASIC BASICFILE BATCH BATCHSIZE BATCH_TABLE_ACCESS_BY_ROWID BECOME BEFORE + BEGIN BEGINNING BEGIN_OUTLINE_DATA BEHALF BEQUEATH BFILENAME BIGFILE BINARY + BINARY_DOUBLE_INFINITY BINARY_DOUBLE_NAN BINARY_FLOAT_INFINITY BINARY_FLOAT_NAN + BINDING BIND_AWARE BIN_TO_NUM BITAND BITMAP BITMAPS BITMAP_AND BITMAP_BIT_POSITION + BITMAP_BUCKET_NUMBER BITMAP_CONSTRUCT_AGG BITMAP_COUNT BITMAP_OR_AGG BITMAP_TREE + BITOR BITS BITXOR BIT_AND_AGG BIT_OR_AGG BIT_XOR_AGG BLOCK BLOCKCHAIN BLOCKING + BLOCKS BLOCKSIZE BLOCK_RANGE BODY BOOL BOOTSTRAP BOTH BOUND BRANCH BREADTH + BROADCAST BSON BUFFER BUFFER_CACHE BUFFER_POOL BUILD BULK BUSHY_JOIN BYPASS_RECURSIVE_CHECK + BYPASS_UJVC CACHE CACHE_CB CACHE_INSTANCES CACHE_TEMP_TABLE CACHING CALCULATED + CALL CALLBACK CANCEL CAPACITY CAPTION CAPTURE CARDINALITY CASCADE CASE CAST + CATALOG_DBLINK CATEGORY CDB$DEFAULT CDB_HTTP_HANDLER CEIL CELLMEMORY CELL_FLASH_CACHE + CERTIFICATE CFILE CHAINED CHANGE CHANGE_DUPKEY_ERROR_INDEX CHARTOROWID CHAR_CS + CHECKPOINT CHECKSUM CHECK_ACL_REWRITE CHILD CHOOSE CHR CHUNK CLASS CLASSIFICATION + CLASSIFIER CLAUSE CLEAN CLEANUP CLEAR CLIENT CLONE CLOSE CLOSEST CLOSE_CACHED_OPEN_CURSORS + CLOUD_IDENTITY CLUSTERING CLUSTERING_FACTOR CLUSTERS CLUSTER_BY_ROWID CLUSTER_DETAILS + CLUSTER_DISTANCE CLUSTER_ID CLUSTER_PROBABILITY CLUSTER_SET COALESCE COALESCE_SQ + COARSE COLAUTH COLD COLLATE COLLATION COLLECT COLUMN COLUMNAR COLUMNS COLUMN_AUTHORIZATION_INDICATOR + COLUMN_AUTH_INDICATOR COLUMN_STATS COLUMN_VALUE COMMENT COMMIT COMMITTED COMMON + COMMON_DATA_MAP COMPACT COMPATIBILITY COMPILE COMPLETE COMPLIANCE COMPONENT + COMPONENTS COMPOSE COMPOSITE COMPOSITE_LIMIT COMPOUND COMPUTATION COMPUTE + CONCAT CONDITION CONDITIONAL CONFIRM CONFORMING CONNECT_BY_CB_WHR_ONLY CONNECT_BY_COMBINE_SW + CONNECT_BY_COST_BASED CONNECT_BY_ELIM_DUPS CONNECT_BY_FILTERING CONNECT_BY_ISCYCLE + CONNECT_BY_ISLEAF CONNECT_BY_ROOT CONNECT_TIME CONSENSUS CONSIDER CONSISTENT + CONST CONSTANT CONSTRAINT CONSTRAINTS CONTAINER CONTAINERS CONTAINERS_DEFAULT + CONTAINER_DATA CONTAINER_DATA_ADMIT_NULL CONTAINER_MAP CONTAINER_MAP_OBJECT + CONTENT CONTENTS CONTEXT CONTINUE CONTROLFILE CONVERSION CONVERT CON_DBID_TO_ID + CON_GUID_TO_ID CON_ID CON_ID_FILTER CON_ID_TO_CON_NAME CON_ID_TO_DBID CON_ID_TO_GUID + CON_ID_TO_UID CON_NAME_TO_ID CON_UID_TO_ID COOKIE COPY CORR CORRUPTION CORRUPT_XID + CORRUPT_XID_ALL CORR_K CORR_S COS COSH COST COST_XML_QUERY_REWRITE COUNT COVAR_POP + COVAR_SAMP CO_AUTH_IND CPU_COSTING CPU_COUNT CPU_PER_CALL CPU_PER_SESSION + CPU_TIME CRASH CREATE_FILE_DEST CREATE_STORED_OUTLINES CREATION CREDENTIAL + CREDENTIALS CRITICAL CROSS CROSSEDITION CSCONVERT CUBE CUBE_AJ CUBE_GB CUBE_SJ + CUME_DIST CUME_DISTM CURRENT CURRENTV CURRENT_DATE CURRENT_INSTANCE CURRENT_PARTSET_KEY + CURRENT_SCHEMA CURRENT_SHARD_KEY CURRENT_TIME CURRENT_TIMESTAMP CURRENT_USER + CURSOR CURSOR_SHARING_EXACT CURSOR_SPECIFIC_SEGMENT CV CYCLE DAGG_OPTIM_GSETS + DANGLING DATA DATABASE DATABASES DATAFILE DATAFILES DATAMOVEMENT DATAOBJNO + DATAOBJ_TO_MAT_PARTITION DATAOBJ_TO_PARTITION DATAPUMP DATASTORE DATA_LINK_DML + DATA_SECURITY_REWRITE_LIMIT DATA_VALIDATE DATE_MODE DAYS DBA DBA_RECYCLEBIN + DBMS_STATS DBSTR2UTF8 DBTIMEZONE DB_ROLE_CHANGE DB_UNIQUE_NAME DB_VERSION + DDL DEALLOCATE DEBUG DEBUGGER DECLARE DECODE DECOMPOSE DECOMPRESS DECORRELATE + DECR DECREMENT DECRYPT DEDUPLICATE DEFAULTS DEFAULT_COLLATION DEFAULT_PDB_HINT + DEFERRABLE DEFERRED DEFINE DEFINED DEFINER DEFINITION DEGREE DELAY DELEGATE + DELETEXML DELETE_ALL DEMAND DENORM_AV DENSE_RANK DENSE_RANKM DEPENDENT DEPTH + DEQUEUE DEREF DEREF_NO_REWRITE DESCENDANT DESCRIPTION DESTROY DETACHED DETERMINED + DETERMINES DETERMINISTIC DG_GATHER_STATS DIAGNOSTICS DICTIONARY DIGEST DIMENSION + DIMENSIONS DIRECT DIRECTORY DIRECT_LOAD DIRECT_PATH DISABLE DISABLE_ALL DISABLE_CONTAINERS_DEFAULT + DISABLE_CONTAINER_MAP DISABLE_PARALLEL_DML DISABLE_PRESET DISABLE_RPKE DISALLOW + DISASSOCIATE DISCARD DISCONNECT DISK DISKGROUP DISKS DISMOUNT DISTINGUISHED + DISTRIBUTE DISTRIBUTED DIST_AGG_PROLLUP_PUSHDOWN DML DML_UPDATE DOCFIDELITY + DOCUMENT DOMAIN_INDEX_FILTER DOMAIN_INDEX_NO_SORT DOMAIN_INDEX_SORT DOWNGRADE + DRAIN_TIMEOUT DRIVING_SITE DROP_COLUMN DROP_GROUP DST_UPGRADE_INSERT_CONV + DUMP DUPLICATE DUPLICATED DV DYNAMIC DYNAMIC_SAMPLING DYNAMIC_SAMPLING_EST_CDN + EACH EDITION EDITIONABLE EDITIONING EDITIONS ELAPSED_TIME ELEMENT ELIMINATE_JOIN + ELIMINATE_OBY ELIMINATE_OUTER_JOIN ELIMINATE_SQ ELIM_GROUPBY EM EMPTY EMPTY_BLOB + EMPTY_CLOB ENABLE ENABLE_ALL ENABLE_PARALLEL_DML ENABLE_PRESET ENCODE ENCODING + ENCRYPT ENCRYPTION END END_OUTLINE_DATA ENFORCE ENFORCED ENQUEUE ENTERPRISE + ENTITYESCAPING ENTRY EQUIPART EQUIVALENT ERROR ERRORS ERROR_ARGUMENT ERROR_ON_OVERLAP_TIME + ESCAPE ESTIMATE EVAL EVALNAME EVALUATE EVALUATION EVEN EVENTS EVERY EXCEPTION + EXCEPTIONS EXCHANGE EXCLUDE EXCLUDING EXECUTE EXEMPT EXISTING EXISTSNODE EXP + EXPAND EXPAND_GSET_TO_UNION EXPAND_TABLE EXPIRE EXPLAIN EXPLOSION EXPORT EXPRESS + EXPR_CORR_CHECK EXTEND EXTENDED EXTENDS EXTENT EXTENTS EXTERNAL EXTERNALLY + EXTRA EXTRACT EXTRACTCLOBXML EXTRACTVALUE FACILITY FACT FACTOR FACTORIZE_JOIN + FAILED FAILED_LOGIN_ATTEMPTS FAILGROUP FAILOVER FAILURE FALSE FAMILY FAR FAST + FBTSCAN FEATURE FEATURE_COMPARE FEATURE_DETAILS FEATURE_ID FEATURE_SET FEATURE_VALUE + FEDERATION FETCH FILE FILEGROUP FILESTORE FILESYSTEM_LIKE_LOGGING FILE_NAME_CONVERT + FILTER FINAL FINE FINISH FIRST FIRSTM FIRST_ROWS FIRST_VALUE FIXED_VIEW_DATA + FLAGGER FLASHBACK FLASH_CACHE FLEX FLOB FLOOR FLUSH FOLDER FOLLOWING FOLLOWS + FORCE FORCE_JSON_TABLE_TRANSFORM FORCE_SPATIAL FORCE_XML_QUERY_REWRITE FOREIGN + FOREVER FORMAT FORWARD FRAGMENT_NUMBER FREE FREELIST FREELISTS FREEPOOLS FRESH + FRESH_MV FROM_TZ FTP FULL FULL_OUTER_JOIN_TO_OUTER FUNCTION FUNCTIONS GATHER_OPTIMIZER_STATISTICS + GATHER_PLAN_STATISTICS GBY_CONC_ROLLUP GBY_PUSHDOWN GENERATED GET GLOBAL GLOBALLY + GLOBAL_NAME GLOBAL_TOPIC_ENABLED GOLDENGATE GOTO GRANTED GRANULAR GREATEST + GROUPING GROUPING_ID GROUPS GROUP_BY GROUP_ID GUARANTEE GUARANTEED GUARD H + HALF_YEARS HASH HASHING HASHKEYS HASHSET_BUILD HASH_AJ HASH_SJ HEADER HEAP + HELP HEXTORAW HEXTOREF HIDDEN HIDE HIERARCHICAL HIERARCHIES HIERARCHY HIER_ANCESTOR + HIER_CAPTION HIER_CHILDREN HIER_CHILD_COUNT HIER_COLUMN HIER_CONDITION HIER_DEPTH + HIER_DESCRIPTION HIER_HAS_CHILDREN HIER_LAG HIER_LEAD HIER_LEVEL HIER_MEMBER_NAME + HIER_MEMBER_UNIQUE_NAME HIER_ORDER HIER_PARENT HIER_WINDOW HIGH HINTSET_BEGIN + HINTSET_END HOST HOT HOUR HOURS HTTP HWM_BROKERED HYBRID ID IDENTIFIER IDENTITY + IDGENERATORS IDLE IDLE_TIME IF IGNORE IGNORE_OPTIM_EMBEDDED_HINTS IGNORE_ROW_ON_DUPKEY_INDEX + IGNORE_WHERE_CLAUSE ILM IMMEDIATE IMMUTABLE IMPACT IMPORT INACTIVE INACTIVE_ACCOUNT_TIME + INCLUDE INCLUDES INCLUDE_VERSION INCLUDING INCOMING INCR INCREMENT INCREMENTAL + INDENT INDEXED INDEXES INDEXING INDEXTYPE INDEXTYPES INDEX_ASC INDEX_COMBINE + INDEX_DESC INDEX_FFS INDEX_FILTER INDEX_JOIN INDEX_ROWS INDEX_RRS INDEX_RS + INDEX_RS_ASC INDEX_RS_DESC INDEX_SCAN INDEX_SKIP_SCAN INDEX_SS INDEX_SS_ASC + INDEX_SS_DESC INDEX_STATS INDICATOR INFINITE INFORMATIONAL INHERIT INITCAP + INITIAL INITIALIZED INITIALLY INITRANS INLINE INLINE_XMLTYPE_NT INLINE_XT + INMEMORY INMEMORY_PRUNING INNER INPLACE INSENSITIVE INSERTCHILDXML INSERTCHILDXMLAFTER + INSERTCHILDXMLBEFORE INSERTXMLAFTER INSERTXMLBEFORE INSTALL INSTANCE INSTANCES + INSTANTIABLE INSTANTLY INSTEAD INSTR INSTR2 INSTR4 INSTRB INSTRC INTERLEAVED + INTERMEDIATE INTERNAL_CONVERT INTERNAL_USE INTERPRETED INTRA_CDB INVALIDATE + INVALIDATION INVISIBLE IN_MEMORY_METADATA IN_XQUERY IOSEEKTIM IOTFRSPEED IO_LOGICAL + IO_MEGABYTES IO_REQUESTS ISOLATE ISOLATION ISOLATION_LEVEL ITERATE ITERATION_NUMBER + JAVA JOB JOIN JSON JSONGET JSONPARSE JSONTOXML JSON_ARRAY JSON_ARRAYAGG JSON_EQUAL + JSON_EQUAL2 JSON_EXISTS JSON_EXISTS2 JSON_HASH JSON_LENGTH JSON_MERGEPATCH + JSON_MKMVI JSON_OBJECT JSON_OBJECTAGG JSON_PATCH JSON_QUERY JSON_SCALAR JSON_SERIALIZE + JSON_TABLE JSON_TEXTCONTAINS JSON_TEXTCONTAINS2 JSON_TRANSFORM JSON_VALUE + KEEP KEEP_DUPLICATES KERBEROS KEY KEYS KEYSIZE KEYSTORE KEY_LENGTH KILL + KURTOSIS_POP KURTOSIS_SAMP LABEL LAG LAG_DIFF LAG_DIFF_PERCENT LANGUAGE LAST + LAST_DAY LAST_VALUE LATERAL LAX LAYER LDAP_REGISTRATION LDAP_REGISTRATION_ENABLED + LDAP_REG_SYNC_INTERVAL LEAD LEADING LEAD_CDB LEAD_CDB_URI LEAD_DIFF LEAD_DIFF_PERCENT + LEAF LEAST LEAVES LEDGER LEFT LENGTH LENGTH2 LENGTH4 LENGTHB LENGTHC LESS + LEVEL LEVELS LIBRARY LIFE LIFECYCLE LIFETIME LIKE2 LIKE4 LIKEC LIMIT LINEAR + LINK LIST LISTAGG LN LNNVL LOAD LOB LOBNVL LOBS LOB_VALUE LOCALTIME LOCALTIMESTAMP + LOCAL_INDEXES LOCATION LOCATOR LOCKDOWN LOCKED LOCKING LOG LOGFILE LOGFILES + LOGGING LOGICAL LOGICAL_READS_PER_CALL LOGICAL_READS_PER_SESSION LOGMINING + LOGOFF LOGON LOG_READ_ONLY_VIOLATIONS LOST LOW LOWER LPAD LTRIM MAIN MAKE_REF + MANAGE MANAGED MANAGEMENT MANAGER MANDATORY MANUAL MAP MAPPER MAPPING MASTER + MATCH MATCHED MATCHES MATCH_NUMBER MATCH_RECOGNIZE MATERIALIZE MATERIALIZED + MATRIX MAX MAXARCHLOGS MAXDATAFILES MAXEXTENTS MAXIMIZE MAXINSTANCES MAXLOGFILES + MAXLOGHISTORY MAXLOGMEMBERS MAXSIZE MAXTRANS MAXVALUE MAX_AUDIT_SIZE MAX_DIAG_SIZE + MAX_PDB_SNAPSHOTS MAX_SHARED_TEMP_SIZE MBRC MEASURE MEASURES MEDIAN MEDIUM + MEMBER MEMCOMPRESS MEMOPTIMIZE MEMOPTIMIZE_WRITE MEMORY MERGE MERGE$ACTIONS + MERGE_AJ MERGE_CONST_ON MERGE_SJ METADATA METADATA_SOURCE_PDB METHOD MIGRATE + MIGRATE_CROSS_CON MIGRATION MIN MINEXTENTS MINIMIZE MINIMUM MINING MINUS_NULL + MINUTE MINUTES MINVALUE MIRROR MIRRORCOLD MIRRORHOT MISMATCH MISSING MLE MLSLABEL + MOD MODEL MODEL_COMPILE_SUBQUERY MODEL_DONTVERIFY_UNIQUENESS MODEL_DYNAMIC_SUBQUERY + MODEL_MIN_ANALYSIS MODEL_NB MODEL_NO_ANALYSIS MODEL_PBY MODEL_PUSH_REF MODEL_SV + MODIFICATION MODIFY MODIFY_COLUMN_TYPE MODULE MONITOR MONITORING MONTHS MONTHS_BETWEEN + MOUNT MOUNTPATH MOUNTPOINT MOVE MOVEMENT MULTIDIMENSIONAL MULTISET MULTIVALUE + MV_MERGE NAME NAMED NAMES NAMESPACE NAN NANVL NATIVE NATIVE_FULL_OUTER_JOIN + NATURAL NAV NCHAR_CS NCHR NEEDED NEG NESTED NESTED_ROLLUP_TOP NESTED_TABLE_FAST_INSERT + NESTED_TABLE_GET_REFS NESTED_TABLE_ID NESTED_TABLE_SET_REFS NESTED_TABLE_SET_SETID + NETWORK NEVER NEW NEW_TIME NEXT NEXT_DAY NLJ_BATCHING NLJ_INDEX_FILTER NLJ_INDEX_SCAN + NLJ_PREFETCH NLSSORT NLS_CALENDAR NLS_CHARACTERSET NLS_CHARSET_DECL_LEN NLS_CHARSET_ID + NLS_CHARSET_NAME NLS_COLLATION_ID NLS_COLLATION_NAME NLS_COMP NLS_CURRENCY + NLS_DATE_FORMAT NLS_DATE_LANGUAGE NLS_INITCAP NLS_ISO_CURRENCY NLS_LANG NLS_LANGUAGE + NLS_LENGTH_SEMANTICS NLS_LOWER NLS_NCHAR_CONV_EXCP NLS_NUMERIC_CHARACTERS + NLS_SORT NLS_SPECIAL_CHARS NLS_TERRITORY NLS_UPPER NL_AJ NL_SJ NO NOAPPEND + NOARCHIVELOG NOAUDIT NOCACHE NOCOPY NOCPU_COSTING NOCYCLE NODELAY NOENTITYESCAPING + NOEXTEND NOFORCE NOGUARANTEE NOKEEP NOLOCAL NOLOGGING NOMAPPING NOMAXVALUE + NOMINIMIZE NOMINVALUE NOMONITORING NONBLOCKING NONE NONEDITIONABLE NONPARTITIONED + NONSCHEMA NOORDER NOOVERRIDE NOPARALLEL NOPARALLEL_INDEX NORELOCATE NORELY + NOREPAIR NOREPLAY NORESETLOGS NOREVERSE NOREWRITE NORMAL NOROWDEPENDENCIES + NOSCALE NOSCHEMACHECK NOSEGMENT NOSHARD NOSORT NOSTRICT NOSWITCH NOTHING NOTIFICATION + NOVALIDATE NOW NO_ACCESS NO_ADAPTIVE_PLAN NO_ANSI_REARCH NO_ANSWER_QUERY_USING_STATS + NO_AUTO_REOPTIMIZE NO_BAND_JOIN NO_BASETABLE_MULTIMV_REWRITE NO_BATCH_TABLE_ACCESS_BY_ROWID + NO_BIND_AWARE NO_BUFFER NO_BUSHY_JOIN NO_CARTESIAN NO_CHECK_ACL_REWRITE NO_CLUSTERING + NO_CLUSTER_BY_ROWID NO_COALESCE_SQ NO_COMMON_DATA NO_CONNECT_BY_CB_WHR_ONLY + NO_CONNECT_BY_COMBINE_SW NO_CONNECT_BY_COST_BASED NO_CONNECT_BY_ELIM_DUPS + NO_CONNECT_BY_FILTERING NO_CONTAINERS NO_COST_XML_QUERY_REWRITE NO_CPU_COSTING + NO_CROSS_CONTAINER NO_DAGG_OPTIM_GSETS NO_DATA_SECURITY_REWRITE NO_DECORRELATE + NO_DIST_AGG_PROLLUP_PUSHDOWN NO_DOMAIN_INDEX_FILTER NO_DST_UPGRADE_INSERT_CONV + NO_ELIMINATE_JOIN NO_ELIMINATE_OBY NO_ELIMINATE_OUTER_JOIN NO_ELIMINATE_SQ + NO_ELIM_GROUPBY NO_EXPAND NO_EXPAND_GSET_TO_UNION NO_EXPAND_TABLE NO_FACT + NO_FACTORIZE_JOIN NO_FILTERING NO_FULL_OUTER_JOIN_TO_OUTER NO_GATHER_OPTIMIZER_STATISTICS + NO_GBY_PUSHDOWN NO_INDEX NO_INDEX_FFS NO_INDEX_SS NO_INMEMORY NO_INMEMORY_PRUNING + NO_JSON_TABLE_TRANSFORM NO_LOAD NO_MERGE NO_MODEL_PUSH_REF NO_MONITOR NO_MONITORING + NO_MULTIMV_REWRITE NO_NATIVE_FULL_OUTER_JOIN NO_NLJ_BATCHING NO_NLJ_PREFETCH + NO_OBJECT_LINK NO_OBY_GBYPD_SEPARATE NO_ORDER_ROLLUPS NO_OR_EXPAND NO_OUTER_JOIN_TO_ANTI + NO_OUTER_JOIN_TO_INNER NO_PARALLEL NO_PARALLEL_INDEX NO_PARTIAL_COMMIT NO_PARTIAL_JOIN + NO_PARTIAL_OSON_UPDATE NO_PARTIAL_ROLLUP_PUSHDOWN NO_PLACE_DISTINCT NO_PLACE_GROUP_BY + NO_PQ_CONCURRENT_UNION NO_PQ_EXPAND_TABLE NO_PQ_MAP NO_PQ_NONLEAF_SKEW NO_PQ_REPLICATE + NO_PQ_SKEW NO_PRUNE_GSETS NO_PULL_PRED NO_PUSH_HAVING_TO_GBY NO_PUSH_PRED + NO_PUSH_SUBQ NO_PX_FAULT_TOLERANCE NO_PX_JOIN_FILTER NO_QKN_BUFF NO_QUERY_TRANSFORMATION + NO_REF_CASCADE NO_REORDER_WIF NO_RESULT_CACHE NO_REWRITE NO_ROOT_SW_FOR_LOCAL + NO_SEMIJOIN NO_SEMI_TO_INNER NO_SET_GBY_PUSHDOWN NO_SET_TO_JOIN NO_SQL_TRANSLATION + NO_SQL_TUNE NO_STAR_TRANSFORMATION NO_STATEMENT_QUEUING NO_STATS_GSETS NO_SUBQUERY_PRUNING + NO_SUBSTRB_PAD NO_SWAP_JOIN_INPUTS NO_TABLE_LOOKUP_BY_NL NO_TEMP_TABLE NO_TRANSFORM_DISTINCT_AGG + NO_UNNEST NO_USE_CUBE NO_USE_DAGG_UNION_ALL_GSETS NO_USE_HASH NO_USE_HASH_AGGREGATION + NO_USE_HASH_GBY_FOR_DAGGPSHD NO_USE_HASH_GBY_FOR_PUSHDOWN NO_USE_INVISIBLE_INDEXES + NO_USE_MERGE NO_USE_NL NO_USE_PARTITION_WISE_DISTINCT NO_USE_PARTITION_WISE_GBY + NO_USE_PARTITION_WISE_WIF NO_USE_SCALABLE_GBY_INVDIST NO_USE_VECTOR_AGGREGATION + NO_VECTOR_TRANSFORM NO_VECTOR_TRANSFORM_DIMS NO_VECTOR_TRANSFORM_FACT NO_XDB_FASTPATH_INSERT + NO_XMLINDEX_REWRITE NO_XMLINDEX_REWRITE_IN_SELECT NO_XML_DML_REWRITE NO_XML_QUERY_REWRITE + NO_ZONEMAP NTH_VALUE NTILE NULLIF NULLS NUMTODSINTERVAL NUMTOYMINTERVAL NUM_INDEX_KEYS + NVL NVL2 OBJECT OBJECT2XML OBJNO OBJNO_REUSE OBJ_ID OBY_GBYPD_SEPARATE OCCURENCES + OCCURRENCES ODD OFF OFFLINE OFFSET OID OIDINDEX OLAP OLD OLD_PUSH_PRED OLS + OLTP OMIT ONE ONLINE ONLY OPAQUE OPAQUE_TRANSFORM OPAQUE_XCANONICAL OPCODE + OPEN OPERATIONS OPERATOR OPTIMAL OPTIMIZE OPTIMIZER_FEATURES_ENABLE OPTIMIZER_GOAL + OPT_ESTIMATE OPT_PARAM ORADEBUG ORA_BRANCH ORA_CHECK_ACL ORA_CHECK_PRIVILEGE + ORA_CHECK_SYS_PRIVILEGE ORA_CLUSTERING ORA_CONCAT_RWKEY ORA_DM_PARTITION_NAME + ORA_DST_AFFECTED ORA_DST_CONVERT ORA_DST_ERROR ORA_GET_ACLIDS ORA_GET_PRIVILEGES + ORA_HASH ORA_INVOKING_USER ORA_INVOKING_USERID ORA_INVOKING_XS_USER ORA_INVOKING_XS_USER_GUID + ORA_NORMALIZE ORA_PARTITION_VALIDATION ORA_RAWCOMPARE ORA_RAWCONCAT ORA_ROWSCN + ORA_ROWSCN_RAW ORA_ROWVERSION ORA_SEARCH_RWKEY ORA_SHARDSPACE_NAME ORA_SHARD_ID + ORA_TABVERSION ORA_WRITE_TIME ORDERED ORDERED_PREDICATES ORDER_KEY_VECTOR_USE + ORDER_SUBQ ORDINALITY ORGANIZATION OR_EXPAND OR_PREDICATES OSON OSON_DIAG + OSON_GET_CONTENT OTHER OTHERS OUTER OUTER_JOIN_TO_ANTI OUTER_JOIN_TO_INNER + OUTLINE OUTLINE_LEAF OUT_OF_LINE OVER OVERFLOW OVERFLOW_NOMOVE OVERLAPS OWN + OWNER OWNERSHIP PACKAGE PACKAGES PARALLEL PARALLEL_INDEX PARAM PARAMETERS + PARENT PARITY PART$NUM$INST PARTIAL PARTIALLY PARTIAL_JOIN PARTIAL_ROLLUP_PUSHDOWN + PARTITION PARTITIONING PARTITIONS PARTITIONSET PARTITION_CONTAINED PARTITION_HASH + PARTITION_LIST PARTITION_RANGE PASSING PASSIVE PASSWORD PASSWORDFILE_METADATA_CACHE + PASSWORD_GRACE_TIME PASSWORD_LIFE_TIME PASSWORD_LOCK_TIME PASSWORD_REUSE_MAX + PASSWORD_REUSE_TIME PASSWORD_ROLLOVER_TIME PASSWORD_VERIFY_FUNCTION PAST PATCH + PATH PATHS PATH_PREFIX PATTERN PBL_HS_BEGIN PBL_HS_END PCTINCREASE PCTTHRESHOLD + PCTUSED PCTVERSION PDB_LOCAL_ONLY PEER PEERS PENDING PER PERCENT PERCENTAGE + PERCENTILE_CONT PERCENTILE_DISC PERCENT_RANK PERCENT_RANKM PERFORMANCE PERIOD + PERMANENT PERMISSION PERMUTE PERSISTABLE PFILE PHV PHYSICAL PIKEY PIVOT PIV_GB + PIV_SSF PLACE_DISTINCT PLACE_GROUP_BY PLAN PLSCOPE_SETTINGS PLSQL_CCFLAGS + PLSQL_CODE_TYPE PLSQL_DEBUG PLSQL_OPTIMIZE_LEVEL PLSQL_WARNINGS PLUGGABLE + PMEM POINT POLICY POOL_16K POOL_2K POOL_32K POOL_4K POOL_8K PORT POSITION + POST_TRANSACTION POWER POWERMULTISET POWERMULTISET_BY_CARDINALITY PQ_CONCURRENT_UNION + PQ_DISTRIBUTE PQ_DISTRIBUTE_WINDOW PQ_EXPAND_TABLE PQ_FILTER PQ_MAP PQ_NOMAP + PQ_NONLEAF_SKEW PQ_REPLICATE PQ_SKEW PREBUILT PRECEDES PRECEDING PRECOMPUTE_SUBQUERY + PREDICATE_REORDERS PREDICTION PREDICTION_BOUNDS PREDICTION_COST PREDICTION_DETAILS + PREDICTION_PROBABILITY PREDICTION_SET PRELOAD PREPARE PRESENT PRESENTNNV PRESENTV + PRESERVE PRESERVE_OID PRETTY PREV PREVIOUS PRIMARY PRINTBLOBTOCLOB PRIORITY + PRIVATE PRIVATE_SGA PRIVILEGE PRIVILEGED PRIVILEGES PROCEDURAL PROCEDURE PROCESS + PROFILE PROGRAM PROJECT PROPAGATE PROPAGATION PROPERTY PROTECTED PROTECTION + PROTOCOL PROXY PRUNING PULL_PRED PURGE PUSH_HAVING_TO_GBY PUSH_PRED PUSH_SUBQ + PX_FAULT_TOLERANCE PX_GRANULE PX_JOIN_FILTER QB_NAME QUALIFY QUARANTINE QUARTERS + QUERY QUERY_BLOCK QUEUE QUEUE_CURR QUEUE_ROWP QUIESCE QUORUM QUOTA QUOTAGROUP + QUOTES RANDOM RANDOM_LOCAL RANGE RANK RANKM RAPIDLY RATIO_TO_REPORT RAWTOHEX + RAWTONHEX RAWTOREF RBA RBO_OUTLINE RDBA READ READS READ_OR_WRITE REALM REBALANCE + REBUILD RECONNECT RECORDS_PER_BLOCK RECOVER RECOVERABLE RECOVERY RECYCLE RECYCLEBIN + REDACTION REDEFINE REDO REDUCED REDUNDANCY REFERENCE REFERENCED REFERENCES + REFERENCING REFRESH REFTOHEX REFTORAW REF_CASCADE_CURSOR REGEXP_COUNT REGEXP_INSTR + REGEXP_LIKE REGEXP_REPLACE REGEXP_SUBSTR REGISTER REGR_AVGX REGR_AVGY REGR_COUNT + REGR_INTERCEPT REGR_R2 REGR_SLOPE REGR_SXX REGR_SXY REGR_SYY REGULAR REJECT + REKEY RELATIONAL RELOCATE RELY REMAINDER REMOTE REMOTE_MAPPED REMOVE REORDER_WIF + REPAIR REPEAT REPLACE REPLICATION REQUIRED RESERVOIR_SAMPLING RESET RESETLOGS + RESIZE RESOLVE RESOLVER RESPECT RESTART RESTORE RESTORE_AS_INTERVALS RESTRICT + RESTRICTED RESTRICT_ALL_REF_CONS RESULT_CACHE RESUMABLE RESUME RETENTION RETRY_ON_ROW_CHANGE + RETURN RETURNING REUSE REVERSE REWRITE REWRITE_OR_ERROR RIGHT RLS_FORCE ROLE + ROLES ROLESET ROLLBACK ROLLING ROLLOVER ROLLUP ROOT ROUND ROUND_TIES_TO_EVEN + ROW ROWDEPENDENCIES ROWIDTOCHAR ROWIDTONCHAR ROWID_MAPPING_TABLE ROWNUM ROWS + ROW_LENGTH ROW_NUMBER RPAD RTRIM RULE RULES RUNNING SALT SAMPLE SAVE SAVEPOINT + SAVE_AS_INTERVALS SB4 SCALAR SCALARS SCALE SCALE_ROWS SCAN SCAN_INSTANCES + SCHEDULER SCHEMA SCHEMACHECK SCN SCN_ASCENDING SCOPE SCRUB SDO_GEOM_KEY SDO_GEOM_MAX_X + SDO_GEOM_MAX_Y SDO_GEOM_MAX_Z SDO_GEOM_MBB SDO_GEOM_MBR SDO_GEOM_MIN_X SDO_GEOM_MIN_Y + SDO_GEOM_MIN_Z SDO_TOLERANCE SD_ALL SD_INHIBIT SD_SHOW SEARCH SECONDS SECRET + SECUREFILE SECUREFILE_DBA SECURITY SEED SEGMENT SEG_BLOCK SEG_FILE SELECTIVITY + SELF SEMIJOIN SEMIJOIN_DRIVER SEMI_TO_INNER SENSITIVE SEQUENCE SEQUENCED SEQUENTIAL + SERIAL SERIALIZABLE SERVERERROR SERVICE SERVICES SERVICE_NAME_CONVERT SESSION + SESSIONS_PER_USER SESSIONTIMEZONE SESSIONTZNAME SESSION_CACHED_CURSORS SETS + SETTINGS SET_GBY_PUSHDOWN SET_TO_JOIN SEVERE SHARD SHARDED SHARDS SHARDSPACE + SHARD_CHUNK_ID SHARED SHARED_POOL SHARE_OF SHARING SHD$COL$MAP SHELFLIFE SHOW + SHRINK SHUTDOWN SIBLING SIBLINGS SID SIGN SIGNAL_COMPONENT SIGNAL_FUNCTION + SIGNATURE SIMPLE SIN SINGLE SINGLETASK SINH SITE SKEWNESS_POP SKEWNESS_SAMP + SKIP SKIP_EXT_OPTIMIZER SKIP_PROXY SKIP_UNQ_UNUSABLE_IDX SKIP_UNUSABLE_INDEXES + SMALLFILE SNAPSHOT SOME SORT SOUNDEX SOURCE SOURCE_FILE_DIRECTORY SOURCE_FILE_NAME_CONVERT + SPACE SPATIAL SPECIFICATION SPFILE SPLIT SPREADSHEET SQL SQLLDR SQL_SCOPE + SQL_TRACE SQL_TRANSLATION_PROFILE SQRT STALE STANDALONE STANDARD_HASH STANDBY + STANDBYS STANDBY_MAX_DATA_DELAY STAR STARTUP STAR_TRANSFORMATION STATE STATEMENT + STATEMENTS STATEMENT_ID STATEMENT_QUEUING STATIC STATISTICS STATS_BINOMIAL_TEST + STATS_CROSSTAB STATS_F_TEST STATS_KS_TEST STATS_MODE STATS_MW_TEST STATS_ONE_WAY_ANOVA + STATS_T_TEST_INDEP STATS_T_TEST_INDEPU STATS_T_TEST_ONE STATS_T_TEST_PAIRED + STATS_WSR_TEST STDDEV STDDEV_POP STDDEV_SAMP STOP STORAGE STORAGE_INDEX STORE + STREAM STREAMS STRICT STRING STRINGS STRIP STRIPE_COLUMNS STRIPE_WIDTH STRUCTURE + SUBMULTISET SUBPARTITION SUBPARTITIONING SUBPARTITIONS SUBPARTITION_REL SUBQUERIES + SUBQUERY_PRUNING SUBSCRIBE SUBSET SUBSTITUTABLE SUBSTR SUBSTR2 SUBSTR4 SUBSTRB + SUBSTRC SUBTYPE SUCCESS SUCCESSFUL SUM SUMMARY SUPPLEMENTAL SUPPRESS_LOAD + SUSPEND SWAP_JOIN_INPUTS SWITCH SWITCHOVER SYNC SYNCHRONOUS SYSASM SYSAUX + SYSBACKUP SYSDATE SYSDBA SYSDG SYSGUID SYSKM SYSOBJ SYSOPER SYSRAC SYSTEM + SYSTEM_DEFINED SYSTEM_STATS SYSTIMESTAMP SYS_AUDIT SYS_CHECKACL SYS_CHECK_PRIVILEGE + SYS_CONNECT_BY_PATH SYS_CONS_ANY_SCALAR SYS_CONTEXT SYS_CTXINFOPK SYS_CTX_CONTAINS2 + SYS_CTX_MKIVIDX SYS_DBURIGEN SYS_DL_CURSOR SYS_DM_RXFORM_CHR SYS_DM_RXFORM_LAB + SYS_DM_RXFORM_NUM SYS_DOM_COMPARE SYS_DST_PRIM2SEC SYS_DST_SEC2PRIM SYS_ET_BFILE_TO_RAW + SYS_ET_BLOB_TO_IMAGE SYS_ET_IMAGE_TO_BLOB SYS_ET_RAW_TO_BFILE SYS_EXTPDTXT + SYS_EXTRACT_UTC SYS_FBT_INSDEL SYS_FILTER_ACLS SYS_FNMATCHES SYS_FNREPLACE + SYS_GETTOKENID SYS_GETXTIVAL SYS_GET_ACLIDS SYS_GET_ANY_SCALAR SYS_GET_COL_ACLIDS + SYS_GET_PRIVILEGES SYS_GUID SYS_MAKEXML SYS_MAKE_XMLNODEID SYS_MKXMLATTR SYS_MKXTI + SYS_OPTLOBPRBSC SYS_OPTXICMP SYS_OPTXQCASTASNQ SYS_OP_ADT2BIN SYS_OP_ADTCONS + SYS_OP_ALSCRVAL SYS_OP_ATG SYS_OP_BIN2ADT SYS_OP_BITVEC SYS_OP_BL2R SYS_OP_BLOOM_FILTER + SYS_OP_BLOOM_FILTER_LIST SYS_OP_C2C SYS_OP_CAST SYS_OP_CEG SYS_OP_CL2C SYS_OP_COMBINED_HASH + SYS_OP_COMP SYS_OP_CONVERT SYS_OP_COUNTCHG SYS_OP_CSCONV SYS_OP_CSCONVTEST + SYS_OP_CSR SYS_OP_CSX_PATCH SYS_OP_CYCLED_SEQ SYS_OP_DECOMP SYS_OP_DESCEND + SYS_OP_DISTINCT SYS_OP_DRA SYS_OP_DSB_DESERIALIZE SYS_OP_DSB_SERIALIZE SYS_OP_DUMP + SYS_OP_DV_CHECK SYS_OP_ENFORCE_NOT_NULL$ SYS_OP_EXTRACT SYS_OP_GROUPING SYS_OP_GUID + SYS_OP_HASH SYS_OP_HCS_TABLE SYS_OP_IIX SYS_OP_INTERVAL_HIGH_BOUND SYS_OP_ITR + SYS_OP_KEY_VECTOR_CREATE SYS_OP_KEY_VECTOR_FILTER SYS_OP_KEY_VECTOR_FILTER_LIST + SYS_OP_KEY_VECTOR_PAYLOAD SYS_OP_KEY_VECTOR_SUCCEEDED SYS_OP_KEY_VECTOR_USE + SYS_OP_LBID SYS_OP_LOBLOC2BLOB SYS_OP_LOBLOC2CLOB SYS_OP_LOBLOC2ID SYS_OP_LOBLOC2NCLOB + SYS_OP_LOBLOC2TYP SYS_OP_LSVI SYS_OP_LVL SYS_OP_MAKEOID SYS_OP_MAP_NONNULL + SYS_OP_MSR SYS_OP_NICOMBINE SYS_OP_NIEXTRACT SYS_OP_NII SYS_OP_NIX SYS_OP_NOEXPAND + SYS_OP_NTCIMG$ SYS_OP_NUMTORAW SYS_OP_OBJ_UPD_IN_TXN SYS_OP_OIDVALUE SYS_OP_OPNSIZE + SYS_OP_PAR SYS_OP_PARGID SYS_OP_PARGID_1 SYS_OP_PART_ID SYS_OP_PAR_1 SYS_OP_PIVOT + SYS_OP_R2O SYS_OP_RAWTONUM SYS_OP_RDTM SYS_OP_REF SYS_OP_RMTD SYS_OP_ROWIDTOOBJ + SYS_OP_RPB SYS_OP_TOSETID SYS_OP_TPR SYS_OP_TRTB SYS_OP_UNDESCEND SYS_OP_VECAND + SYS_OP_VECBIT SYS_OP_VECOR SYS_OP_VECTOR_GROUP_BY SYS_OP_VECXOR SYS_OP_VERSION + SYS_OP_VREF SYS_OP_VVD SYS_OP_XMLCONS_FOR_CSX SYS_OP_XPTHATG SYS_OP_XPTHIDX + SYS_OP_XPTHOP SYS_OP_XTNN SYS_OP_XTXT2SQLT SYS_OP_ZONE_ID SYS_ORDERKEY_DEPTH + SYS_ORDERKEY_MAXCHILD SYS_ORDERKEY_PARENT SYS_PARALLEL_TXN SYS_PATHID_IS_ATTR + SYS_PATHID_IS_NMSPC SYS_PATHID_LASTNAME SYS_PATHID_LASTNMSPC SYS_PATH_REVERSE + SYS_PLSQL_COUNT SYS_PLSQL_CPU SYS_PLSQL_IO SYS_PXQEXTRACT SYS_RAW_TO_XSID + SYS_REMAP_XMLTYPE SYS_RID_ORDER SYS_ROW_DELTA SYS_SC_2_XMLT SYS_SYNRCIREDO + SYS_TYPEID SYS_UMAKEXML SYS_XMLANALYZE SYS_XMLCONTAINS SYS_XMLCONV SYS_XMLEXNSURI + SYS_XMLGEN SYS_XMLINSTR SYS_XMLI_LOC_ISNODE SYS_XMLI_LOC_ISTEXT SYS_XMLLOCATOR_GETSVAL + SYS_XMLNODEID SYS_XMLNODEID_GETCID SYS_XMLNODEID_GETLOCATOR SYS_XMLNODEID_GETOKEY + SYS_XMLNODEID_GETPATHID SYS_XMLNODEID_GETPTRID SYS_XMLNODEID_GETRID SYS_XMLNODEID_GETSVAL + SYS_XMLNODEID_GETTID SYS_XMLTRANSLATE SYS_XMLTYPE2SQL SYS_XMLT_2_SC SYS_XQBASEURI + SYS_XQCASTABLEERRH SYS_XQCODEP2STR SYS_XQCODEPEQ SYS_XQCON2SEQ SYS_XQCONCAT + SYS_XQDELETE SYS_XQDFLTCOLATION SYS_XQDOC SYS_XQDOCURI SYS_XQDURDIV SYS_XQED4URI + SYS_XQENDSWITH SYS_XQERR SYS_XQERRH SYS_XQESHTMLURI SYS_XQEXLOBVAL SYS_XQEXSTWRP + SYS_XQEXTRACT SYS_XQEXTRREF SYS_XQEXVAL SYS_XQFB2STR SYS_XQFNBOOL SYS_XQFNCMP + SYS_XQFNDATIM SYS_XQFNLNAME SYS_XQFNNM SYS_XQFNNSURI SYS_XQFNPREDTRUTH SYS_XQFNQNM + SYS_XQFNROOT SYS_XQFORMATNUM SYS_XQFTCONTAIN SYS_XQFUNCR SYS_XQGETCONTENT + SYS_XQINDXOF SYS_XQINSERT SYS_XQINSPFX SYS_XQIRI2URI SYS_XQLANG SYS_XQLLNMFRMQNM + SYS_XQMKNODEREF SYS_XQNILLED SYS_XQNODENAME SYS_XQNORMSPACE SYS_XQNORMUCODE + SYS_XQNSP4PFX SYS_XQNSPFRMQNM SYS_XQPFXFRMQNM SYS_XQPOLYABS SYS_XQPOLYADD + SYS_XQPOLYCEL SYS_XQPOLYCST SYS_XQPOLYCSTBL SYS_XQPOLYDIV SYS_XQPOLYFLR SYS_XQPOLYMOD + SYS_XQPOLYMUL SYS_XQPOLYRND SYS_XQPOLYSQRT SYS_XQPOLYSUB SYS_XQPOLYUMUS SYS_XQPOLYUPLS + SYS_XQPOLYVEQ SYS_XQPOLYVGE SYS_XQPOLYVGT SYS_XQPOLYVLE SYS_XQPOLYVLT SYS_XQPOLYVNE + SYS_XQREF2VAL SYS_XQRENAME SYS_XQREPLACE SYS_XQRESVURI SYS_XQRNDHALF2EVN SYS_XQRSLVQNM + SYS_XQRYENVPGET SYS_XQRYVARGET SYS_XQRYWRP SYS_XQSEQ2CON SYS_XQSEQ2CON4XC + SYS_XQSEQDEEPEQ SYS_XQSEQINSB SYS_XQSEQRM SYS_XQSEQRVS SYS_XQSEQSUB SYS_XQSEQTYPMATCH + SYS_XQSTARTSWITH SYS_XQSTATBURI SYS_XQSTR2CODEP SYS_XQSTRJOIN SYS_XQSUBSTRAFT + SYS_XQSUBSTRBEF SYS_XQTOKENIZE SYS_XQTREATAS SYS_XQXFORM SYS_XQ_ASQLCNV SYS_XQ_ATOMCNVCHK + SYS_XQ_NRNG SYS_XQ_PKSQL2XML SYS_XQ_UPKXML2SQL SYS_XSID_TO_RAW SYS_ZMAP_FILTER + SYS_ZMAP_REFRESH TABAUTH TABLES TABLESPACE TABLESPACE_NO TABLE_LOOKUP_BY_NL + TABLE_STATS TABNO TAG TAN TANH TARGET TBL$OR$IDX$PART$NUM TEMP TEMPFILE TEMPLATE + TEMPORARY TEMP_TABLE TENANT_ID TEST TEXT THAN THE THREAD THROUGH TIER TIES + TIMEOUT TIMES TIMESTAMP_TO_NUMBER TIMEZONE_ABBR TIMEZONE_HOUR TIMEZONE_MINUTE + TIMEZONE_OFFSET TIMEZONE_REGION TIME_ZONE TIV_GB TIV_SSF TOKEN TOPLEVEL TO_ACLID + TO_APPROX_COUNT_DISTINCT TO_APPROX_PERCENTILE TO_BINARY_DOUBLE TO_BINARY_FLOAT + TO_BLOB TO_CHAR TO_CLOB TO_DATE TO_DSINTERVAL TO_ISO_STRING TO_LOB TO_MULTI_BYTE + TO_NCHAR TO_NCLOB TO_NUMBER TO_SINGLE_BYTE TO_TIME TO_TIMESTAMP TO_TIMESTAMP_TZ + TO_TIME_TZ TO_UTC_TIMESTAMP_TZ TO_YMINTERVAL TRACE TRACING TRACKING TRAILING + TRANSACTION TRANSFORM TRANSFORM_DISTINCT_AGG TRANSITION TRANSITIONAL TRANSLATE + TRANSLATION TRANSPORTABLE TREAT TRIGGERS TRIM TRUE TRUNC TRUNCATE TRUST TRUSTED + TUNING TX TYPE TYPENAME TYPES TZ_OFFSET UB2 UBA UCS2 UID UNARCHIVED UNBOUND + UNBOUNDED UNCONDITIONAL UNDER UNDO UNDROP UNIFORM UNINSTALL UNION_ALL UNISTR + UNITE UNIXTIME UNLIMITED UNLOAD UNLOCK UNMATCHED UNNEST UNNEST_INNERJ_DISTINCT_VIEW + UNNEST_NOSEMIJ_NODISTINCTVIEW UNNEST_SEMIJ_VIEW UNPACKED UNPIVOT UNPLUG UNPROTECTED + UNQUIESCE UNRECOVERABLE UNRESTRICTED UNSUBSCRIBE UNTIL UNUSABLE UNUSED UPDATABLE + UPDATED UPDATEXML UPD_INDEXES UPD_JOININDEX UPGRADE UPPER UPSERT USABLE USAGE + USE USER USERENV USERGROUP USERS USER_DATA USER_DEFINED USER_RECYCLEBIN USER_TABLESPACES + USE_ANTI USE_CONCAT USE_CUBE USE_DAGG_UNION_ALL_GSETS USE_HASH USE_HASH_AGGREGATION + USE_HASH_GBY_FOR_DAGGPSHD USE_HASH_GBY_FOR_PUSHDOWN USE_HIDDEN_PARTITIONS + USE_INVISIBLE_INDEXES USE_MERGE USE_MERGE_CARTESIAN USE_NL USE_NL_WITH_INDEX + USE_PARTITION_WISE_DISTINCT USE_PARTITION_WISE_GBY USE_PARTITION_WISE_WIF + USE_PRIVATE_OUTLINES USE_SCALABLE_GBY_INVDIST USE_SEMI USE_STORED_OUTLINES + USE_TTT_FOR_GSETS USE_VECTOR_AGGREGATION USE_WEAK_NAME_RESL USING USING_NO_EXPAND + UTF16BE UTF16LE UTF32 UTF8 V1 V2 VALIDATE VALIDATE_CONVERSION VALIDATION VALID_TIME_END + VALUE VARIANCE VARRAY VARRAYS VAR_POP VAR_SAMP VECTOR VECTOR_ENCODE VECTOR_READ + VECTOR_READ_TRACE VECTOR_TRANSFORM VECTOR_TRANSFORM_DIMS VECTOR_TRANSFORM_FACT + VERIFIER VERIFY VERSION VERSIONING VERSIONS VERSIONS_ENDSCN VERSIONS_ENDTIME + VERSIONS_OPERATION VERSIONS_STARTSCN VERSIONS_STARTTIME VERSIONS_XID VIEWS + VIOLATION VIRTUAL VISIBILITY VISIBLE VOLUME VSIZE WAIT WALLET WEEK WEEKS WELLFORMED + WHEN WHENEVER WHITESPACE WIDTH_BUCKET WINDOW WITHIN WITHOUT WITH_EXPRESSION + WITH_PLSQL WORK WRAPPED WRAPPER WRITE XDB_FASTPATH_INSERT XID XML XML2OBJECT + XMLATTRIBUTES XMLCAST XMLCDATA XMLCOLATTVAL XMLCOMMENT XMLCONCAT XMLDIFF XMLELEMENT + XMLEXISTS XMLEXISTS2 XMLFOREST XMLINDEX_REWRITE XMLINDEX_REWRITE_IN_SELECT + XMLINDEX_SEL_IDX_TBL XMLISNODE XMLISVALID XMLNAMESPACES XMLPARSE XMLPATCH + XMLPI XMLQUERY XMLQUERYVAL XMLROOT XMLSCHEMA XMLSERIALIZE XMLTABLE XMLTOJSON + XMLTOKENSET XMLTRANSFORM XMLTRANSFORMBLOB XMLTSET_DML_ENABLE XML_DIAG XML_DML_RWT_STMT + XPATHTABLE XS XS_SYS_CONTEXT X_DYN_PRUNE YEARS YES ZONEMAP + ) + + KEYWORDS_FUNC = Set.new %w( + ABS ACOS ADD_MONTHS APPROX_COUNT APPROX_COUNT_DISTINCT APPROX_COUNT_DISTINCT_AGG + APPROX_COUNT_DISTINCT_DETAIL APPROX_MEDIAN APPROX_PERCENTILE APPROX_PERCENTILE_AGG + APPROX_PERCENTILE_DETAIL APPROX_RANK APPROX_SUM ASCII ASCIISTR ASIN ATAN ATAN2 + AVG BFILENAME BIN_TO_NUM BITAND CARDINALITY CAST CEIL CHARTOROWID CHR CLUSTER_DETAILS + CLUSTER_DISTANCE CLUSTER_ID CLUSTER_PROBABILITY CLUSTER_SET COALESCE COLLATION + COLLECT COMPOSE CONCAT CONVERT CON_DBID_TO_ID CON_GUID_TO_ID CON_NAME_TO_ID + CON_UID_TO_ID CORR COS COSH COUNT COVAR_POP COVAR_SAMP CUME_DIST CURRENT_DATE + CURRENT_TIMESTAMP CV DATAOBJ_TO_MAT_PARTITION DATAOBJ_TO_PARTITION DBTIMEZONE + DECODE DECOMPOSE DENSE_RANK DEPTH DEREF DUMP EMPTY_BLOB EMPTY_CLOB EXISTSNODE + EXP EXTRACT EXTRACTVALUE FEATURE_COMPARE FEATURE_DETAILS FEATURE_ID FEATURE_SET + FEATURE_VALUE FIRST FIRST_VALUE FLOOR FROM_TZ GREATEST GROUPING GROUPING_ID + GROUP_ID HEXTORAW INITCAP INSTR ITERATION_NUMBER JSON_ARRAY JSON_ARRAYAGG + JSON_OBJECT JSON_OBJECTAGG JSON_QUERY JSON_TABLE JSON_VALUE LAG LAST LAST_DAY + LAST_VALUE LEAD LEAST LENGTH LISTAGG LN LNNVL LOCALTIMESTAMP LOG LOWER LPAD + LTRIM MAKE_REF MAX MEDIAN MIN MOD MONTHS_BETWEEN NANVL NCHR NEW_TIME NEXT_DAY + NLSSORT NLS_CHARSET_DECL_LEN NLS_CHARSET_ID NLS_CHARSET_NAME NLS_COLLATION_ID + NLS_COLLATION_NAME NLS_INITCAP NLS_LOWER NLS_UPPER NTH_VALUE NTILE NULLIF + NUMTODSINTERVAL NUMTOYMINTERVAL NVL NVL2 ORA_DM_PARTITION_NAME ORA_DST_AFFECTED + ORA_DST_CONVERT ORA_DST_ERROR ORA_HASH ORA_INVOKING_USER ORA_INVOKING_USERID + PATH PERCENTILE_CONT PERCENTILE_DISC PERCENT_RANK POWER POWERMULTISET POWERMULTISET_BY_CARDINALITY + PREDICTION PREDICTION_BOUNDS PREDICTION_COST PREDICTION_DETAILS PREDICTION_PROBABILITY + PREDICTION_SET PRESENTNNV PRESENTV PREVIOUS RANK RATIO_TO_REPORT RAWTOHEX + RAWTONHEX REFTOHEX REGEXP_COUNT REGEXP_INSTR REGEXP_REPLACE REGEXP_SUBSTR + REMAINDER REPLACE ROUND ROWIDTOCHAR ROWIDTONCHAR ROW_NUMBER RPAD RTRIM + SCN_TO_TIMESTAMP SESSIONTIMEZONE SET SIGN SIN SINH SOUNDEX SQRT STANDARD_HASH + STATS_BINOMIAL_TEST STATS_CROSSTAB STATS_F_TEST STATS_KS_TEST STATS_MODE STATS_MW_TEST + STATS_ONE_WAY_ANOVA STATS_WSR_TEST STDDEV STDDEV_POP STDDEV_SAMP SUBSTR SUM + SYSDATE SYSTIMESTAMP SYS_CONNECT_BY_PATH SYS_CONTEXT SYS_DBURIGEN SYS_EXTRACT_UTC + SYS_GUID SYS_OP_ZONE_ID SYS_TYPEID SYS_XMLAGG SYS_XMLGEN TAN TANH TIMESTAMP_TO_SCN + TO_APPROX_COUNT_DISTINCT TO_APPROX_PERCENTILE TO_BINARY_DOUBLE TO_BINARY_FLOAT + TO_BLOB TO_CHAR TO_CLOB TO_DATE TO_DSINTERVAL TO_LOB TO_MULTI_BYTE TO_NCHAR + TO_NCLOB TO_NUMBER TO_SINGLE_BYTE TO_TIMESTAMP TO_TIMESTAMP_TZ TO_YMINTERVAL + TRANSLATE TREAT TRIM TRUNC TZ_OFFSET UID UNISTR UPPER USER USERENV VALIDATE_CONVERSION + VALUE VARIANCE VAR_POP VAR_SAMP VSIZE WIDTH_BUCKET XMLAGG XMLCAST XMLCDATA + XMLCOLATTVAL XMLCOMMENT XMLCONCAT XMLDIFF XMLELEMENT XMLEXISTS XMLFOREST XMLISVALID + XMLPARSE XMLPATCH XMLPI XMLQUERY XMLROOT XMLSEQUENCE XMLSERIALIZE XMLTABLE + XMLTRANSFORM + ) + + KEYWORDS_TYPE = Set.new %w( + CHAR BYTE VARCHAR2 NCHAR NVARCHAR2 + NUMBER FLOAT BINARY_FLOAT BINARY_DOUBLE + LONG RAW + DATE TIMESTAMP INTERVAL LOCAL TIME ZONE TO MONTH SECOND YEAR DAY + BLOB CLOB NCLOB BFILE + UROWID + CHARACTER VARYING VARCHAR NATIONAL + NUMERIC DECIMAL DEC INTEGER INT SMALLINT DOUBLE PRECISION REAL + SDO_GEOMETRY SDO_TOPO_GEOMETRY SDO_GEORASTER + REF ANYTYPE ANYDATA ANYDATASET XMLTYPE HTTPURITYPE XDBURITYPE DUBRITYPE + BOOLEAN PLS_INTEGER BINARY_INTEGER SIMPLE_FLOAT SIMPLE_INTEGER SIMPLE_DOUBLE SYS_REFCURSOR + ) + end + end +end From ce98847edef6e2b589807da5412ebcc8e883bab5 Mon Sep 17 00:00:00 2001 From: jneen Date: Sat, 20 Jun 2026 01:40:15 -0400 Subject: [PATCH 124/154] use the keywords api for pony --- lib/rouge/lexers/pony.rb | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/lib/rouge/lexers/pony.rb b/lib/rouge/lexers/pony.rb index b0510b6505..a7b6f2e883 100644 --- a/lib/rouge/lexers/pony.rb +++ b/lib/rouge/lexers/pony.rb @@ -7,7 +7,7 @@ class Pony < RegexLexer tag 'pony' filenames '*.pony' - keywords = Set.new %w( + KEYWORDS = Set.new %w( actor addressof and as be break class compiler_intrinsic consume continue @@ -28,11 +28,11 @@ class compiler_intrinsic consume continue where while with ) - capabilities = Set.new %w( + CAPABILITIES = Set.new %w( box iso ref tag trn val ) - types = Set.new %w( + TYPES = Set.new %w( Number Signed Unsigned Float I8 I16 I32 I64 I128 U8 U32 U64 U128 F32 F64 EventID Align IntFormat NumberPrefix FloatFormat @@ -61,18 +61,11 @@ class compiler_intrinsic consume continue rule %r/0[xX][0-9a-fA-F_]+/, Num::Hex rule %r/(0|[1-9][0-9_]*)([LUu]|Lu|LU|uL|UL)?/, Num::Integer - rule %r/[a-z_][a-z0-9_]*/io do |m| - match = m[0] - - if capabilities.include?(match) - token Keyword::Declaration - elsif keywords.include?(match) - token Keyword::Reserved - elsif types.include?(match) - token Keyword::Type - else - token Name - end + keywords %r/[a-z_][a-z0-9_]*/i do + rule CAPABILITIES, Keyword::Declaration + rule KEYWORDS, Keyword::Reserved + rule TYPES, Keyword::Type + default Name end end From 3867a9aba787185cb878c47fb3de891f75d22e59 Mon Sep 17 00:00:00 2001 From: jneen Date: Sat, 20 Jun 2026 01:40:28 -0400 Subject: [PATCH 125/154] use the keywords api for postscript --- .rubocop_todo.yml | 1 - lib/rouge/lexers/postscript.rb | 34 ++++++++++++++++++---------------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index e25d403d5e..54e3b72cba 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -24,7 +24,6 @@ Naming/VariableName: # Offense count: 113 Rouge/NoBuildingAlternationPatternInRegexp: Exclude: - - 'lib/rouge/lexers/postscript.rb' - 'lib/rouge/lexers/r.rb' - 'lib/rouge/lexers/rust.rb' - 'lib/rouge/lexers/scala.rb' diff --git a/lib/rouge/lexers/postscript.rb b/lib/rouge/lexers/postscript.rb index 36031a3f7b..a9b4f4e31b 100644 --- a/lib/rouge/lexers/postscript.rb +++ b/lib/rouge/lexers/postscript.rb @@ -16,16 +16,17 @@ def self.detect?(text) return true if /^%!/ =~ text end - delimiter = :"()<>\\[\\]{}/%\\s" - delimiter_end = Regexp.new("(?=[#{delimiter}])") - valid_name_chars = Regexp.new("[^#{delimiter}]") - valid_name = /#{valid_name_chars}+#{delimiter_end}/ + delimiter = %r([()<>\[\]{}/%\s]) + name_char = %r([^()<>\[\]{}/%\s]) + delimiter_end = /(?=#{delimiter}|\z)/ + valid_name = /#{name_char}+/ # These keywords taken from # # Is there an authoritative list anywhere that doesn't involve # trawling documentation? - keywords = %w/abs add aload arc arcn array atan begin + BUILTINS = Set.new %w( + abs add aload arc arcn array atan begin bind ceiling charpath clip closepath concat concatmatrix copy cos currentlinewidth currentmatrix currentpoint curveto cvi cvs def defaultmatrix @@ -42,9 +43,14 @@ def self.detect?(text) setrgbcolor shfill show showpage sin sqrt stack stringwidth stroke strokepath sub syntaxerror transform translate truncate typecheck undefined - undefinedfilename undefinedresult/ + undefinedfilename undefinedresult + ) + + RESERVED = Set.new %w(eq ne ge gt le lt and or not if ifelse for forall) state :root do + rule %r'\s+', Text + # All comment types rule %r'^%!.+?$', Comment::Preproc rule %r'%%.*?$', Comment::Special @@ -59,12 +65,6 @@ def self.detect?(text) rule %r'[{}<>\[\]]', Punctuation - rule %r'(?:#{keywords.join('|')})#{delimiter_end}', Name::Builtin - - # Conditionals / flow control - rule %r'(eq|ne|g[et]|l[et]|and|or|not|if(?:else)?|for(?:all)?)#{delimiter_end}', Keyword::Reserved - rule %r'(false|true)#{delimiter_end}', Keyword::Constant - # Numbers rule %r'<[0-9A-Fa-f]+>#{delimiter_end}', Num::Hex # Slight abuse: use Oct to signify any explicit base system @@ -72,10 +72,12 @@ def self.detect?(text) rule %r'(\-|\+)?([0-9]+\.?|[0-9]*\.[0-9]+|[0-9]+\.[0-9]*)((e|E)[0-9]+)?#{delimiter_end}', Num::Float rule %r'(\-|\+)?[0-9]+#{delimiter_end}', Num::Integer - # Names - rule valid_name, Name::Function # Anything else is executed - - rule %r'\s+', Text + keywords valid_name do + rule Set['false', 'true'], Keyword::Constant + rule RESERVED, Keyword::Reserved + rule BUILTINS, Name::Builtin + default Name::Function + end end state :stringliteral do From f4dec25c06a129748f314961dac148d696d37031 Mon Sep 17 00:00:00 2001 From: jneen Date: Sun, 21 Jun 2026 01:09:14 -0400 Subject: [PATCH 126/154] use the keywords api for powershell --- lib/rouge/lexers/powershell.rb | 75 ++++++++++++++++++++-------------- 1 file changed, 45 insertions(+), 30 deletions(-) diff --git a/lib/rouge/lexers/powershell.rb b/lib/rouge/lexers/powershell.rb index dff7ca17a5..f5f4639481 100644 --- a/lib/rouge/lexers/powershell.rb +++ b/lib/rouge/lexers/powershell.rb @@ -19,31 +19,30 @@ class Powershell < RegexLexer ) # https://docs.microsoft.com/en-us/powershell/module/microsoft.powershell.core/about/about_automatic_variables?view=powershell-6 - AUTO_VARS = %w( - \$\$ \$\? \$\^ \$_ - \$args \$ConsoleFileName \$Error \$Event \$EventArgs \$EventSubscriber - \$ExecutionContext \$false \$foreach \$HOME \$Host \$input \$IsCoreCLR - \$IsLinux \$IsMacOS \$IsWindows \$LastExitCode \$Matches \$MyInvocation - \$NestedPromptLevel \$null \$PID \$PROFILE \$PSBoundParameters \$PSCmdlet - \$PSCommandPath \$PSCulture \$PSDebugContext \$PSHOME \$PSItem - \$PSScriptRoot \$PSSenderInfo \$PSUICulture \$PSVersionTable \$PWD - \$REPORTERRORSHOWEXCEPTIONCLASS \$REPORTERRORSHOWINNEREXCEPTION - \$REPORTERRORSHOWSOURCE \$REPORTERRORSHOWSTACKTRACE - \$SENDER \$ShellId \$StackTrace \$switch \$this \$true - ).join('|') + AUTO_VARS = Set.new %w( + $args $ConsoleFileName $Error $Event $EventArgs $EventSubscriber + $ExecutionContext $false $foreach $HOME $Host $input $IsCoreCLR + $IsLinux $IsMacOS $IsWindows $LastExitCode $Matches $MyInvocation + $NestedPromptLevel $null $PID $PROFILE $PSBoundParameters $PSCmdlet + $PSCommandPath $PSCulture $PSDebugContext $PSHOME $PSItem + $PSScriptRoot $PSSenderInfo $PSUICulture $PSVersionTable $PWD + $REPORTERRORSHOWEXCEPTIONCLASS $REPORTERRORSHOWINNEREXCEPTION + $REPORTERRORSHOWSOURCE $REPORTERRORSHOWSTACKTRACE + $SENDER $ShellId $StackTrace $switch $this $true + ) # https://docs.microsoft.com/en-us/powershell/module/microsoft.powershell.core/about/about_reserved_words?view=powershell-6 - KEYWORDS = %w( + KEYWORDS = Set.new %w( assembly exit process base filter public begin finally return break for sequence catch foreach static class from switch command function throw configuration hidden trap continue if try data in type define inlinescript until do interface using dynamicparam module var else namespace while elseif parallel workflow end param enum private - ).join('|') + ) # https://devblogs.microsoft.com/scripting/powertip-find-a-list-of-powershell-type-accelerators/ # ([PSObject].Assembly.GetType("System.Management.Automation.TypeAccelerators")::Get).Keys -join ' ' - KEYWORDS_TYPE = %w( + KEYWORDS_TYPE = Set.new %w( Alias AllowEmptyCollection AllowEmptyString AllowNull ArgumentCompleter array bool byte char CmdletBinding datetime decimal double DscResource float single guid hashtable int int32 int16 long int64 ciminstance @@ -60,34 +59,42 @@ class Powershell < RegexLexer wmisearcher mailaddress scriptblock psvariable type psmoduleinfo powershell runspacefactory runspace initialsessionstate psscriptmethod psscriptproperty psnoteproperty psaliasproperty psvariableproperty - ).join('|') + ) - OPERATORS = %w( + OPERATORS = Set.new %w( -split -isplit -csplit -join -is -isnot -as -eq -ieq -ceq -ne -ine -cne -gt -igt -cgt -ge -ige -cge -lt -ilt -clt -le -ile -cle -like -ilike -clike -notlike -inotlike -cnotlike -match -imatch -cmatch -notmatch -inotmatch -cnotmatch -contains -icontains -ccontains -notcontains -inotcontains -cnotcontains -replace -ireplace -creplace -shl -shr -band - -bor -bxor -and -or -xor -not \+= -= \*= \/= %= - ).join('|') + -bor -bxor -and -or -xor -not + ) - MULTILINE_KEYWORDS = %w( + MULTILINE_KEYWORDS = Set.new %w( synopsis description parameter example inputs outputs notes link component role functionality forwardhelptargetname forwardhelpcategory remotehelprunspace externalhelp - ).join('|') + ) state :variable do - rule %r/#{AUTO_VARS}/, Name::Builtin::Pseudo + rule %r/[$][$?^_]/, Name::Builtin::Pseudo + + keywords %r/\$\w+/ do + rule AUTO_VARS, Name::Builtin::Pseudo + end + rule %r/(\$)(?:(\w+)(:))?(\w+|\{(?:[^`]|`.)+?\})/ do groups Name::Variable, Name::Namespace, Punctuation, Name::Variable end - rule %r/\$\w+/, Name::Variable + rule %r/\$\{(?:[^`]|`.)+?\}/, Name::Variable end state :multiline do - rule %r/\.(?:#{MULTILINE_KEYWORDS})/i, Comment::Special + keywords %r/[.](\w+)/ do + group 1 + rule MULTILINE_KEYWORDS, Comment::Special + end rule %r/#>/, Comment::Multiline, :pop! rule %r/[^#.]+?/m, Comment::Multiline rule %r/[#.]+/, Comment::Multiline @@ -169,9 +176,9 @@ class Powershell < RegexLexer state :bracket do rule %r/\]/, Punctuation, :pop! - rule %r/[A-Za-z]\w+\./, Name + rule %r/[a-z]\w+\./i, Name - keywords %r/([A-Za-z]\w+)/ do + keywords %r/[a-z]\w+/i do rule ATTRIBUTES, Name::Builtin::Pseudo default Name end @@ -204,18 +211,25 @@ class Powershell < RegexLexer rule %r/#requires\s-version \d(?:\.\d+)?/, Comment::Preproc rule %r/\.\.(?=\.?\d)/, Operator - rule %r/(?:#{OPERATORS})\b/i, Operator rule %r/(class)(\s+)(\w+)/i do groups Keyword::Reserved, Text::Whitespace, Name::Class push :class end + rule %r/(function)(\s+)(?:(\w+)(:))?(\w[-\w]+)/i do groups Keyword::Reserved, Text::Whitespace, Name::Namespace, Punctuation, Name::Function end - rule %r/(?:#{KEYWORDS})\b(?![-.])/i, Keyword::Reserved - rule %r/-{1,2}\w+/, Name::Tag + keywords %r/[-.\w]+/ do + transform(&:downcase) + rule KEYWORDS, Keyword::Reserved + end + + keywords %r/-{1,2}\w+/ do + rule OPERATORS, Operator + default Name::Tag + end rule %r/(\.)?([-\w]+)(\[)/ do |m| groups Operator, Name, Punctuation @@ -237,7 +251,8 @@ class Powershell < RegexLexer mixin :expr mixin :variable - rule %r/[-+*\/%=!.&|]/, Operator + rule %r([-+*/%=!.&|]), Operator + rule %r([-+*/%]=), Operator rule %r/[{}(),:;]/, Punctuation rule %r/`$/, Str::Escape # line continuation From dd918a2d6b146801f24389505c841d13b27d5873 Mon Sep 17 00:00:00 2001 From: jneen Date: Sun, 21 Jun 2026 01:21:08 -0400 Subject: [PATCH 127/154] use the keywords api for praat --- lib/rouge/lexers/praat.rb | 195 ++++------------------------- lib/rouge/lexers/praat/builtins.rb | 89 +++++++++++++ 2 files changed, 112 insertions(+), 172 deletions(-) create mode 100644 lib/rouge/lexers/praat/builtins.rb diff --git a/lib/rouge/lexers/praat.rb b/lib/rouge/lexers/praat.rb index 1dc78a760d..89676169ea 100644 --- a/lib/rouge/lexers/praat.rb +++ b/lib/rouge/lexers/praat.rb @@ -15,129 +15,7 @@ def self.detect?(text) return true if text.shebang? 'praat' end - def self.keywords - @keywords ||= %w( - if then else elsif elif endif fi for from to endfor endproc while - endwhile repeat until select plus minus demo assert stopwatch - nocheck nowarn noprogress editor endeditor clearinfo - ) - end - - def self.functions_string - @functions_string ||= %w( - backslashTrigraphsToUnicode$ chooseDirectory$ chooseReadFile$ - chooseWriteFile$ date$ demoKey$ do$ environment$ extractLine$ extractWord$ - fixed$ info$ left$ mid$ percent$ readFile$ replace$ replace_regex$ right$ - selected$ string$ unicodeToBackslashTrigraphs$ - ) - end - - def self.functions_numeric - @functions_numeric ||= %w( - abs appendFile appendFileLine appendInfo appendInfoLine arccos arccosh - arcsin arcsinh arctan arctan2 arctanh barkToHertz beginPause - beginSendPraat besselI besselK beta beta2 binomialP binomialQ boolean - ceiling chiSquareP chiSquareQ choice comment cos cosh createDirectory - deleteFile demoClicked demoClickedIn demoCommandKeyPressed - demoExtraControlKeyPressed demoInput demoKeyPressed - demoOptionKeyPressed demoShiftKeyPressed demoShow demoWaitForInput - demoWindowTitle demoX demoY differenceLimensToPhon do editor endPause - endSendPraat endsWith erb erbToHertz erf erfc exitScript exp - extractNumber fileReadable fisherP fisherQ floor gaussP gaussQ hash - hertzToBark hertzToErb hertzToMel hertzToSemitones imax imin - incompleteBeta incompleteGammaP index index_regex integer invBinomialP - invBinomialQ invChiSquareQ invFisherQ invGaussQ invSigmoid invStudentQ - length ln lnBeta lnGamma log10 log2 max melToHertz min minusObject - natural number numberOfColumns numberOfRows numberOfSelected - objectsAreIdentical option optionMenu pauseScript - phonToDifferenceLimens plusObject positive randomBinomial randomGauss - randomInteger randomPoisson randomUniform real readFile removeObject - rindex rindex_regex round runScript runSystem runSystem_nocheck - selectObject selected semitonesToHertz sentence sentencetext sigmoid - sin sinc sincpi sinh soundPressureToPhon sqrt startsWith studentP - studentQ tan tanh text variableExists word writeFile writeFileLine - writeInfo writeInfoLine - ) - end - - def self.functions_array - @functions_array ||= %w( - linear# randomGauss# randomInteger# randomUniform# zero# - ) - end - - def self.functions_matrix - @functions_matrix ||= %w( - linear## mul## mul_fast## mul_metal## mul_nt## mul_tn## mul_tt## outer## peaks## - randomGamma## randomGauss## randomInteger## randomUniform## softmaxPerRow## - solve## transpose## zero## - ) - end - - def self.functions_string_vector - @functions_string_vector ||= %w( - empty$# fileNames$# folderNames$# readLinesFromFile$# splitByWhitespace$# - ) - end - - def self.functions_builtin - @functions_builtin ||= - self.functions_string | - self.functions_numeric | - self.functions_array | - self.functions_matrix | - self.functions_string_vector - end - - def self.objects - @objects ||= %w( - Activation AffineTransform AmplitudeTier Art Artword Autosegment - BarkFilter BarkSpectrogram CCA Categories Cepstrogram Cepstrum - Cepstrumc ChebyshevSeries ClassificationTable Cochleagram Collection - ComplexSpectrogram Configuration Confusion ContingencyTable Corpus - Correlation Covariance CrossCorrelationTable CrossCorrelationTableList - CrossCorrelationTables DTW DataModeler Diagonalizer Discriminant - Dissimilarity Distance Distributions DurationTier EEG ERP ERPTier - EditCostsTable EditDistanceTable Eigen Excitation Excitations - ExperimentMFC FFNet FeatureWeights FileInMemory FilesInMemory Formant - FormantFilter FormantGrid FormantModeler FormantPoint FormantTier - GaussianMixture HMM HMM_Observation HMM_ObservationSequence HMM_State - HMM_StateSequence HMMObservation HMMObservationSequence HMMState - HMMStateSequence Harmonicity ISpline Index Intensity IntensityTier - IntervalTier KNN KlattGrid KlattTable LFCC LPC Label LegendreSeries - LinearRegression LogisticRegression LongSound Ltas MFCC MSpline ManPages - Manipulation Matrix MelFilter MelSpectrogram MixingMatrix Movie Network - OTGrammar OTHistory OTMulti PCA PairDistribution ParamCurve Pattern - Permutation Photo Pitch PitchModeler PitchTier PointProcess Polygon - Polynomial PowerCepstrogram PowerCepstrum Procrustes RealPoint RealTier - ResultsMFC Roots SPINET SSCP SVD Salience ScalarProduct Similarity - SimpleString SortedSetOfString Sound Speaker Spectrogram Spectrum - SpectrumTier SpeechSynthesizer SpellingChecker Strings StringsIndex - Table TableOfReal TextGrid TextInterval TextPoint TextTier Tier - Transition VocalTract VocalTractTier Weight WordList - ) - end - - def self.variables_numeric - @variables_numeric ||= %w( - all average e left macintosh mono pi praatVersion right stereo - undefined unix windows - ) - end - - def self.variables_string - @variables_string ||= %w( - praatVersion$ tab$ shellDirectory$ homeDirectory$ - preferencesDirectory$ newline$ temporaryDirectory$ - defaultDirectory$ - ) - end - - def self.object_attributes - @object_attributes ||= %w( - ncol nrow xmin ymin xmax ymax nx ny dx dy - ) - end + lazy { require_relative 'praat/builtins' } state :root do rule %r/(\s+)(#.*?$)/ do @@ -184,26 +62,17 @@ def self.object_attributes rule %r/"/, Literal::String, :string - rule %r/\b([A-Z][a-zA-Z0-9]+)(?=\s+\S+\n)/ do |m| - match = m[0] - if self.class.objects.include?(match) - token Name::Class - push :string_unquoted - else - token Keyword - end + keywords %r/\b([A-Z][a-zA-Z0-9]+)(?=\s+\S+\n)/ do + rule OBJECTS, Name::Class, :string_unquoted + default Keyword end rule %r/\b(?=[A-Z])/, Text, :command rule %r/(\.{3}|[)(,\$])/, Punctuation - rule %r/[a_z]+/ do |m| - match = m[0] - if self.class.keywords.include?(match) - token Keyword - else - token Text - end + keywords %r/[a_z]+/ do |m| + rule KEYWORDS, Keyword + default Text end end @@ -250,16 +119,10 @@ def self.object_attributes end state :function_call do - rule %r/\b([a-z][a-zA-Z0-9_.]+)(\$#|##|\$|#)?(?=\s*[:(])/ do |m| - match = m[0] - if self.class.functions_builtin.include?(match) - token Name::Function - push :function - elsif self.class.keywords.include?(match) - token Keyword - else - token Operator::Word - end + keywords %r/\b([a-z][a-zA-Z0-9_.]+)(\$#|##|\$|#)?(?=\s*[:(])/ do + rule FUNCTIONS, Name::Function, :function + rule KEYWORDS, Keyword + default Operator::Word end end @@ -314,26 +177,17 @@ def self.object_attributes mixin :operator mixin :number - rule %r/\b([A-Z][a-zA-Z0-9]+)_/ do |m| - match = m[1] - if (['Object'] | self.class.objects).include?(match) - token Name::Builtin - push :object_reference - else - token Name::Variable - end + keywords %r/\b([A-Z][a-zA-Z0-9]+)_/ do + group 1 + rule Set['Object'], Name::Builtin, :object_reference + rule OBJECTS, Name::Builtin, :object_reference + default Name::Variable end - rule %r/\.?[a-z][a-zA-Z0-9_.]*(\$#|##|\$|#)?/ do |m| - match = m[0] - if self.class.variables_string.include?(match) || - self.class.variables_numeric.include?(match) - token Name::Builtin - elsif self.class.keywords.include?(match) - token Keyword - else - token Name::Variable - end + keywords %r/\.?[a-z][a-zA-Z0-9_.]*(\$#|##|\$|#)?/ do |m| + rule VARIABLES, Name::Builtin + rule KEYWORDS, Keyword + default Name::Variable end rule %r/[\[\]]/, Text, :comma_list @@ -348,12 +202,9 @@ def self.object_attributes mixin :string_interpolated rule %r/([a-z][a-zA-Z0-9_]*|\d+)/, Name::Builtin - rule %r/\.([a-z]+)\b/ do |m| - match = m[1] - if self.class.object_attributes.include?(match) - token Name::Builtin - pop! - end + keywords %r/\.([a-z]+)\b/ do |m| + group 1 + rule ATTRIBUTES, Name::Builtin, :pop! end rule %r/\$/, Name::Builtin diff --git a/lib/rouge/lexers/praat/builtins.rb b/lib/rouge/lexers/praat/builtins.rb new file mode 100644 index 0000000000..0242030aec --- /dev/null +++ b/lib/rouge/lexers/praat/builtins.rb @@ -0,0 +1,89 @@ +module Rouge + module Lexers + class Praat + KEYWORDS = Set.new %w( + if then else elsif elif endif fi for from to endfor endproc while + endwhile repeat until select plus minus demo assert stopwatch + nocheck nowarn noprogress editor endeditor clearinfo + ) + + FUNCTIONS = Set.new %w( + backslashTrigraphsToUnicode$ chooseDirectory$ chooseReadFile$ + chooseWriteFile$ date$ demoKey$ do$ environment$ extractLine$ extractWord$ + fixed$ info$ left$ mid$ percent$ readFile$ replace$ replace_regex$ right$ + selected$ string$ unicodeToBackslashTrigraphs$ + + abs appendFile appendFileLine appendInfo appendInfoLine arccos arccosh + arcsin arcsinh arctan arctan2 arctanh barkToHertz beginPause + beginSendPraat besselI besselK beta beta2 binomialP binomialQ boolean + ceiling chiSquareP chiSquareQ choice comment cos cosh createDirectory + deleteFile demoClicked demoClickedIn demoCommandKeyPressed + demoExtraControlKeyPressed demoInput demoKeyPressed + demoOptionKeyPressed demoShiftKeyPressed demoShow demoWaitForInput + demoWindowTitle demoX demoY differenceLimensToPhon do editor endPause + endSendPraat endsWith erb erbToHertz erf erfc exitScript exp + extractNumber fileReadable fisherP fisherQ floor gaussP gaussQ hash + hertzToBark hertzToErb hertzToMel hertzToSemitones imax imin + incompleteBeta incompleteGammaP index index_regex integer invBinomialP + invBinomialQ invChiSquareQ invFisherQ invGaussQ invSigmoid invStudentQ + length ln lnBeta lnGamma log10 log2 max melToHertz min minusObject + natural number numberOfColumns numberOfRows numberOfSelected + objectsAreIdentical option optionMenu pauseScript + phonToDifferenceLimens plusObject positive randomBinomial randomGauss + randomInteger randomPoisson randomUniform real readFile removeObject + rindex rindex_regex round runScript runSystem runSystem_nocheck + selectObject selected semitonesToHertz sentence sentencetext sigmoid + sin sinc sincpi sinh soundPressureToPhon sqrt startsWith studentP + studentQ tan tanh text variableExists word writeFile writeFileLine + writeInfo writeInfoLine + + linear# randomGauss# randomInteger# randomUniform# zero# + + linear## mul## mul_fast## mul_metal## mul_nt## mul_tn## mul_tt## outer## peaks## + randomGamma## randomGauss## randomInteger## randomUniform## softmaxPerRow## + solve## transpose## zero## + + empty$# fileNames$# folderNames$# readLinesFromFile$# splitByWhitespace$# + ) + + OBJECTS = Set.new %w( + Activation AffineTransform AmplitudeTier Art Artword Autosegment + BarkFilter BarkSpectrogram CCA Categories Cepstrogram Cepstrum + Cepstrumc ChebyshevSeries ClassificationTable Cochleagram Collection + ComplexSpectrogram Configuration Confusion ContingencyTable Corpus + Correlation Covariance CrossCorrelationTable CrossCorrelationTableList + CrossCorrelationTables DTW DataModeler Diagonalizer Discriminant + Dissimilarity Distance Distributions DurationTier EEG ERP ERPTier + EditCostsTable EditDistanceTable Eigen Excitation Excitations + ExperimentMFC FFNet FeatureWeights FileInMemory FilesInMemory Formant + FormantFilter FormantGrid FormantModeler FormantPoint FormantTier + GaussianMixture HMM HMM_Observation HMM_ObservationSequence HMM_State + HMM_StateSequence HMMObservation HMMObservationSequence HMMState + HMMStateSequence Harmonicity ISpline Index Intensity IntensityTier + IntervalTier KNN KlattGrid KlattTable LFCC LPC Label LegendreSeries + LinearRegression LogisticRegression LongSound Ltas MFCC MSpline ManPages + Manipulation Matrix MelFilter MelSpectrogram MixingMatrix Movie Network + OTGrammar OTHistory OTMulti PCA PairDistribution ParamCurve Pattern + Permutation Photo Pitch PitchModeler PitchTier PointProcess Polygon + Polynomial PowerCepstrogram PowerCepstrum Procrustes RealPoint RealTier + ResultsMFC Roots SPINET SSCP SVD Salience ScalarProduct Similarity + SimpleString SortedSetOfString Sound Speaker Spectrogram Spectrum + SpectrumTier SpeechSynthesizer SpellingChecker Strings StringsIndex + Table TableOfReal TextGrid TextInterval TextPoint TextTier Tier + Transition VocalTract VocalTractTier Weight WordList + ) + + VARIABLES = Set.new %w( + all average e left macintosh mono pi praatVersion right stereo + undefined unix windows + + praatVersion$ tab$ shellDirectory$ homeDirectory$ + preferencesDirectory$ newline$ temporaryDirectory$ + defaultDirectory$ + ) + + ATTRIBUTES = Set.new %w(ncol nrow xmin ymin xmax ymax nx ny dx dy) + + end + end +end From d66b7559d1725cd0d77a79b5e7c2c8f313859623 Mon Sep 17 00:00:00 2001 From: jneen Date: Mon, 22 Jun 2026 11:54:50 -0400 Subject: [PATCH 128/154] use the keywords api for prometheus --- lib/rouge/lexers/prometheus.rb | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/lib/rouge/lexers/prometheus.rb b/lib/rouge/lexers/prometheus.rb index c27f7bccfd..ebb5ce3dd3 100644 --- a/lib/rouge/lexers/prometheus.rb +++ b/lib/rouge/lexers/prometheus.rb @@ -10,17 +10,15 @@ class Prometheus < RegexLexer mimetypes 'text/x-prometheus', 'application/x-prometheus' - def self.functions - @functions ||= Set.new %w( - abs absent ceil changes clamp_max clamp_min count_scalar day_of_month - day_of_week days_in_month delta deriv drop_common_labels exp floor - histogram_quantile holt_winters hour idelta increase irate label_replace - ln log2 log10 month predict_linear rate resets round scalar sort - sort_desc sqrt time vector year avg_over_time min_over_time - max_over_time sum_over_time count_over_time quantile_over_time - stddev_over_time stdvar_over_time - ) - end + FUNCTIONS = Set.new %w( + abs absent ceil changes clamp_max clamp_min count_scalar day_of_month + day_of_week days_in_month delta deriv drop_common_labels exp floor + histogram_quantile holt_winters hour idelta increase irate label_replace + ln log2 log10 month predict_linear rate resets round scalar sort + sort_desc sqrt time vector year avg_over_time min_over_time + max_over_time sum_over_time count_over_time quantile_over_time + stddev_over_time stdvar_over_time + ) state :root do mixin :strings @@ -40,12 +38,10 @@ def self.functions end rule %r/(bool|offset)\b/, Keyword rule %r/(without|by)\b/, Keyword, :label_list - rule %r/[\w:]+/ do |m| - if self.class.functions.include?(m[0]) - token Name::Builtin - else - token Name - end + + keywords %r/[\w:]+/ do + rule FUNCTIONS, Name::Builtin + default Name end mixin :metrics From bf2466f60b8483c6bea684abfed2c0854a5673a5 Mon Sep 17 00:00:00 2001 From: jneen Date: Mon, 22 Jun 2026 11:58:43 -0400 Subject: [PATCH 129/154] use the keywords api for puppet --- lib/rouge/lexers/puppet.rb | 45 +++++++++++--------------------------- 1 file changed, 13 insertions(+), 32 deletions(-) diff --git a/lib/rouge/lexers/puppet.rb b/lib/rouge/lexers/puppet.rb index 9726bd9ecf..fa9d9001b5 100644 --- a/lib/rouge/lexers/puppet.rb +++ b/lib/rouge/lexers/puppet.rb @@ -15,24 +15,14 @@ def self.detect?(text) return true if text.shebang? 'puppet' end - def self.keywords - @keywords ||= Set.new %w( - and case class default define else elsif if in import inherits - node unless - ) - end + KEYWORDS = Set.new %w( + and case class default define else elsif if in import inherits + node unless + ) - def self.constants - @constants ||= Set.new %w( - false true undef - ) - end + CONSTANTS = Set.new %w(false true undef) - def self.metaparameters - @metaparameters ||= Set.new %w( - before require notify subscribe - ) - end + METAPARAMETERS = Set.new %w(before require notify subscribe) id = /[a-z]\w*/ cap_id = /[A-Z]\w*/ @@ -47,12 +37,9 @@ def self.metaparameters mixin :whitespace rule %r/[$]#{qualname}/, Name::Variable - rule %r/(#{id})(?=\s*[=+]>)/m do |m| - if self.class.metaparameters.include? m[0] - token Keyword::Pseudo - else - token Name::Property - end + keywords %r/(#{id})(?=\s*[=+]>)/m do + rule METAPARAMETERS, Keyword::Pseudo + default Name::Property end rule %r/(#{qualname})(?=\s*[(])/m, Name::Function @@ -81,16 +68,10 @@ def self.metaparameters rule %r/\d+([.]\d+)?(e[+-]\d+)?/, Num - # a valid regex. TODO: regexes are only allowed - # in certain places in puppet. - rule qualname do |m| - if self.class.keywords.include? m[0] - token Keyword - elsif self.class.constants.include? m[0] - token Keyword::Constant - else - token Name - end + keywords qualname do + rule KEYWORDS, Keyword + rule CONSTANTS, Keyword::Constant + default Name end end From 5c83b68bd97142fe3a171165c72a5a44c230aabb Mon Sep 17 00:00:00 2001 From: jneen Date: Mon, 22 Jun 2026 12:02:48 -0400 Subject: [PATCH 130/154] use the keywords api for Q --- lib/rouge/lexers/q.rb | 64 +++++++++++++++++++++---------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/lib/rouge/lexers/q.rb b/lib/rouge/lexers/q.rb index b0d49c5b2f..b66355506b 100644 --- a/lib/rouge/lexers/q.rb +++ b/lib/rouge/lexers/q.rb @@ -12,27 +12,27 @@ class Q < RegexLexer identifier = /\.?[a-z][a-z0-9_.]*/i - def self.keywords - @keywords ||= %w[do if while select update delete exec from by] - end - - def self.word_operators - @word_operators ||= %w[ - and or except inter like each cross vs sv within where in asof bin binr cor cov cut ej fby - div ij insert lj ljf mavg mcount mdev mmax mmin mmu mod msum over prior peach pj scan scov setenv ss - sublist uj union upsert wavg wsum xasc xbar xcol xcols xdesc xexp xgroup xkey xlog xprev xrank - ] - end - - def self.builtins - @builtins ||= %w[ - first enlist value type get set count string key max min sum prd last flip distinct raze neg - desc differ dsave dev eval exit exp fills fkeys floor getenv group gtime hclose hcount hdel hopen hsym - iasc idesc inv keys load log lsq ltime ltrim maxs md5 med meta mins next parse plist prds prev rand rank ratios - read0 read1 reciprocal reverse rload rotate rsave rtrim save sdev show signum sin sqrt ssr sums svar system - tables tan til trim txf ungroup var view views wj wj1 ww - ] - end + KEYWORDS = Set.new %w[do if while select update delete exec from by] + + WORD_OPERATORS = Set.new %w[ + and or except inter like each cross vs sv within where in + asof bin binr cor cov cut ej fby div ij insert lj ljf mavg + mcount mdev mmax mmin mmu mod msum over prior peach pj scan + scov setenv ss sublist uj union upsert wavg wsum xasc xbar + xcol xcols xdesc xexp xgroup xkey xlog xprev xrank + ] + + BUILTINS = Set.new %w[ + first enlist value type get set count string key max min + sum prd last flip distinct raze neg desc differ dsave dev + eval exit exp fills fkeys floor getenv group gtime hclose + hcount hdel hopen hsym iasc idesc inv keys load log lsq + ltime ltrim maxs md5 med meta mins next parse plist prds + prev rand rank ratios read0 read1 reciprocal reverse rload + rotate rsave rtrim save sdev show signum sin sqrt ssr sums + svar system tables tan til trim txf ungroup var view views + wj wj1 ww + ] state :root do # q allows a file to start with a shebang @@ -52,17 +52,17 @@ def self.builtins rule(/^.\)/, Keyword::Declaration) # Identifiers, word operators, etc. - rule %r/#{identifier}/ do |m| - if self.class.keywords.include? m[0] - token Keyword - elsif self.class.word_operators.include? m[0] - token Operator::Word - elsif self.class.builtins.include? m[0] - token Name::Builtin - elsif /^\.[zQqho]\./ =~ m[0] - token Name::Constant - else - token Name + keywords identifier do + rule KEYWORDS, Keyword + rule WORD_OPERATORS, Operator::Word + rule BUILTINS, Name::Builtin + + default do |m| + if /^\.[zQqho]\./ =~ m[0] + token Name::Constant + else + token Name + end end end From 709e16a3edae14c3dcfe73ee243e6b441a2cefbd Mon Sep 17 00:00:00 2001 From: jneen Date: Mon, 22 Jun 2026 12:07:50 -0400 Subject: [PATCH 131/154] use the keywords api for R --- .rubocop_todo.yml | 1 - lib/rouge/lexers/r.rb | 27 ++++++++++++--------------- 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index 54e3b72cba..00f300a3d3 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -24,7 +24,6 @@ Naming/VariableName: # Offense count: 113 Rouge/NoBuildingAlternationPatternInRegexp: Exclude: - - 'lib/rouge/lexers/r.rb' - 'lib/rouge/lexers/rust.rb' - 'lib/rouge/lexers/scala.rb' - 'lib/rouge/lexers/sml.rb' diff --git a/lib/rouge/lexers/r.rb b/lib/rouge/lexers/r.rb index dc500e64a4..0c91ca06fe 100644 --- a/lib/rouge/lexers/r.rb +++ b/lib/rouge/lexers/r.rb @@ -13,18 +13,18 @@ class R < RegexLexer mimetypes 'text/x-r', 'application/x-r' - KEYWORDS = %w(if else for while repeat in next break function) + KEYWORDS = Set.new %w(if else for while repeat in next break function) - KEYWORD_CONSTANTS = %w( + KEYWORD_CONSTANTS = Set.new %w( NULL Inf TRUE FALSE NaN NA NA_integer_ NA_real_ NA_complex_ NA_character_ ) - BUILTIN_CONSTANTS = %w(LETTERS letters month.abb month.name pi T F) + BUILTIN_CONSTANTS = Set.new %w(LETTERS letters month.abb month.name pi T F) # These are all the functions in `base` that are implemented as a # `.Primitive`, minus those functions that are also keywords. - PRIMITIVE_FUNCTIONS = %w( + PRIMITIVE_FUNCTIONS = Set.new %w( abs acos acosh all any anyNA Arg as.call as.character as.complex as.double as.environment as.integer as.logical as.null.default as.numeric as.raw asin asinh atan atanh attr @@ -66,18 +66,15 @@ def self.detect?(text) # function call, i.e. followed by an opening parenthesis. # `Name::Builtin` would be more logical, but is usually not # highlighted specifically; thus use `Name::Function`. - rule %r/\b(? Date: Mon, 22 Jun 2026 12:12:45 -0400 Subject: [PATCH 132/154] use the keywords api for racket --- lib/rouge/lexers/racket.rb | 480 +--------------------------- lib/rouge/lexers/racket/builtins.rb | 465 +++++++++++++++++++++++++++ 2 files changed, 471 insertions(+), 474 deletions(-) create mode 100644 lib/rouge/lexers/racket/builtins.rb diff --git a/lib/rouge/lexers/racket.rb b/lib/rouge/lexers/racket.rb index fb67ab4bde..20712c39c9 100644 --- a/lib/rouge/lexers/racket.rb +++ b/lib/rouge/lexers/racket.rb @@ -11,6 +11,8 @@ class Racket < RegexLexer filenames '*.rkt', '*.rktd', '*.rktl' mimetypes 'text/x-racket', 'application/x-racket' + lazy { require_relative 'racket/builtins' } + def self.detect?(text) text =~ /\A#lang\s*(.*?)$/ lang_attr = $1 @@ -18,469 +20,6 @@ def self.detect?(text) return true if lang_attr =~ /racket|scribble/ end - def self.keywords - @keywords ||= Set.new %w( - ... and begin begin-for-syntax begin0 case case-lambda cond - datum->syntax-object define define-for-syntax define-logger - define-struct define-syntax define-syntax-rule - define-syntaxes define-values define-values-for-syntax delay - do expand-path fluid-let force hash-table-copy - hash-table-count hash-table-for-each hash-table-get - hash-table-iterate-first hash-table-iterate-key - hash-table-iterate-next hash-table-iterate-value - hash-table-map hash-table-put! hash-table-remove! - hash-table? if lambda let let* let*-values let-struct - let-syntax let-syntaxes let-values let/cc let/ec letrec - letrec-syntax letrec-syntaxes letrec-syntaxes+values - letrec-values list-immutable make-hash-table - make-immutable-hash-table make-namespace module module* - module-identifier=? module-label-identifier=? - module-template-identifier=? module-transformer-identifier=? - namespace-transformer-require or parameterize parameterize* - parameterize-break promise? prop:method-arity-error provide - provide-for-label provide-for-syntax quasiquote quasisyntax - quasisyntax/loc quote quote-syntax quote-syntax/prune - require require-for-label require-for-syntax - require-for-template set! set!-values syntax syntax-case - syntax-case* syntax-id-rules syntax-object->datum - syntax-rules syntax/loc tcp-abandon-port tcp-accept - tcp-accept-evt tcp-accept-ready? tcp-accept/enable-break - tcp-addresses tcp-close tcp-connect tcp-connect/enable-break - tcp-listen tcp-listener? tcp-port? time transcript-off - transcript-on udp-addresses udp-bind! udp-bound? udp-close - udp-connect! udp-connected? udp-multicast-interface - udp-multicast-join-group! udp-multicast-leave-group! - udp-multicast-loopback? udp-multicast-set-interface! - udp-multicast-set-loopback! udp-multicast-set-ttl! - udp-multicast-ttl udp-open-socket udp-receive! udp-receive!* - udp-receive!-evt udp-receive!/enable-break - udp-receive-ready-evt udp-send udp-send* udp-send-evt - udp-send-ready-evt udp-send-to udp-send-to* udp-send-to-evt - udp-send-to/enable-break udp-send/enable-break udp? unless - unquote unquote-splicing unsyntax unsyntax-splicing when - with-continuation-mark with-handlers with-handlers* - with-syntax λ) - end - - def self.builtins - @builtins ||= Set.new %w( - * + - / < <= = > >= - abort-current-continuation abs absolute-path? acos add1 - alarm-evt always-evt andmap angle append apply - arithmetic-shift arity-at-least arity-at-least-value - arity-at-least? asin assoc assq assv atan banner bitwise-and - bitwise-bit-field bitwise-bit-set? bitwise-ior bitwise-not - bitwise-xor boolean? bound-identifier=? box box-cas! - box-immutable box? break-enabled break-thread build-path - build-path/convention-type byte-pregexp byte-pregexp? - byte-ready? byte-regexp byte-regexp? byte? bytes - bytes->immutable-bytes bytes->list bytes->path - bytes->path-element bytes->string/latin-1 - bytes->string/locale bytes->string/utf-8 bytes-append - bytes-close-converter bytes-convert bytes-convert-end - bytes-converter? bytes-copy bytes-copy! - bytes-environment-variable-name? bytes-fill! bytes-length - bytes-open-converter bytes-ref bytes-set! bytes-utf-8-index - bytes-utf-8-length bytes-utf-8-ref bytes? - bytes? caaaar caaadr caaar caadar caaddr caadr caar cadaar - cadadr cadar caddar cadddr caddr cadr call-in-nested-thread - call-with-break-parameterization - call-with-composable-continuation - call-with-continuation-barrier call-with-continuation-prompt - call-with-current-continuation - call-with-default-reading-parameterization - call-with-escape-continuation call-with-exception-handler - call-with-immediate-continuation-mark call-with-input-file - call-with-output-file call-with-parameterization - call-with-semaphore call-with-semaphore/enable-break - call-with-values call/cc call/ec car cdaaar cdaadr cdaar - cdadar cdaddr cdadr cdar cddaar cddadr cddar cdddar cddddr - cdddr cddr cdr ceiling channel-get channel-put - channel-put-evt channel-put-evt? channel-try-get channel? - chaperone-box chaperone-continuation-mark-key chaperone-evt - chaperone-hash chaperone-of? chaperone-procedure - chaperone-prompt-tag chaperone-struct chaperone-struct-type - chaperone-vector chaperone? char->integer char-alphabetic? - char-blank? char-ci<=? char-ci=? - char-ci>? char-downcase char-foldcase char-general-category - char-graphic? char-iso-control? char-lower-case? - char-numeric? char-punctuation? char-ready? char-symbolic? - char-title-case? char-titlecase char-upcase char-upper-case? - char-utf-8-length char-whitespace? char<=? char=? char>? char? check-duplicate-identifier - checked-procedure-check-and-extract choice-evt cleanse-path - close-input-port close-output-port collect-garbage - collection-file-path collection-path compile - compile-allow-set!-undefined - compile-context-preservation-enabled - compile-enforce-module-constants compile-syntax - compiled-expression? compiled-module-expression? - complete-path? complex? cons continuation-mark-key? - continuation-mark-set->context continuation-mark-set->list - continuation-mark-set->list* continuation-mark-set-first - continuation-mark-set? continuation-marks - continuation-prompt-available? continuation-prompt-tag? - continuation? copy-file cos current-break-parameterization - current-code-inspector current-command-line-arguments - current-compile current-compiled-file-roots - current-continuation-marks current-custodian - current-directory current-directory-for-user current-drive - current-environment-variables current-error-port - current-eval current-evt-pseudo-random-generator - current-gc-milliseconds current-get-interaction-input-port - current-inexact-milliseconds current-input-port - current-inspector current-library-collection-paths - current-load current-load-extension - current-load-relative-directory current-load/use-compiled - current-locale current-memory-use current-milliseconds - current-module-declare-name current-module-declare-source - current-module-name-resolver current-module-path-for-load - current-namespace current-output-port - current-parameterization - current-preserved-thread-cell-values current-print - current-process-milliseconds current-prompt-read - current-pseudo-random-generator current-read-interaction - current-reader-guard current-readtable current-seconds - current-security-guard current-subprocess-custodian-mode - current-thread current-thread-group - current-thread-initial-stack-size - current-write-relative-directory custodian-box-value - custodian-box? custodian-limit-memory custodian-managed-list - custodian-memory-accounting-available? - custodian-require-memory custodian-shutdown-all custodian? - custom-print-quotable-accessor custom-print-quotable? - custom-write-accessor custom-write? date date* - date*-nanosecond date*-time-zone-name date*? date-day - date-dst? date-hour date-minute date-month date-second - date-time-zone-offset date-week-day date-year date-year-day - date? datum-intern-literal default-continuation-prompt-tag - delete-directory delete-file denominator directory-exists? - directory-list display displayln dump-memory-stats - dynamic-require dynamic-require-for-syntax dynamic-wind - environment-variables-copy environment-variables-names - environment-variables-ref environment-variables-set! - environment-variables? eof eof-object? ephemeron-value - ephemeron? eprintf eq-hash-code eq? equal-hash-code - equal-secondary-hash-code equal? equal?/recur eqv-hash-code - eqv? error error-display-handler error-escape-handler - error-print-context-length error-print-source-location - error-print-width error-value->string-handler eval - eval-jit-enabled eval-syntax even? evt? exact->inexact - exact-integer? exact-nonnegative-integer? - exact-positive-integer? exact? executable-yield-handler exit - exit-handler exn exn-continuation-marks exn-message - exn:break exn:break-continuation exn:break:hang-up - exn:break:hang-up? exn:break:terminate exn:break:terminate? - exn:break? exn:fail exn:fail:contract - exn:fail:contract:arity exn:fail:contract:arity? - exn:fail:contract:continuation - exn:fail:contract:continuation? - exn:fail:contract:divide-by-zero - exn:fail:contract:divide-by-zero? - exn:fail:contract:non-fixnum-result - exn:fail:contract:non-fixnum-result? - exn:fail:contract:variable exn:fail:contract:variable-id - exn:fail:contract:variable? exn:fail:contract? - exn:fail:filesystem exn:fail:filesystem:errno - exn:fail:filesystem:errno-errno exn:fail:filesystem:errno? - exn:fail:filesystem:exists exn:fail:filesystem:exists? - exn:fail:filesystem:missing-module - exn:fail:filesystem:missing-module-path - exn:fail:filesystem:missing-module? - exn:fail:filesystem:version exn:fail:filesystem:version? - exn:fail:filesystem? exn:fail:network exn:fail:network:errno - exn:fail:network:errno-errno exn:fail:network:errno? - exn:fail:network? exn:fail:out-of-memory - exn:fail:out-of-memory? exn:fail:read exn:fail:read-srclocs - exn:fail:read:eof exn:fail:read:eof? exn:fail:read:non-char - exn:fail:read:non-char? exn:fail:read? exn:fail:syntax - exn:fail:syntax-exprs exn:fail:syntax:missing-module - exn:fail:syntax:missing-module-path - exn:fail:syntax:missing-module? exn:fail:syntax:unbound - exn:fail:syntax:unbound? exn:fail:syntax? - exn:fail:unsupported exn:fail:unsupported? exn:fail:user - exn:fail:user? exn:fail? exn:missing-module-accessor - exn:missing-module? exn:srclocs-accessor exn:srclocs? exn? - exp expand expand-once expand-syntax expand-syntax-once - expand-syntax-to-top-form expand-to-top-form - expand-user-path explode-path expt file-exists? - file-or-directory-identity file-or-directory-modify-seconds - file-or-directory-permissions file-position file-position* - file-size file-stream-buffer-mode file-stream-port? - file-truncate filesystem-change-evt - filesystem-change-evt-cancel filesystem-change-evt? - filesystem-root-list find-executable-path - find-library-collection-paths find-system-path fixnum? - floating-point-bytes->real flonum? floor flush-output - for-each format fprintf free-identifier=? gcd - generate-temporaries gensym get-output-bytes - get-output-string getenv global-port-print-handler guard-evt - handle-evt handle-evt? hash hash-equal? hash-eqv? - hash-has-key? hash-placeholder? hash-ref! hasheq hasheqv - identifier-binding identifier-binding-symbol - identifier-label-binding identifier-prune-lexical-context - identifier-prune-to-source-module - identifier-remove-from-definition-context - identifier-template-binding identifier-transformer-binding - identifier? imag-part immutable? impersonate-box - impersonate-continuation-mark-key impersonate-hash - impersonate-procedure impersonate-prompt-tag - impersonate-struct impersonate-vector impersonator-ephemeron - impersonator-of? impersonator-prop:application-mark - impersonator-property-accessor-procedure? - impersonator-property? impersonator? inexact->exact - inexact-real? inexact? input-port? inspector? integer->char - integer->integer-bytes integer-bytes->integer integer-length - integer-sqrt integer-sqrt/remainder integer? - internal-definition-context-seal - internal-definition-context? keyword->string keywordbytes list->string - list->vector list-ref list-tail list? load load-extension - load-on-demand-enabled load-relative load-relative-extension - load/cd load/use-compiled local-expand - local-expand/capture-lifts local-transformer-expand - local-transformer-expand/capture-lifts - locale-string-encoding log log-max-level magnitude - make-arity-at-least make-bytes make-channel - make-continuation-mark-key make-continuation-prompt-tag - make-custodian make-custodian-box make-date make-date* - make-derived-parameter make-directory - make-environment-variables make-ephemeron make-exn - make-exn:break make-exn:break:hang-up - make-exn:break:terminate make-exn:fail - make-exn:fail:contract make-exn:fail:contract:arity - make-exn:fail:contract:continuation - make-exn:fail:contract:divide-by-zero - make-exn:fail:contract:non-fixnum-result - make-exn:fail:contract:variable make-exn:fail:filesystem - make-exn:fail:filesystem:errno - make-exn:fail:filesystem:exists - make-exn:fail:filesystem:missing-module - make-exn:fail:filesystem:version make-exn:fail:network - make-exn:fail:network:errno make-exn:fail:out-of-memory - make-exn:fail:read make-exn:fail:read:eof - make-exn:fail:read:non-char make-exn:fail:syntax - make-exn:fail:syntax:missing-module - make-exn:fail:syntax:unbound make-exn:fail:unsupported - make-exn:fail:user make-file-or-directory-link - make-hash-placeholder make-hasheq-placeholder make-hasheqv - make-hasheqv-placeholder make-immutable-hasheqv - make-impersonator-property make-input-port make-inspector - make-known-char-range-list make-output-port make-parameter - make-phantom-bytes make-pipe make-placeholder make-polar - make-prefab-struct make-pseudo-random-generator - make-reader-graph make-readtable make-rectangular - make-rename-transformer make-resolved-module-path - make-security-guard make-semaphore make-set!-transformer - make-shared-bytes make-sibling-inspector - make-special-comment make-srcloc make-string - make-struct-field-accessor make-struct-field-mutator - make-struct-type make-struct-type-property - make-syntax-delta-introducer make-syntax-introducer - make-thread-cell make-thread-group make-vector make-weak-box - make-weak-hasheqv make-will-executor map max mcar mcdr mcons - member memq memv min module->exports module->imports - module->language-info module->namespace - module-compiled-cross-phase-persistent? - module-compiled-exports module-compiled-imports - module-compiled-language-info module-compiled-name - module-compiled-submodules module-declared? - module-path-index-join module-path-index-resolve - module-path-index-split module-path-index-submodule - module-path-index? module-path? module-predefined? - module-provide-protected? modulo mpair? nack-guard-evt - namespace-attach-module namespace-attach-module-declaration - namespace-base-phase namespace-mapped-symbols - namespace-module-identifier namespace-module-registry - namespace-require namespace-require/constant - namespace-require/copy namespace-require/expansion-time - namespace-set-variable-value! namespace-symbol->identifier - namespace-syntax-introduce namespace-undefine-variable! - namespace-unprotect-module namespace-variable-value - namespace? negative? never-evt newline normal-case-path not - null null? number->string number? numerator object-name odd? - open-input-bytes open-input-file open-input-output-file - open-input-string open-output-bytes open-output-file - open-output-string ormap output-port? pair? - parameter-procedure=? parameter? parameterization? - path->bytes path->complete-path path->directory-path - path->string path-add-suffix path-convention-type - path-element->bytes path-element->string - path-for-some-system? path-list-string->path-list - path-replace-suffix path-string? path? peek-byte - peek-byte-or-special peek-bytes peek-bytes! - peek-bytes-avail! peek-bytes-avail!* - peek-bytes-avail!/enable-break peek-char - peek-char-or-special peek-string peek-string! phantom-bytes? - pipe-content-length placeholder-get placeholder-set! - placeholder? poll-guard-evt port-closed-evt port-closed? - port-commit-peeked port-count-lines! - port-count-lines-enabled port-counts-lines? - port-display-handler port-file-identity port-file-unlock - port-next-location port-print-handler port-progress-evt - port-provides-progress-evts? port-read-handler - port-try-file-lock? port-write-handler port-writes-atomic? - port-writes-special? port? positive? prefab-key->struct-type - prefab-key? prefab-struct-key pregexp pregexp? - primitive-closure? primitive-result-arity primitive? print - print-as-expression print-boolean-long-form print-box - print-graph print-hash-table print-mpair-curly-braces - print-pair-curly-braces print-reader-abbreviations - print-struct print-syntax-width print-unreadable - print-vector-length printf procedure->method procedure-arity - procedure-arity-includes? procedure-arity? - procedure-closure-contents-eq? procedure-extract-target - procedure-reduce-arity procedure-rename - procedure-struct-type? procedure? progress-evt? - prop:arity-string prop:checked-procedure - prop:custom-print-quotable prop:custom-write prop:equal+hash - prop:evt prop:exn:missing-module prop:exn:srclocs - prop:impersonator-of prop:input-port - prop:liberal-define-context prop:output-port prop:procedure - prop:rename-transformer prop:set!-transformer - pseudo-random-generator->vector - pseudo-random-generator-vector? pseudo-random-generator? - putenv quotient quotient/remainder raise - raise-argument-error raise-arguments-error raise-arity-error - raise-mismatch-error raise-range-error raise-result-error - raise-syntax-error raise-type-error raise-user-error random - random-seed rational? rationalize read read-accept-bar-quote - read-accept-box read-accept-compiled read-accept-dot - read-accept-graph read-accept-infix-dot read-accept-lang - read-accept-quasiquote read-accept-reader read-byte - read-byte-or-special read-bytes read-bytes! - read-bytes-avail! read-bytes-avail!* - read-bytes-avail!/enable-break read-bytes-line - read-case-sensitive read-char read-char-or-special - read-curly-brace-as-paren read-decimal-as-inexact - read-eval-print-loop read-language read-line - read-on-demand-source read-square-bracket-as-paren - read-string read-string! read-syntax read-syntax/recursive - read/recursive readtable-mapping readtable? - real->double-flonum real->floating-point-bytes - real->single-flonum real-part real? regexp regexp-match - regexp-match-peek regexp-match-peek-immediate - regexp-match-peek-positions - regexp-match-peek-positions-immediate - regexp-match-peek-positions-immediate/end - regexp-match-peek-positions/end regexp-match-positions - regexp-match-positions/end regexp-match/end regexp-match? - regexp-max-lookbehind regexp-replace regexp-replace* regexp? - relative-path? remainder rename-file-or-directory - rename-transformer-target rename-transformer? reroot-path - resolve-path resolved-module-path-name resolved-module-path? - reverse round seconds->date security-guard? - semaphore-peek-evt semaphore-peek-evt? semaphore-post - semaphore-try-wait? semaphore-wait - semaphore-wait/enable-break semaphore? - set!-transformer-procedure set!-transformer? set-box! - set-mcar! set-mcdr! set-phantom-bytes! - set-port-next-location! shared-bytes shell-execute - simplify-path sin single-flonum? sleep special-comment-value - special-comment? split-path sqrt srcloc srcloc->string - srcloc-column srcloc-line srcloc-position srcloc-source - srcloc-span srcloc? string string->bytes/latin-1 - string->bytes/locale string->bytes/utf-8 - string->immutable-string string->keyword string->list - string->number string->path string->path-element - string->symbol string->uninterned-symbol - string->unreadable-symbol string-append string-ci<=? - string-ci=? string-ci>? string-copy - string-copy! string-downcase - string-environment-variable-name? string-fill! - string-foldcase string-length string-locale-ci? string-locale-downcase - string-locale-upcase string-locale? string-normalize-nfc string-normalize-nfd - string-normalize-nfkc string-normalize-nfkd string-ref - string-set! string-titlecase string-upcase - string-utf-8-length string<=? string=? - string>? string? struct->vector struct-accessor-procedure? - struct-constructor-procedure? struct-info - struct-mutator-procedure? struct-predicate-procedure? - struct-type-info struct-type-make-constructor - struct-type-make-predicate - struct-type-property-accessor-procedure? - struct-type-property? struct-type? struct:arity-at-least - struct:date struct:date* struct:exn struct:exn:break - struct:exn:break:hang-up struct:exn:break:terminate - struct:exn:fail struct:exn:fail:contract - struct:exn:fail:contract:arity - struct:exn:fail:contract:continuation - struct:exn:fail:contract:divide-by-zero - struct:exn:fail:contract:non-fixnum-result - struct:exn:fail:contract:variable struct:exn:fail:filesystem - struct:exn:fail:filesystem:errno - struct:exn:fail:filesystem:exists - struct:exn:fail:filesystem:missing-module - struct:exn:fail:filesystem:version struct:exn:fail:network - struct:exn:fail:network:errno struct:exn:fail:out-of-memory - struct:exn:fail:read struct:exn:fail:read:eof - struct:exn:fail:read:non-char struct:exn:fail:syntax - struct:exn:fail:syntax:missing-module - struct:exn:fail:syntax:unbound struct:exn:fail:unsupported - struct:exn:fail:user struct:srcloc struct? sub1 subbytes - subprocess subprocess-group-enabled subprocess-kill - subprocess-pid subprocess-status subprocess-wait subprocess? - substring symbol->string symbol-interned? symbol-unreadable? - symbol? sync sync/enable-break sync/timeout - sync/timeout/enable-break syntax->list syntax-arm - syntax-column syntax-disarm syntax-e syntax-line - syntax-local-bind-syntaxes syntax-local-certifier - syntax-local-context syntax-local-expand-expression - syntax-local-get-shadower syntax-local-introduce - syntax-local-lift-context syntax-local-lift-expression - syntax-local-lift-module-end-declaration - syntax-local-lift-provide syntax-local-lift-require - syntax-local-lift-values-expression - syntax-local-make-definition-context - syntax-local-make-delta-introducer - syntax-local-module-defined-identifiers - syntax-local-module-exports - syntax-local-module-required-identifiers syntax-local-name - syntax-local-phase-level syntax-local-submodules - syntax-local-transforming-module-provides? - syntax-local-value syntax-local-value/immediate - syntax-original? syntax-position syntax-property - syntax-property-symbol-keys syntax-protect syntax-rearm - syntax-recertify syntax-shift-phase-level syntax-source - syntax-source-module syntax-span syntax-taint - syntax-tainted? syntax-track-origin - syntax-transforming-module-expression? syntax-transforming? - syntax? system-big-endian? system-idle-evt - system-language+country system-library-subpath - system-path-convention-type system-type tan terminal-port? - thread thread-cell-ref thread-cell-set! thread-cell-values? - thread-cell? thread-dead-evt thread-dead? thread-group? - thread-resume thread-resume-evt thread-rewind-receive - thread-running? thread-suspend thread-suspend-evt - thread-wait thread/suspend-to-kill thread? time-apply - truncate unbox uncaught-exception-handler - use-collection-link-paths use-compiled-file-paths - use-user-specific-search-paths values - variable-reference->empty-namespace - variable-reference->module-base-phase - variable-reference->module-declaration-inspector - variable-reference->module-path-index - variable-reference->module-source - variable-reference->namespace variable-reference->phase - variable-reference->resolved-module-path - variable-reference-constant? variable-reference? vector - vector->immutable-vector vector->list - vector->pseudo-random-generator - vector->pseudo-random-generator! vector->values vector-fill! - vector-immutable vector-length vector-ref vector-set! - vector-set-performance-stats! vector? version void void? - weak-box-value weak-box? will-execute will-executor? - will-register will-try-execute with-input-from-file - with-output-to-file wrap-evt write write-byte write-bytes - write-bytes-avail write-bytes-avail* write-bytes-avail-evt - write-bytes-avail/enable-break write-char write-special - write-special-avail* write-special-evt write-string zero? - ) - end - # Since Racket allows identifiers to consist of nearly anything, # it's simpler to describe what an ID is _not_. id = /[^\s\(\)\[\]\{\}'`,.]+/i @@ -548,21 +87,14 @@ def self.builtins end state :command do - rule id do |m| - if self.class.keywords.include? m[0] - token Keyword - elsif self.class.builtins.include? m[0] - token Name::Builtin - else - token Name::Function - end - - pop! + keywords id do + rule KEYWORDS, Keyword, :pop! + rule BUILTINS, Name::Builtin, :pop! + default Name::Function, :pop! end rule(//) { pop! } end - end end end diff --git a/lib/rouge/lexers/racket/builtins.rb b/lib/rouge/lexers/racket/builtins.rb new file mode 100644 index 0000000000..06acf3fdb2 --- /dev/null +++ b/lib/rouge/lexers/racket/builtins.rb @@ -0,0 +1,465 @@ +module Rouge + module Lexers + class Racket + KEYWORDS = Set.new %w( + ... and begin begin-for-syntax begin0 case case-lambda cond + datum->syntax-object define define-for-syntax define-logger + define-struct define-syntax define-syntax-rule + define-syntaxes define-values define-values-for-syntax delay + do expand-path fluid-let force hash-table-copy + hash-table-count hash-table-for-each hash-table-get + hash-table-iterate-first hash-table-iterate-key + hash-table-iterate-next hash-table-iterate-value + hash-table-map hash-table-put! hash-table-remove! + hash-table? if lambda let let* let*-values let-struct + let-syntax let-syntaxes let-values let/cc let/ec letrec + letrec-syntax letrec-syntaxes letrec-syntaxes+values + letrec-values list-immutable make-hash-table + make-immutable-hash-table make-namespace module module* + module-identifier=? module-label-identifier=? + module-template-identifier=? module-transformer-identifier=? + namespace-transformer-require or parameterize parameterize* + parameterize-break promise? prop:method-arity-error provide + provide-for-label provide-for-syntax quasiquote quasisyntax + quasisyntax/loc quote quote-syntax quote-syntax/prune + require require-for-label require-for-syntax + require-for-template set! set!-values syntax syntax-case + syntax-case* syntax-id-rules syntax-object->datum + syntax-rules syntax/loc tcp-abandon-port tcp-accept + tcp-accept-evt tcp-accept-ready? tcp-accept/enable-break + tcp-addresses tcp-close tcp-connect tcp-connect/enable-break + tcp-listen tcp-listener? tcp-port? time transcript-off + transcript-on udp-addresses udp-bind! udp-bound? udp-close + udp-connect! udp-connected? udp-multicast-interface + udp-multicast-join-group! udp-multicast-leave-group! + udp-multicast-loopback? udp-multicast-set-interface! + udp-multicast-set-loopback! udp-multicast-set-ttl! + udp-multicast-ttl udp-open-socket udp-receive! udp-receive!* + udp-receive!-evt udp-receive!/enable-break + udp-receive-ready-evt udp-send udp-send* udp-send-evt + udp-send-ready-evt udp-send-to udp-send-to* udp-send-to-evt + udp-send-to/enable-break udp-send/enable-break udp? unless + unquote unquote-splicing unsyntax unsyntax-splicing when + with-continuation-mark with-handlers with-handlers* + with-syntax λ + ) + + BUILTINS = Set.new %w( + * + - / < <= = > >= + abort-current-continuation abs absolute-path? acos add1 + alarm-evt always-evt andmap angle append apply + arithmetic-shift arity-at-least arity-at-least-value + arity-at-least? asin assoc assq assv atan banner bitwise-and + bitwise-bit-field bitwise-bit-set? bitwise-ior bitwise-not + bitwise-xor boolean? bound-identifier=? box box-cas! + box-immutable box? break-enabled break-thread build-path + build-path/convention-type byte-pregexp byte-pregexp? + byte-ready? byte-regexp byte-regexp? byte? bytes + bytes->immutable-bytes bytes->list bytes->path + bytes->path-element bytes->string/latin-1 + bytes->string/locale bytes->string/utf-8 bytes-append + bytes-close-converter bytes-convert bytes-convert-end + bytes-converter? bytes-copy bytes-copy! + bytes-environment-variable-name? bytes-fill! bytes-length + bytes-open-converter bytes-ref bytes-set! bytes-utf-8-index + bytes-utf-8-length bytes-utf-8-ref bytes? + bytes? caaaar caaadr caaar caadar caaddr caadr caar cadaar + cadadr cadar caddar cadddr caddr cadr call-in-nested-thread + call-with-break-parameterization + call-with-composable-continuation + call-with-continuation-barrier call-with-continuation-prompt + call-with-current-continuation + call-with-default-reading-parameterization + call-with-escape-continuation call-with-exception-handler + call-with-immediate-continuation-mark call-with-input-file + call-with-output-file call-with-parameterization + call-with-semaphore call-with-semaphore/enable-break + call-with-values call/cc call/ec car cdaaar cdaadr cdaar + cdadar cdaddr cdadr cdar cddaar cddadr cddar cdddar cddddr + cdddr cddr cdr ceiling channel-get channel-put + channel-put-evt channel-put-evt? channel-try-get channel? + chaperone-box chaperone-continuation-mark-key chaperone-evt + chaperone-hash chaperone-of? chaperone-procedure + chaperone-prompt-tag chaperone-struct chaperone-struct-type + chaperone-vector chaperone? char->integer char-alphabetic? + char-blank? char-ci<=? char-ci=? + char-ci>? char-downcase char-foldcase char-general-category + char-graphic? char-iso-control? char-lower-case? + char-numeric? char-punctuation? char-ready? char-symbolic? + char-title-case? char-titlecase char-upcase char-upper-case? + char-utf-8-length char-whitespace? char<=? char=? char>? char? check-duplicate-identifier + checked-procedure-check-and-extract choice-evt cleanse-path + close-input-port close-output-port collect-garbage + collection-file-path collection-path compile + compile-allow-set!-undefined + compile-context-preservation-enabled + compile-enforce-module-constants compile-syntax + compiled-expression? compiled-module-expression? + complete-path? complex? cons continuation-mark-key? + continuation-mark-set->context continuation-mark-set->list + continuation-mark-set->list* continuation-mark-set-first + continuation-mark-set? continuation-marks + continuation-prompt-available? continuation-prompt-tag? + continuation? copy-file cos current-break-parameterization + current-code-inspector current-command-line-arguments + current-compile current-compiled-file-roots + current-continuation-marks current-custodian + current-directory current-directory-for-user current-drive + current-environment-variables current-error-port + current-eval current-evt-pseudo-random-generator + current-gc-milliseconds current-get-interaction-input-port + current-inexact-milliseconds current-input-port + current-inspector current-library-collection-paths + current-load current-load-extension + current-load-relative-directory current-load/use-compiled + current-locale current-memory-use current-milliseconds + current-module-declare-name current-module-declare-source + current-module-name-resolver current-module-path-for-load + current-namespace current-output-port + current-parameterization + current-preserved-thread-cell-values current-print + current-process-milliseconds current-prompt-read + current-pseudo-random-generator current-read-interaction + current-reader-guard current-readtable current-seconds + current-security-guard current-subprocess-custodian-mode + current-thread current-thread-group + current-thread-initial-stack-size + current-write-relative-directory custodian-box-value + custodian-box? custodian-limit-memory custodian-managed-list + custodian-memory-accounting-available? + custodian-require-memory custodian-shutdown-all custodian? + custom-print-quotable-accessor custom-print-quotable? + custom-write-accessor custom-write? date date* + date*-nanosecond date*-time-zone-name date*? date-day + date-dst? date-hour date-minute date-month date-second + date-time-zone-offset date-week-day date-year date-year-day + date? datum-intern-literal default-continuation-prompt-tag + delete-directory delete-file denominator directory-exists? + directory-list display displayln dump-memory-stats + dynamic-require dynamic-require-for-syntax dynamic-wind + environment-variables-copy environment-variables-names + environment-variables-ref environment-variables-set! + environment-variables? eof eof-object? ephemeron-value + ephemeron? eprintf eq-hash-code eq? equal-hash-code + equal-secondary-hash-code equal? equal?/recur eqv-hash-code + eqv? error error-display-handler error-escape-handler + error-print-context-length error-print-source-location + error-print-width error-value->string-handler eval + eval-jit-enabled eval-syntax even? evt? exact->inexact + exact-integer? exact-nonnegative-integer? + exact-positive-integer? exact? executable-yield-handler exit + exit-handler exn exn-continuation-marks exn-message + exn:break exn:break-continuation exn:break:hang-up + exn:break:hang-up? exn:break:terminate exn:break:terminate? + exn:break? exn:fail exn:fail:contract + exn:fail:contract:arity exn:fail:contract:arity? + exn:fail:contract:continuation + exn:fail:contract:continuation? + exn:fail:contract:divide-by-zero + exn:fail:contract:divide-by-zero? + exn:fail:contract:non-fixnum-result + exn:fail:contract:non-fixnum-result? + exn:fail:contract:variable exn:fail:contract:variable-id + exn:fail:contract:variable? exn:fail:contract? + exn:fail:filesystem exn:fail:filesystem:errno + exn:fail:filesystem:errno-errno exn:fail:filesystem:errno? + exn:fail:filesystem:exists exn:fail:filesystem:exists? + exn:fail:filesystem:missing-module + exn:fail:filesystem:missing-module-path + exn:fail:filesystem:missing-module? + exn:fail:filesystem:version exn:fail:filesystem:version? + exn:fail:filesystem? exn:fail:network exn:fail:network:errno + exn:fail:network:errno-errno exn:fail:network:errno? + exn:fail:network? exn:fail:out-of-memory + exn:fail:out-of-memory? exn:fail:read exn:fail:read-srclocs + exn:fail:read:eof exn:fail:read:eof? exn:fail:read:non-char + exn:fail:read:non-char? exn:fail:read? exn:fail:syntax + exn:fail:syntax-exprs exn:fail:syntax:missing-module + exn:fail:syntax:missing-module-path + exn:fail:syntax:missing-module? exn:fail:syntax:unbound + exn:fail:syntax:unbound? exn:fail:syntax? + exn:fail:unsupported exn:fail:unsupported? exn:fail:user + exn:fail:user? exn:fail? exn:missing-module-accessor + exn:missing-module? exn:srclocs-accessor exn:srclocs? exn? + exp expand expand-once expand-syntax expand-syntax-once + expand-syntax-to-top-form expand-to-top-form + expand-user-path explode-path expt file-exists? + file-or-directory-identity file-or-directory-modify-seconds + file-or-directory-permissions file-position file-position* + file-size file-stream-buffer-mode file-stream-port? + file-truncate filesystem-change-evt + filesystem-change-evt-cancel filesystem-change-evt? + filesystem-root-list find-executable-path + find-library-collection-paths find-system-path fixnum? + floating-point-bytes->real flonum? floor flush-output + for-each format fprintf free-identifier=? gcd + generate-temporaries gensym get-output-bytes + get-output-string getenv global-port-print-handler guard-evt + handle-evt handle-evt? hash hash-equal? hash-eqv? + hash-has-key? hash-placeholder? hash-ref! hasheq hasheqv + identifier-binding identifier-binding-symbol + identifier-label-binding identifier-prune-lexical-context + identifier-prune-to-source-module + identifier-remove-from-definition-context + identifier-template-binding identifier-transformer-binding + identifier? imag-part immutable? impersonate-box + impersonate-continuation-mark-key impersonate-hash + impersonate-procedure impersonate-prompt-tag + impersonate-struct impersonate-vector impersonator-ephemeron + impersonator-of? impersonator-prop:application-mark + impersonator-property-accessor-procedure? + impersonator-property? impersonator? inexact->exact + inexact-real? inexact? input-port? inspector? integer->char + integer->integer-bytes integer-bytes->integer integer-length + integer-sqrt integer-sqrt/remainder integer? + internal-definition-context-seal + internal-definition-context? keyword->string keywordbytes list->string + list->vector list-ref list-tail list? load load-extension + load-on-demand-enabled load-relative load-relative-extension + load/cd load/use-compiled local-expand + local-expand/capture-lifts local-transformer-expand + local-transformer-expand/capture-lifts + locale-string-encoding log log-max-level magnitude + make-arity-at-least make-bytes make-channel + make-continuation-mark-key make-continuation-prompt-tag + make-custodian make-custodian-box make-date make-date* + make-derived-parameter make-directory + make-environment-variables make-ephemeron make-exn + make-exn:break make-exn:break:hang-up + make-exn:break:terminate make-exn:fail + make-exn:fail:contract make-exn:fail:contract:arity + make-exn:fail:contract:continuation + make-exn:fail:contract:divide-by-zero + make-exn:fail:contract:non-fixnum-result + make-exn:fail:contract:variable make-exn:fail:filesystem + make-exn:fail:filesystem:errno + make-exn:fail:filesystem:exists + make-exn:fail:filesystem:missing-module + make-exn:fail:filesystem:version make-exn:fail:network + make-exn:fail:network:errno make-exn:fail:out-of-memory + make-exn:fail:read make-exn:fail:read:eof + make-exn:fail:read:non-char make-exn:fail:syntax + make-exn:fail:syntax:missing-module + make-exn:fail:syntax:unbound make-exn:fail:unsupported + make-exn:fail:user make-file-or-directory-link + make-hash-placeholder make-hasheq-placeholder make-hasheqv + make-hasheqv-placeholder make-immutable-hasheqv + make-impersonator-property make-input-port make-inspector + make-known-char-range-list make-output-port make-parameter + make-phantom-bytes make-pipe make-placeholder make-polar + make-prefab-struct make-pseudo-random-generator + make-reader-graph make-readtable make-rectangular + make-rename-transformer make-resolved-module-path + make-security-guard make-semaphore make-set!-transformer + make-shared-bytes make-sibling-inspector + make-special-comment make-srcloc make-string + make-struct-field-accessor make-struct-field-mutator + make-struct-type make-struct-type-property + make-syntax-delta-introducer make-syntax-introducer + make-thread-cell make-thread-group make-vector make-weak-box + make-weak-hasheqv make-will-executor map max mcar mcdr mcons + member memq memv min module->exports module->imports + module->language-info module->namespace + module-compiled-cross-phase-persistent? + module-compiled-exports module-compiled-imports + module-compiled-language-info module-compiled-name + module-compiled-submodules module-declared? + module-path-index-join module-path-index-resolve + module-path-index-split module-path-index-submodule + module-path-index? module-path? module-predefined? + module-provide-protected? modulo mpair? nack-guard-evt + namespace-attach-module namespace-attach-module-declaration + namespace-base-phase namespace-mapped-symbols + namespace-module-identifier namespace-module-registry + namespace-require namespace-require/constant + namespace-require/copy namespace-require/expansion-time + namespace-set-variable-value! namespace-symbol->identifier + namespace-syntax-introduce namespace-undefine-variable! + namespace-unprotect-module namespace-variable-value + namespace? negative? never-evt newline normal-case-path not + null null? number->string number? numerator object-name odd? + open-input-bytes open-input-file open-input-output-file + open-input-string open-output-bytes open-output-file + open-output-string ormap output-port? pair? + parameter-procedure=? parameter? parameterization? + path->bytes path->complete-path path->directory-path + path->string path-add-suffix path-convention-type + path-element->bytes path-element->string + path-for-some-system? path-list-string->path-list + path-replace-suffix path-string? path? peek-byte + peek-byte-or-special peek-bytes peek-bytes! + peek-bytes-avail! peek-bytes-avail!* + peek-bytes-avail!/enable-break peek-char + peek-char-or-special peek-string peek-string! phantom-bytes? + pipe-content-length placeholder-get placeholder-set! + placeholder? poll-guard-evt port-closed-evt port-closed? + port-commit-peeked port-count-lines! + port-count-lines-enabled port-counts-lines? + port-display-handler port-file-identity port-file-unlock + port-next-location port-print-handler port-progress-evt + port-provides-progress-evts? port-read-handler + port-try-file-lock? port-write-handler port-writes-atomic? + port-writes-special? port? positive? prefab-key->struct-type + prefab-key? prefab-struct-key pregexp pregexp? + primitive-closure? primitive-result-arity primitive? print + print-as-expression print-boolean-long-form print-box + print-graph print-hash-table print-mpair-curly-braces + print-pair-curly-braces print-reader-abbreviations + print-struct print-syntax-width print-unreadable + print-vector-length printf procedure->method procedure-arity + procedure-arity-includes? procedure-arity? + procedure-closure-contents-eq? procedure-extract-target + procedure-reduce-arity procedure-rename + procedure-struct-type? procedure? progress-evt? + prop:arity-string prop:checked-procedure + prop:custom-print-quotable prop:custom-write prop:equal+hash + prop:evt prop:exn:missing-module prop:exn:srclocs + prop:impersonator-of prop:input-port + prop:liberal-define-context prop:output-port prop:procedure + prop:rename-transformer prop:set!-transformer + pseudo-random-generator->vector + pseudo-random-generator-vector? pseudo-random-generator? + putenv quotient quotient/remainder raise + raise-argument-error raise-arguments-error raise-arity-error + raise-mismatch-error raise-range-error raise-result-error + raise-syntax-error raise-type-error raise-user-error random + random-seed rational? rationalize read read-accept-bar-quote + read-accept-box read-accept-compiled read-accept-dot + read-accept-graph read-accept-infix-dot read-accept-lang + read-accept-quasiquote read-accept-reader read-byte + read-byte-or-special read-bytes read-bytes! + read-bytes-avail! read-bytes-avail!* + read-bytes-avail!/enable-break read-bytes-line + read-case-sensitive read-char read-char-or-special + read-curly-brace-as-paren read-decimal-as-inexact + read-eval-print-loop read-language read-line + read-on-demand-source read-square-bracket-as-paren + read-string read-string! read-syntax read-syntax/recursive + read/recursive readtable-mapping readtable? + real->double-flonum real->floating-point-bytes + real->single-flonum real-part real? regexp regexp-match + regexp-match-peek regexp-match-peek-immediate + regexp-match-peek-positions + regexp-match-peek-positions-immediate + regexp-match-peek-positions-immediate/end + regexp-match-peek-positions/end regexp-match-positions + regexp-match-positions/end regexp-match/end regexp-match? + regexp-max-lookbehind regexp-replace regexp-replace* regexp? + relative-path? remainder rename-file-or-directory + rename-transformer-target rename-transformer? reroot-path + resolve-path resolved-module-path-name resolved-module-path? + reverse round seconds->date security-guard? + semaphore-peek-evt semaphore-peek-evt? semaphore-post + semaphore-try-wait? semaphore-wait + semaphore-wait/enable-break semaphore? + set!-transformer-procedure set!-transformer? set-box! + set-mcar! set-mcdr! set-phantom-bytes! + set-port-next-location! shared-bytes shell-execute + simplify-path sin single-flonum? sleep special-comment-value + special-comment? split-path sqrt srcloc srcloc->string + srcloc-column srcloc-line srcloc-position srcloc-source + srcloc-span srcloc? string string->bytes/latin-1 + string->bytes/locale string->bytes/utf-8 + string->immutable-string string->keyword string->list + string->number string->path string->path-element + string->symbol string->uninterned-symbol + string->unreadable-symbol string-append string-ci<=? + string-ci=? string-ci>? string-copy + string-copy! string-downcase + string-environment-variable-name? string-fill! + string-foldcase string-length string-locale-ci? string-locale-downcase + string-locale-upcase string-locale? string-normalize-nfc string-normalize-nfd + string-normalize-nfkc string-normalize-nfkd string-ref + string-set! string-titlecase string-upcase + string-utf-8-length string<=? string=? + string>? string? struct->vector struct-accessor-procedure? + struct-constructor-procedure? struct-info + struct-mutator-procedure? struct-predicate-procedure? + struct-type-info struct-type-make-constructor + struct-type-make-predicate + struct-type-property-accessor-procedure? + struct-type-property? struct-type? struct:arity-at-least + struct:date struct:date* struct:exn struct:exn:break + struct:exn:break:hang-up struct:exn:break:terminate + struct:exn:fail struct:exn:fail:contract + struct:exn:fail:contract:arity + struct:exn:fail:contract:continuation + struct:exn:fail:contract:divide-by-zero + struct:exn:fail:contract:non-fixnum-result + struct:exn:fail:contract:variable struct:exn:fail:filesystem + struct:exn:fail:filesystem:errno + struct:exn:fail:filesystem:exists + struct:exn:fail:filesystem:missing-module + struct:exn:fail:filesystem:version struct:exn:fail:network + struct:exn:fail:network:errno struct:exn:fail:out-of-memory + struct:exn:fail:read struct:exn:fail:read:eof + struct:exn:fail:read:non-char struct:exn:fail:syntax + struct:exn:fail:syntax:missing-module + struct:exn:fail:syntax:unbound struct:exn:fail:unsupported + struct:exn:fail:user struct:srcloc struct? sub1 subbytes + subprocess subprocess-group-enabled subprocess-kill + subprocess-pid subprocess-status subprocess-wait subprocess? + substring symbol->string symbol-interned? symbol-unreadable? + symbol? sync sync/enable-break sync/timeout + sync/timeout/enable-break syntax->list syntax-arm + syntax-column syntax-disarm syntax-e syntax-line + syntax-local-bind-syntaxes syntax-local-certifier + syntax-local-context syntax-local-expand-expression + syntax-local-get-shadower syntax-local-introduce + syntax-local-lift-context syntax-local-lift-expression + syntax-local-lift-module-end-declaration + syntax-local-lift-provide syntax-local-lift-require + syntax-local-lift-values-expression + syntax-local-make-definition-context + syntax-local-make-delta-introducer + syntax-local-module-defined-identifiers + syntax-local-module-exports + syntax-local-module-required-identifiers syntax-local-name + syntax-local-phase-level syntax-local-submodules + syntax-local-transforming-module-provides? + syntax-local-value syntax-local-value/immediate + syntax-original? syntax-position syntax-property + syntax-property-symbol-keys syntax-protect syntax-rearm + syntax-recertify syntax-shift-phase-level syntax-source + syntax-source-module syntax-span syntax-taint + syntax-tainted? syntax-track-origin + syntax-transforming-module-expression? syntax-transforming? + syntax? system-big-endian? system-idle-evt + system-language+country system-library-subpath + system-path-convention-type system-type tan terminal-port? + thread thread-cell-ref thread-cell-set! thread-cell-values? + thread-cell? thread-dead-evt thread-dead? thread-group? + thread-resume thread-resume-evt thread-rewind-receive + thread-running? thread-suspend thread-suspend-evt + thread-wait thread/suspend-to-kill thread? time-apply + truncate unbox uncaught-exception-handler + use-collection-link-paths use-compiled-file-paths + use-user-specific-search-paths values + variable-reference->empty-namespace + variable-reference->module-base-phase + variable-reference->module-declaration-inspector + variable-reference->module-path-index + variable-reference->module-source + variable-reference->namespace variable-reference->phase + variable-reference->resolved-module-path + variable-reference-constant? variable-reference? vector + vector->immutable-vector vector->list + vector->pseudo-random-generator + vector->pseudo-random-generator! vector->values vector-fill! + vector-immutable vector-length vector-ref vector-set! + vector-set-performance-stats! vector? version void void? + weak-box-value weak-box? will-execute will-executor? + will-register will-try-execute with-input-from-file + with-output-to-file wrap-evt write write-byte write-bytes + write-bytes-avail write-bytes-avail* write-bytes-avail-evt + write-bytes-avail/enable-break write-char write-special + write-special-avail* write-special-evt write-string zero? + ) + end + end +end From 3503dd1fbe760dd3e2c39bfcd971c8814d5f1509 Mon Sep 17 00:00:00 2001 From: jneen Date: Mon, 22 Jun 2026 12:20:08 -0400 Subject: [PATCH 133/154] use the keywords api for Reason and OCaml --- lib/rouge/lexers/ocaml.rb | 10 +++++----- lib/rouge/lexers/ocaml/common.rb | 14 +++++++------- lib/rouge/lexers/reasonml.rb | 25 ++++++++----------------- 3 files changed, 20 insertions(+), 29 deletions(-) diff --git a/lib/rouge/lexers/ocaml.rb b/lib/rouge/lexers/ocaml.rb index ab74cea783..0d02aa83b1 100644 --- a/lib/rouge/lexers/ocaml.rb +++ b/lib/rouge/lexers/ocaml.rb @@ -21,15 +21,15 @@ def self.keywords state :root do rule %r/\s+/m, Text rule %r/false|true|[(][)]|\[\]/, Name::Builtin::Pseudo - rule %r/#{@@upper_id}(?=\s*[.])/, Name::Namespace, :dotted - rule %r/`#{@@id}/, Name::Tag - rule @@upper_id, Name::Class + rule %r/#{OCAML_UPPER_ID}(?=\s*[.])/, Name::Namespace, :dotted + rule %r/`#{OCAML_ID}/, Name::Tag + rule OCAML_UPPER_ID, Name::Class rule %r/[(][*](?![)])/, Comment, :comment mixin :keywords_and_names rule %r/[(){}\[\];]+/, Punctuation - rule @@operator, Operator + rule OCAML_OPERATOR, Operator rule %r/-?\d[\d_]*(.[\d_]*)?(e[+-]?\d[\d_]*)/i, Num::Float rule %r/0x\h[\h_]*/i, Num::Hex @@ -41,7 +41,7 @@ def self.keywords rule %r/'[.]'/, Str::Char rule %r/'/, Keyword rule %r/"/, Str::Double, :string - rule %r/[~?]#{@@id}/, Name::Variable + rule %r/[~?]#{OCAML_ID}/, Name::Variable end state :comment do diff --git a/lib/rouge/lexers/ocaml/common.rb b/lib/rouge/lexers/ocaml/common.rb index f17884c46c..728c215438 100644 --- a/lib/rouge/lexers/ocaml/common.rb +++ b/lib/rouge/lexers/ocaml/common.rb @@ -23,9 +23,9 @@ def self.primitives @primitives ||= Set.new %w(unit int float bool string char list array) end - @@operator = %r([;,_!$%&*+./:<=>?@^|~#-]+) - @@id = /[a-z_][\w']*/i - @@upper_id = /[A-Z][\w']*/ + OCAML_OPERATOR = %r([;,_!$%&*+./:<=>?@^|~#-]+) + OCAML_ID = /[a-z_][\w']*/i + OCAML_UPPER_ID = /[A-Z][\w']*/ state :string do rule %r/[^\\"]+/, Str::Double @@ -43,14 +43,14 @@ def self.primitives state :dotted do rule %r/\s+/m, Text rule %r/[.]/, Punctuation - rule %r/#{@@upper_id}(?=\s*[.])/, Name::Namespace - rule @@upper_id, Name::Class, :pop! - rule @@id, Name, :pop! + rule %r/#{OCAML_UPPER_ID}(?=\s*[.])/, Name::Namespace + rule OCAML_UPPER_ID, Name::Class, :pop! + rule OCAML_ID, Name, :pop! rule %r/[({\[]/, Punctuation, :pop! end state :keywords_and_names do - keywords @@id do + keywords OCAML_ID do rule :keywords, Keyword rule :word_operators, Operator::Word rule :primitives, Keyword::Type diff --git a/lib/rouge/lexers/reasonml.rb b/lib/rouge/lexers/reasonml.rb index 6938f845cd..df8964f71a 100644 --- a/lib/rouge/lexers/reasonml.rb +++ b/lib/rouge/lexers/reasonml.rb @@ -9,6 +9,7 @@ class ReasonML < OCamlCommon title "ReasonML" desc 'New syntax on top of OCaml ecosystem (reasonml.github.io)' tag 'reasonml' + aliases 'reason' filenames '*.re', '*.rei' mimetypes 'text/x-reasonml' @@ -21,26 +22,16 @@ def self.keywords state :root do rule %r/\s+/m, Text rule %r/false|true|[(][)]|\[\]/, Name::Builtin::Pseudo - rule %r/#{@@upper_id}(?=\s*[.])/, Name::Namespace, :dotted - rule %r/`#{@@id}/, Name::Tag - rule @@upper_id, Name::Class + rule %r/#{OCAML_UPPER_ID}(?=\s*[.])/, Name::Namespace, :dotted + rule %r/`#{OCAML_ID}/, Name::Tag + rule OCAML_UPPER_ID, Name::Class rule %r(//.*), Comment::Single rule %r(/\*), Comment::Multiline, :comment - rule @@id do |m| - match = m[0] - if self.class.keywords.include? match - token Keyword - elsif self.class.word_operators.include? match - token Operator::Word - elsif self.class.primitives.include? match - token Keyword::Type - else - token Name - end - end + + mixin :keywords_and_names rule %r/[(){}\[\];]+/, Punctuation - rule @@operator, Operator + rule OCAML_OPERATOR, Operator rule %r/-?\d[\d_]*(.[\d_]*)?(e[+-]?\d[\d_]*)/i, Num::Float rule %r/0x\h[\h_]*/i, Num::Hex @@ -52,7 +43,7 @@ def self.keywords rule %r/'[^'\/]'/, Str::Char rule %r/'/, Keyword rule %r/"/, Str::Double, :string - rule %r/[~?]#{@@id}/, Name::Variable + rule %r/[~?]#{OCAML_ID}/, Name::Variable end state :comment do From de4cd447a48a9200d12e9aad21939e953f7388a5 Mon Sep 17 00:00:00 2001 From: jneen Date: Mon, 22 Jun 2026 12:21:52 -0400 Subject: [PATCH 134/154] use the keywords api for rego --- lib/rouge/lexers/rego.rb | 27 ++++++++------------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/lib/rouge/lexers/rego.rb b/lib/rouge/lexers/rego.rb index 9eb217410a..3325268087 100644 --- a/lib/rouge/lexers/rego.rb +++ b/lib/rouge/lexers/rego.rb @@ -9,17 +9,10 @@ class Rego < RegexLexer tag 'rego' filenames '*.rego' - def self.constants - @constants ||= Set.new %w( - true false null - ) - end - - def self.operators - @operators ||= Set.new %w( - as default else import not package some with - ) - end + CONSTANTS = Set.new %w(true false null) + OPERATORS = Set.new %w( + as default else import not package some with + ) state :basic do rule %r/\s+/, Text @@ -45,14 +38,10 @@ def self.operators mixin :basic mixin :operators - rule %r/[[:word:]]+/ do |m| - if self.class.constants.include? m[0] - token Keyword::Constant - elsif self.class.operators.include? m[0] - token Operator::Word - else - token Name - end + keywords %r/[[:word:]]+/ do + rule CONSTANTS, Keyword::Constant + rule OPERATORS, Operator::Word + default Name end end end From 498cb729520210891ea40d72bebaee3a77c8e89a Mon Sep 17 00:00:00 2001 From: jneen Date: Mon, 22 Jun 2026 12:23:36 -0400 Subject: [PATCH 135/154] use the new OCamlCommon naming for ReScript --- lib/rouge/lexers/rescript.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/rouge/lexers/rescript.rb b/lib/rouge/lexers/rescript.rb index f6df8bb496..2b3a8e22c6 100644 --- a/lib/rouge/lexers/rescript.rb +++ b/lib/rouge/lexers/rescript.rb @@ -39,16 +39,16 @@ def self.word_operators rule %r/\btrue|false\b/, Keyword::Constant # Module chain - rule %r/#{@@upper_id}(?=\s*[.])/, Name::Namespace, :dotted + rule %r/#{OCAML_UPPER_ID}(?=\s*[.])/, Name::Namespace, :dotted # Decorator - rule %r/@#{@@id}(\.#{@@id})*/, Name::Decorator + rule %r/@#{OCAML_ID}(\.#{OCAML_ID})*/, Name::Decorator # Poly variant - rule %r/\##{@@id}/, Name::Class + rule %r/\##{OCAML_ID}/, Name::Class # Variant or Module - rule @@upper_id, Name::Class + rule OCAML_UPPER_ID, Name::Class # Comments rule %r(//.*), Comment::Single From 58e6b52a20f2a8bed42a53c47e73462644095f51 Mon Sep 17 00:00:00 2001 From: jneen Date: Mon, 22 Jun 2026 12:24:27 -0400 Subject: [PATCH 136/154] remove racket from the todo file --- .rubocop_todo.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index 00f300a3d3..b9e990610b 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -35,6 +35,5 @@ Rouge/NoBuildingAlternationPatternInRegexp: # Offense count: 20 Rouge/NoHugeCollections: Exclude: - - 'lib/rouge/lexers/racket.rb' - 'lib/rouge/lexers/sas.rb' - 'lib/rouge/lexers/stata.rb' From 408dff78c4629741aa44921ddcb62f63645a4561 Mon Sep 17 00:00:00 2001 From: jneen Date: Mon, 22 Jun 2026 16:45:27 -0400 Subject: [PATCH 137/154] use the keywords api for RML, and remove overhighlighting --- lib/rouge/lexers/rml.rb | 85 +++++++++++++++++------------------------ 1 file changed, 36 insertions(+), 49 deletions(-) diff --git a/lib/rouge/lexers/rml.rb b/lib/rouge/lexers/rml.rb index 0891470ee7..ac13c43577 100644 --- a/lib/rouge/lexers/rml.rb +++ b/lib/rouge/lexers/rml.rb @@ -9,18 +9,14 @@ class RML < RegexLexer tag 'rml' filenames '*.rml' - def self.keywords - @keywords ||= Set.new %w( - matches not with empty - all if else true false - ) - end + KEYWORDS = Set.new %w( + matches not with empty + all if else true false + ) - def self.arithmetic_keywords - @arithmetic_keywords ||= Set.new %w( - abs sin cos tan min max - ) - end + ARITHMETIC_KEYWORDS = Set.new %w( + abs sin cos tan min max + ) id_char = /[a-zA-Z0-9_]/ uppercase_id = /[A-Z]#{id_char}*/ @@ -35,11 +31,11 @@ def self.arithmetic_keywords comment = /\/\/[^\r\n]*/ state :common_rules do - rule %r/#{whitespace}/, Text - rule %r/#{comment}/, Comment::Single - rule %r/#{string}/, Literal::String - rule %r/#{float}/, Num::Float - rule %r/#{int}/, Num::Integer + rule whitespace, Text + rule comment, Comment::Single + rule string, Literal::String + rule float, Num::Float + rule int, Num::Integer end state :root do @@ -48,19 +44,16 @@ def self.arithmetic_keywords groups Name::Function, Operator push :event_type_params end - rule %r/#{lowercase_id}/ do |m| - if m[0] == 'with' - token Keyword - push :data_expression_with - elsif self.class.keywords.include? m[0] - token Keyword - else - token Name::Function - end + + keywords lowercase_id do + rule Set['with'], Keyword, :data_expression_with + rule KEYWORDS, Keyword + default Name end + rule %r/\(|\{|\[/, Operator, :event_type_params rule %r/[_\|]/, Operator - rule %r/#{uppercase_id}/, Name::Class, :equation_block_expression + rule uppercase_id, Name::Class, :equation_block_expression rule %r/;/, Operator end @@ -68,22 +61,19 @@ def self.arithmetic_keywords mixin :common_rules rule %r/\(|\{|\[/, Operator, :push rule %r/\)|\}|\]/, Operator, :pop! - rule %r/#{lowercase_id}(?=:)/, Name::Entity - rule %r/(#{lowercase_id})/ do |m| - if self.class.keywords.include? m[0] - token Keyword - else - token Literal::String::Regex - end + rule %r/#{lowercase_id}(?=:)/, Name::Attribute + keywords lowercase_id do + rule KEYWORDS, Keyword + default Name end - rule %r/#{ellipsis}/, Literal::String::Symbol + rule ellipsis, Str::Symbol rule %r/[_\|;,:]/, Operator end state :equation_block_expression do mixin :common_rules rule %r/[<,>]/, Operator - rule %r/#{lowercase_id}/, Literal::String::Regex + rule lowercase_id, Str::Regex rule %r/=/ do token Operator goto :exp @@ -102,28 +92,26 @@ def self.arithmetic_keywords groups Name::Function, Operator push :event_type_params end - rule %r/(#{lowercase_id})/ do |m| - if self.class.keywords.include? m[0] - token Keyword - else - token Name::Function - end + + keywords lowercase_id do + rule KEYWORDS, Keyword + default Name end + rule %r/#{uppercase_id}(?=<)/, Name::Class, :data_expression - rule %r/#{uppercase_id}/, Name::Class + rule uppercase_id, Name::Class rule %r/[=(){}*+\/\\\|!>?]/, Operator rule %r/;/, Operator, :pop! end state :data_expression do mixin :common_rules - rule %r/#{lowercase_id}/ do |m| - if (self.class.arithmetic_keywords | self.class.keywords).include? m[0] - token Keyword - else - token Literal::String::Regex - end + keywords lowercase_id do + rule ARITHMETIC_KEYWORDS, Keyword + rule KEYWORDS, Keyword + default Name end + rule %r/\(/, Operator, :push rule %r/\)/, Operator, :pop! rule %r/(>)(?=[^A-Z;]+[A-Z;>])/, Operator, :pop! @@ -135,7 +123,6 @@ def self.arithmetic_keywords mixin :common_rules rule %r/>/, Operator mixin :data_expression - end end end From 1c964aa1580e73ea3e536b5b7f50941a4c918fc7 Mon Sep 17 00:00:00 2001 From: jneen Date: Mon, 22 Jun 2026 18:08:27 -0400 Subject: [PATCH 138/154] use the keywords api for ruby --- lib/rouge/lexers/ruby.rb | 57 +++++++++++++--------------------------- 1 file changed, 18 insertions(+), 39 deletions(-) diff --git a/lib/rouge/lexers/ruby.rb b/lib/rouge/lexers/ruby.rb index 85ea9e51aa..142b1d31ab 100644 --- a/lib/rouge/lexers/ruby.rb +++ b/lib/rouge/lexers/ruby.rb @@ -105,19 +105,19 @@ def self.detect?(text) end end - keywords = %w( + KEYWORDS = Set.new %w( BEGIN END alias begin break case defined? do else elsif end ensure for if in next redo rescue raise retry return super then undef unless until when while yield ) - keywords_pseudo = %w( + KEYWORDS_PSEUDO = Set.new %w( loop include extend raise alias_method attr catch throw private module_function public protected true false nil __FILE__ __LINE__ ) - builtins_g = %w( + BUILTINS = Set.new %w( attr_reader attr_writer attr_accessor __id__ __send__ abort ancestors at_exit autoload binding callcc @@ -137,14 +137,14 @@ def self.detect?(text) taint test throw to_a to_s trace_var trap untaint untrace_var warn ) - builtins_q = %w( - autoload block_given const_defined eql equal frozen - include instance_of is_a iterator kind_of method_defined - nil private_method_defined protected_method_defined - public_method_defined respond_to tainted - ) + BUILTINS_SIGIL = Set.new %w( + autoload? block_given? const_defined? eql? equal? frozen? + include? instance_of? is_a? iterator? kind_of? method_defined? + nil? private_method_defined? protected_method_defined? + public_method_defined? respond_to? tainted? - builtins_b = %w(chomp chop exit gsub sub) + chomp! chop! exit! gsub! sub! + ) start do push :expr_start @@ -186,20 +186,12 @@ def self.detect?(text) mixin :strings - rule %r/\w+[?]?/ do |m| - if keywords.include?(m[0]) - token Keyword - elsif keywords_pseudo.include?(m[0]) - token Keyword::Pseudo - else - fallthrough! - end - - push :expr_start + keywords %r/\w+[?]?/ do + rule KEYWORDS, Keyword, :expr_start + rule KEYWORDS_PSEUDO, Keyword::Pseudo, :expr_start + rule Set['not', 'and', 'or'], Operator::Word, :expr_start end - rule %r/(not|and|or)\b/, Operator::Word, :expr_start - rule %r( (module) (\s+) @@ -218,25 +210,12 @@ def self.detect?(text) push :classname end - rule %r/(\w+)([?!])?/ do |m| - if m[2] == "?" && builtins_q.include?(m[1]) - token Name::Builtin - elsif m[2] == "!" && builtins_b.include?(m[1]) - token Name::Builtin - else - fallthrough! - end - - push :expr_start + keywords %r/\w+[?!]/ do |m| + rule BUILTINS_SIGIL, Name::Builtin, :expr_start end - rule %r/(? Date: Tue, 23 Jun 2026 09:27:46 -0400 Subject: [PATCH 139/154] use the keywords api for rust --- .rubocop_todo.yml | 1 - lib/rouge/lexers/rust.rb | 84 +++++++++++++++++++--------------------- spec/visual/samples/rust | 2 - 3 files changed, 39 insertions(+), 48 deletions(-) diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index b9e990610b..7051194f92 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -24,7 +24,6 @@ Naming/VariableName: # Offense count: 113 Rouge/NoBuildingAlternationPatternInRegexp: Exclude: - - 'lib/rouge/lexers/rust.rb' - 'lib/rouge/lexers/scala.rb' - 'lib/rouge/lexers/sml.rb' - 'lib/rouge/lexers/stata.rb' diff --git a/lib/rouge/lexers/rust.rb b/lib/rouge/lexers/rust.rb index d11659242b..cf39d933c6 100644 --- a/lib/rouge/lexers/rust.rb +++ b/lib/rouge/lexers/rust.rb @@ -20,35 +20,31 @@ def self.detect?(text) return true if text.shebang? 'rustc' end - def self.keywords - @keywords ||= %w( - as async await break const continue crate dyn else enum extern false - fn for if impl in let log loop match mod move mut pub ref return self - Self static struct super trait true type unsafe use where while - abstract become box do final macro - override priv typeof unsized virtual - yield try - union - ) - end - - def self.builtins - @builtins ||= Set.new %w( - Add BitAnd BitOr BitXor bool c_char c_double c_float char - c_int clock_t c_long c_longlong Copy c_schar c_short - c_uchar c_uint c_ulong c_ulonglong c_ushort c_void dev_t DIR - dirent Div Eq Err f32 f64 FILE float fpos_t - i16 i32 i64 i8 isize Index ino_t int intptr_t mode_t Mul - Neg None off_t Ok Option Ord Owned pid_t ptrdiff_t - Send Shl Shr size_t Some ssize_t str Sub time_t - u16 u32 u64 u8 usize uint uintptr_t - Box Vec String Rc Arc - u128 i128 Result Sync Pin Unpin Sized Drop drop Fn FnMut FnOnce - Clone PartialEq PartialOrd AsMut AsRef From Into Default - DoubleEndedIterator ExactSizeIterator Extend IntoIterator Iterator - FromIterator ToOwned ToString TryFrom TryInto - ) - end + KEYWORDS = Set.new %w( + as async await break const continue crate dyn else enum extern false + fn for if impl in let log loop match mod move mut pub ref return self + Self static struct super trait true type unsafe use where while + abstract become box do final macro + override priv typeof unsized virtual + yield try + union + ) + + BUILTINS = Set.new %w( + Add BitAnd BitOr BitXor bool c_char c_double c_float char + c_int clock_t c_long c_longlong Copy c_schar c_short + c_uchar c_uint c_ulong c_ulonglong c_ushort c_void dev_t DIR + dirent Div Eq Err f32 f64 FILE float fpos_t + i16 i32 i64 i8 isize Index ino_t int intptr_t mode_t Mul + Neg None off_t Ok Option Ord Owned pid_t ptrdiff_t + Send Shl Shr size_t Some ssize_t str Sub time_t + u16 u32 u64 u8 usize uint uintptr_t + Box Vec String Rc Arc + u128 i128 Result Sync Pin Unpin Sized Drop drop Fn FnMut FnOnce + Clone PartialEq PartialOrd AsMut AsRef From Into Default + DoubleEndedIterator ExactSizeIterator Extend IntoIterator Iterator + FromIterator ToOwned ToString TryFrom TryInto + ) def macro_closed? @macro_delims.values.all?(&:zero?) @@ -151,9 +147,22 @@ def macro_closed? rule %r/\n/, Text, :bol mixin :whitespace rule %r/#!?\[/, Name::Decorator, :attribute - rule %r/\b(?:#{Rust.keywords.join('|')})\b/, Keyword + + rule %r/(#{id})(::)/m do + groups Name::Namespace, Punctuation + end + + # macros + rule %r/\bmacro_rules!/, Name::Decorator, :macro_rules + rule %r/#{id}!/, Name::Decorator, :macro mixin :has_literals + keywords id do + rule KEYWORDS, Keyword + rule BUILTINS, Name::Builtin + default Name + end + rule %r([=-]>), Keyword rule %r(<->), Keyword rule %r/[()\[\]{}|,:;]/, Punctuation @@ -163,24 +172,9 @@ def macro_closed? rule %r/[.]\s*await\b/, Keyword rule %r/[.]\s*#{id}/, Name::Property rule %r/[.]\s*\d+/, Name::Attribute - rule %r/(#{id})(::)/m do - groups Name::Namespace, Punctuation - end - - # macros - rule %r/\bmacro_rules!/, Name::Decorator, :macro_rules - rule %r/#{id}!/, Name::Decorator, :macro rule %r/'static\b/, Keyword rule %r/'#{id}/, Name::Variable - rule %r/#{id}/ do |m| - name = m[0] - if self.class.builtins.include? name - token Name::Builtin - else - token Name - end - end end state :macro do diff --git a/spec/visual/samples/rust b/spec/visual/samples/rust index d046cce07f..4f72e2db84 100644 --- a/spec/visual/samples/rust +++ b/spec/visual/samples/rust @@ -191,7 +191,6 @@ mod test_stmt_multi_attr_outer { #[attr2 = "val"] fn f() { } - /* FIXME: Issue #493 #[attr1 = "val"] #[attr2 = "val"] mod mod1 { @@ -204,7 +203,6 @@ mod test_stmt_multi_attr_outer { extern mod rustrt { #[legacy_exports]; } - */ } } From bf5e99222332bc52b48fa86780dd70a8f17d23fd Mon Sep 17 00:00:00 2001 From: jneen Date: Tue, 23 Jun 2026 09:48:45 -0400 Subject: [PATCH 140/154] use the keywords api for SAS --- .rubocop_todo.yml | 1 - lib/rouge/lexers/sas.rb | 428 +++---------------------------- lib/rouge/lexers/sas/keywords.rb | 349 +++++++++++++++++++++++++ 3 files changed, 379 insertions(+), 399 deletions(-) create mode 100644 lib/rouge/lexers/sas/keywords.rb diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index 7051194f92..50cc8e5351 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -34,5 +34,4 @@ Rouge/NoBuildingAlternationPatternInRegexp: # Offense count: 20 Rouge/NoHugeCollections: Exclude: - - 'lib/rouge/lexers/sas.rb' - 'lib/rouge/lexers/stata.rb' diff --git a/lib/rouge/lexers/sas.rb b/lib/rouge/lexers/sas.rb index 521ffa2d43..26886c27b2 100644 --- a/lib/rouge/lexers/sas.rb +++ b/lib/rouge/lexers/sas.rb @@ -9,369 +9,7 @@ class SAS < RegexLexer filenames '*.sas' mimetypes 'application/x-sas', 'application/x-stat-sas', 'application/x-sas-syntax' - def self.data_step_statements - # from Data step statements - SAS 9.4 Statements reference - # http://support.sas.com/documentation/cdl/en/lestmtsref/68024/PDF/default/lestmtsref.pdf - @data_step_statements ||= Set.new %w( - ABORT ARRAY ATTRIB BY CALL CARDS CARDS4 CATNAME CHECKPOINT - EXECUTE_ALWAYS CONTINUE DATA DATALINES DATALINES4 DELETE DESCRIBE - DISPLAY DM DO UNTIL WHILE DROP END ENDSAS ERROR EXECUTE FILE FILENAME - FOOTNOTE FORMAT GO TO IF THEN ELSE INFILE INFORMAT INPUT - KEEP LABEL LEAVE LENGTH LIBNAME LINK LIST LOCK LOSTCARD MERGE - MISSING MODIFY OPTIONS OUTPUT PAGE PUT PUTLOG REDIRECT REMOVE RENAME - REPLACE RESETLINE RETAIN RETURN RUN SASFILE SELECT SET SKIP STOP - SYSECHO TITLE UPDATE WHERE WINDOW X - ) - # label: - # Sum - end - - def self.sas_functions - # from SAS 9.4 Functions and CALL Routines reference - # http://support.sas.com/documentation/cdl/en/lefunctionsref/67960/PDF/default/lefunctionsref.pdf - @sas_functions ||= Set.new %w( - ABS ADDR ADDRLONG AIRY ALLCOMB ALLPERM ANYALNUM ANYALPHA ANYCNTRL - ANYDIGIT ANYFIRST ANYGRAPH ANYLOWER ANYNAME ANYPRINT ANYPUNCT - ANYSPACE ANYUPPER ANYXDIGIT ARCOS ARCOSH ARSIN ARSINH ARTANH ATAN - ATAN2 ATTRC ATTRN BAND BETA BETAINV BLACKCLPRC BLACKPTPRC - BLKSHCLPRC BLKSHPTPRC BLSHIFT BNOT BOR BRSHIFT BXOR BYTE CAT CATQ - CATS CATT CATX CDF CEIL CEILZ CEXIST CHAR CHOOSEC CHOOSEN CINV - CLOSE CMISS CNONCT COALESCE COALESCEC COLLATE COMB COMPARE COMPBL - COMPFUZZ COMPGED COMPLEV COMPOUND COMPRESS CONSTANT CONVX CONVXP - COS COSH COT COUNT COUNTC COUNTW CSC CSS CUMIPMT CUMPRINC CUROBS - CV DACCDB DACCDBSL DACCSL DACCSYD DACCTAB DAIRY DATDIF DATE - DATEJUL DATEPART DATETIME DAY DCLOSE DCREATE DEPDB DEPDBSL DEPSL - DEPSYD DEPTAB DEQUOTE DEVIANCE DHMS DIF DIGAMMA DIM DINFO DIVIDE - DNUM DOPEN DOPTNAME DOPTNUM DOSUBL DREAD DROPNOTE DSNAME - DSNCATLGD DUR DURP EFFRATE ENVLEN ERF ERFC EUCLID EXIST EXP FACT - FAPPEND FCLOSE FCOL FCOPY FDELETE FETCH FETCHOBS FEXIST FGET - FILEEXIST FILENAME FILEREF FINANCE FIND FINDC FINDW FINFO FINV - FIPNAME FIPNAMEL FIPSTATE FIRST FLOOR FLOORZ FMTINFO FNONCT FNOTE - FOPEN FOPTNAME FOPTNUM FPOINT FPOS FPUT FREAD FREWIND FRLEN FSEP - FUZZ FWRITE GAMINV GAMMA GARKHCLPRC GARKHPTPRC GCD GEODIST - GEOMEAN GEOMEANZ GETOPTION GETVARC GETVARN GRAYCODE HARMEAN - HARMEANZ HBOUND HMS HOLIDAY HOLIDAYCK HOLIDAYCOUNT HOLIDAYNAME - HOLIDAYNX HOLIDAYNY HOLIDAYTEST HOUR HTMLDECODE HTMLENCODE - IBESSEL IFC IFN INDEX INDEXC INDEXW INPUT INPUTC INPUTN INT - INTCINDEX INTCK INTCYCLE INTFIT INTFMT INTGET INTINDEX INTNX - INTRR INTSEAS INTSHIFT INTTEST INTZ IORCMSG IPMT IQR IRR JBESSEL - JULDATE JULDATE7 KURTOSIS LAG LARGEST LBOUND LCM LCOMB LEFT - LENGTH LENGTHC LENGTHM LENGTHN LEXCOMB LEXCOMBI LEXPERK LEXPERM - LFACT LGAMMA LIBNAME LIBREF LOG LOG1PX LOG10 LOG2 LOGBETA LOGCDF - LOGISTIC LOGPDF LOGSDF LOWCASE LPERM LPNORM MAD MARGRCLPRC - MARGRPTPRC MAX MD5 MDY MEAN MEDIAN MIN MINUTE MISSING MOD - MODEXIST MODULE MODULEC MODULEN MODZ MONTH MOPEN MORT MSPLINT - MVALID N NETPV NLITERAL NMISS NOMRATE NORMAL NOTALNUM NOTALPHA - NOTCNTRL NOTDIGIT NOTE NOTFIRST NOTGRAPH NOTLOWER NOTNAME - NOTPRINT NOTPUNCT NOTSPACE NOTUPPER NOTXDIGIT NPV NVALID NWKDOM - OPEN ORDINAL PATHNAME PCTL PDF PEEK PEEKC PEEKCLONG PEEKLONG PERM - PMT POINT POISSON PPMT PROBBETA PROBBNML PROBBNRM PROBCHI PROBF - PROBGAM PROBHYPR PROBIT PROBMC PROBNEGB PROBNORM PROBT PROPCASE - PRXCHANGE PRXMATCH PRXPAREN PRXPARSE PRXPOSN PTRLONGADD PUT PUTC - PUTN PVP QTR QUANTILE QUOTE RANBIN RANCAU RAND RANEXP RANGAM - RANGE RANK RANNOR RANPOI RANTBL RANTRI RANUNI RENAME REPEAT - RESOLVE REVERSE REWIND RIGHT RMS ROUND ROUNDE ROUNDZ SAVING - SAVINGS SCAN SDF SEC SECOND SHA256 SHA256HEX SHA256HMACHEX SIGN - SIN SINH SKEWNESS SLEEP SMALLEST SOAPWEB SOAPWEBMETA - SOAPWIPSERVICE SOAPWIPSRS SOAPWS SOAPWSMETA SOUNDEX SPEDIS SQRT - SQUANTILE STD STDERR STFIPS STNAME STNAMEL STRIP SUBPAD SUBSTR - SUBSTRN SUM SUMABS SYMEXIST SYMGET SYMGLOBL SYMLOCAL SYSEXIST - SYSGET SYSMSG SYSPARM SYSPROCESSID SYSPROCESSNAME SYSPROD SYSRC - SYSTEM TAN TANH TIME TIMEPART TIMEVALUE TINV TNONCT TODAY - TRANSLATE TRANSTRN TRANWRD TRIGAMMA TRIM TRIMN TRUNC TSO TYPEOF - TZONEID TZONENAME TZONEOFF TZONES2U TZONEU2S UNIFORM UPCASE - URLDECODE URLENCODE USS UUIDGEN VAR VARFMT VARINFMT VARLABEL - VARLEN VARNAME VARNUM VARRAY VARRAYX VARTYPE VERIFY VFORMAT - VFORMATD VFORMATDX VFORMATN VFORMATNX VFORMATW VFORMATWX VFORMATX - VINARRAY VINARRAYX VINFORMAT VINFORMATD VINFORMATDX VINFORMATN - VINFORMATNX VINFORMATW VINFORMATWX VINFORMATX VLABEL VLABELX - VLENGTH VLENGTHX VNAME VNAMEX VTYPE VTYPEX VVALUE VVALUEX WEEK - WEEKDAY WHICHC WHICHN WTO YEAR YIELDP YRDIF YYQ ZIPCITY - ZIPCITYDISTANCE ZIPFIPS ZIPNAME ZIPNAMEL ZIPSTATE - ) - end - - def self.sas_macro_statements - # from SAS 9.4 Macro Language Reference - # Chapter 12 - @sas_macro_statements ||= Set.new %w( - %COPY %DISPLAY %GLOBAL %INPUT %LET %MACRO %PUT %SYMDEL %SYSCALL - %SYSEXEC %SYSLPUT %SYSMACDELETE %SYSMSTORECLEAR %SYSRPUT %WINDOW - %ABORT %DO %TO %UNTIL %WHILE %END %GOTO %IF %THEN %ELSE %LOCAL - %RETURN - %INCLUDE %LIST %RUN - ) - # Omitted: - # %label: Identifies the destination of a %GOTO statement. - # %MEND - end - - def self.sas_macro_functions - # from SAS 9.4 Macro Language Reference - # Chapter 12 - - @sas_macro_functions ||= Set.new %w( - %BQUOTE %NRBQUOTE %EVAL %INDEX %LENGTH %QUOTE %NRQUOTE %SCAN - %QSCAN %STR %NRSTR %SUBSTR %QSUBSTR %SUPERQ %SYMEXIST %SYMGLOBL - %SYMLOCAL %SYSEVALF %SYSFUNC %QSYSFUNC %SYSGET %SYSMACEXEC - %SYSMACEXIST %SYSMEXECDEPTH %SYSMEXECNAME %SYSPROD %UNQUOTE - %UPCASE %QUPCASE - ) - end - - def self.sas_auto_macro_vars - # from SAS 9.4 Macro Language Reference - # Chapter 12 - - @sas_auto_macro_vars ||= Set.new %w( - &SYSADDRBITS &SYSBUFFR &SYSCC &SYSCHARWIDTH &SYSCMD &SYSDATASTEPPHASE &SYSDATE - &SYSDATE9 &SYSDAY &SYSDEVIC &SYSDMG &SYSDSN &SYSENCODING &SYSENDIAN &SYSENV - &SYSERR &SYSERRORTEXT &SYSFILRC &SYSHOSTINFOLONG &SYSHOSTNAME &SYSINDEX - &SYSINFO &SYSJOBID &SYSLAST &SYSLCKRC &SYSLIBRC &SYSLOGAPPLNAME &SYSMACRONAME - &SYSMENV &SYSMSG &SYSNCPU &SYSNOBS &SYSODSESCAPECHAR &SYSODSPATH &SYSPARM - &SYSPBUFF &SYSPRINTTOLIST &SYSPRINTTOLOG &SYSPROCESSID &SYSPROCESSMODE - &SYSPROCESSNAME &SYSPROCNAME &SYSRC &SYSSCP &SYSSCPL &SYSSITE &SYSSIZEOFLONG - &SYSSIZEOFPTR &SYSSIZEOFUNICODE &SYSSTARTID &SYSSTARTNAME &SYSTCPIPHOSTNAME - &SYSTIME &SYSTIMEZONE &SYSTIMEZONEIDENT &SYSTIMEZONEOFFSET &SYSUSERID &SYSVER - &SYSVLONG &SYSVLONG4 &SYSWARNINGTEXT - ) - end - - def self.proc_keywords - # Create a hash with keywords for common PROCs, keyed by PROC name - @proc_keywords ||= {} - - @proc_keywords["SQL"] ||= Set.new %w( - ALTER TABLE CONNECT CREATE INDEX VIEW DELETE DESCRIBE DISCONNECT DROP EXECUTE - INSERT RESET SELECT UPDATE VALIDATE ADD CONSTRAINT FOREIGN KEY PRIMARY - MODIFY LIKE AS ORDER BY USING FROM INTO SET VALUES DISTINCT UNIQUE - WHERE GROUP HAVING LEFT RIGHT INNER JOIN ON - ) - # from SAS 9.4 SQL Procedure User's Guide - - @proc_keywords["MEANS"] ||= Set.new %w( - BY CLASS FREQ ID OUTPUT OUT TYPES VAR WAYS WEIGHT - ATTRIB FORMAT LABEL WHERE - DESCENDING NOTSORTED - NOTHREADS NOTRAP PCTLDEF SUMSIZE THREADS CLASSDATA COMPLETETYPES - EXCLUSIVE MISSING FW MAXDEC NONOBS NOPRINT ORDER FORMATTED - UNFORMATTED PRINT PRINTALLTYPES PRINTIDVARS STACKODSOUTPUT - CHARTYPE DESCENDTYPES IDMIN - ALPHA EXCLNPWGT QMARKERS QMETHOD QNTLDEF VARDEF - CLM CSS CV KURTOSIS KURT LCLM MAX MEAN MIN MODE N - NMISS RANGE SKEWNESS SKEW STDDEV STD STDERR SUM SUMWGT UCLM USS - MEDIAN P50 Q1 P25 Q3 P75 P1 P90 P5 P95 P10 P99 P20 P30 P40 P60 P70 - P80 QRANGE - PROBT PRT T - ASCENDING GROUPINTERNAL MLF PRELOADFMT - MAXID AUTOLABEL AUTONAME KEEPLEN LEVELS NOINHERIT - ) - # from BASE SAS 9.4 Procedures Guide, Fifth Edition - - @proc_keywords["DATASETS"] ||= Set.new %w( - AGE APPEND ATTRIB AUDIT CHANGE CONTENTS COPY DELETE EXCHANGE - EXCLUDE FORMAT IC CREATE REACTIVATE INDEX CENTILES INFORMAT - INITIATE LABEL LOG MODIFY REBUILD RENAME REPAIR RESUME SAVE SELECT - SUSPEND TERMINATE USER_VAR XATTR ADD OPTIONS REMOVE SET - ) - # from BASE SAS 9.4 Procedures Guide, Fifth Edition - - @proc_keywords["SORT"] ||= Set.new %w( - BY DESCENDING KEY ASCENDING ASC DESC DATECOPY FORCE OVERWRITE - PRESORTED SORTSIZE TAGSORT DUPOUT OUT UNIQUEOUT NODUPKEY NOUNIQUEKEY - NOTHREADS THREADS EQUALS NOEQUALS - ATTRIB FORMAT LABEL WHERE - ) - # from BASE SAS 9.4 Procedures Guide, Fifth Edition - - @proc_keywords["PRINT"] ||= Set.new %w( - BY DESCENDING NOTSORTED PAGEBY SUMBY ID STYLE SUM VAR CONTENTS DATA - GRANDTOTAL_LABEL HEADING LABEL SPLIT SUMLABEL NOSUMLABEL - BLANKLINE COUNT DOUBLE N NOOBS OBS ROUND - ROWS UNIFORM WIDTH - ATTRIB FORMAT WHERE - ) - # from BASE SAS 9.4 Procedures Guide, Fifth Edition - - @proc_keywords["APPEND"] ||= Set.new %w( - BASE APPENDVER DATA ENCRYPTKEY FORCE GETSORT NOWARN - ATTRIB FORMAT LABEL WHERE - ) - # from BASE SAS 9.4 Procedures Guide, Fifth Edition - - @proc_keywords["TRANSPOSE"] ||= Set.new %w( - DELIMITER LABEL LET NAME OUT PREFIX SUFFIX BY DESCENDING NOTSORTED - COPY ID IDLABEL VAR INDB - ATTRIB FORMAT WHERE - ) - # from BASE SAS 9.4 Procedures Guide, Fifth Edition - - @proc_keywords["FREQ"] ||= Set.new %w( - BY EXACT OUTPUT TABLES TEST WEIGHT - COMPRESS DATA FORMCHAR NLEVELS NOPRINT ORDER PAGE FORMATTED FREQ - INTERNAL - AGREE BARNARD BINOMIAL BIN CHISQ COMOR EQOR ZELEN FISHER JT KAPPA - KENTB TAUB LRCHI MCNEM MEASURES MHCHI OR ODDSRATIO PCHI PCORR RELRISK - RISKDIFF SCORR SMDCR SMDRC STUTC TAUC TREND WTKAP WTKAPPA - OUT AJCHI ALL BDCHI CMH CMH1 CMH2 CMHCOR CMHGA CMHRMS COCHQ CONTGY - CRAMV EQKAP EQWKP GAMMA GS GAILSIMON LAMCR LAMDAS LAMRC LGOR LGRRC1 - LGRRC2 MHOR MHRRC1 MHRRC2 N NMISS PHI PLCORR RDIF1 RDIF2 RISKDIFF1 - RISKDIFF2 RRC1 RELRISK1 RRC2 RELRISK2 RSK1 RISK1 RSK11 RISK11 RSK12 - RISK12 RSK21 RISK21 RSK22 RISK22 TSYMM BOWKER U UCR URC - CELLCHI2 CUMCOL DEVIATION EXPECTED MISSPRINT PEARSONREF PRINTWKTS - SCOROUT SPARSE STDRES TOTPCT - CONTENTS CROSSLIST FORMAT LIST MAXLEVELS NOCOL NOCUM NOFREQ NOPERCENT NOROW NOSPARSE NOWARN PLOTS OUTCUM OUTEXPECT OUTPCT - ZEROS - ) - # from Base SAS 9.4 Procedures Guide: Statistical Procedures, Fourth Edition - - @proc_keywords["CORR"] ||= Set.new %w( - BY FREQ ID PARTIAL VAR WEIGHT WITH - DATA OUTH OUTK OUTP OUTPLC OUTPLS OUTS - EXCLNPWGHT FISHER HOEFFDING KENDALL NOMISS PEARSON POLYCHORIC - POLYSERIAL ALPHA COV CSSCP SINGULAR SSCP VARDEF PLOTS MATRIX SCATTER - BEST NOCORR NOPRINT NOPROB NOSIMPLE RANK - ) - # from Base SAS 9.4 Procedures Guide: Statistical Procedures, Fourth Edition - - @proc_keywords["REPORT"] ||= Set.new %w( - BREAK BY DESCENDING NOTSORTED COLUMN COMPUTE STYLE LINE ENDCOMP - CALL DEFINE _ROW_ FREQ RBREAK WEIGHT - ATTRIB FORMAT LABEL WHERE - DATA NOALIAS NOCENTER NOCOMPLETECOLS NOCOMPLETEROWS NOTHREADS - NOWINDOWS OUT PCTLDEF THREADS WINDOWS COMPLETECOLS - COMPLETEROWS CONTENTS SPANROWS COMMAND HELP PROMPT - BOX BYPAGENO CENTER COLWIDTH FORMCHAR LS MISSING PANELS PS - PSPACE SHOWALL SPACING WRAP EXCLNPWGT QMARKERS QMETHOD QNTLDEF VARDEF - NAMED NOHEADER SPLIT HEADLINE HEADSKIP LIST NOEXEC OUTREPT PROFILE - REPORT - COLOR DOL DUL OL PAGE SKIP SUMMARIZE SUPPRESS UL - BLINK HIGHLIGHT RVSVIDEO MERGE REPLACE URL URLBP URLP - AFTER BEFORE _PAGE_ LEFT RIGHT CHARACTER LENGTH - EXCLUSIVE MLF ORDER FORMATTED INTERNAL PRELOADFMT - WIDTH - ACROSS ANALYSIS COMPUTED DISPLAY GROUP FLOW ID NOPRINT NOZERO - CSS CV MAX MEAN MIN MODE N NMISS PCTN PCTSUM RANGE STD STDERR SUM - SUMWGT USS VAR - MEDIAN P50 Q1 P25 Q3 P75 P1 P90 P5 P95 P10 P99 P20 P30 P40 P60 P70 - P80 QRANGE - PROBT PRT T - ) - # from BASE SAS 9.4 Procedures Guide, Fifth Edition - - @proc_keywords["METALIB"] ||= Set.new %w( - OMR DBAUTH DBUSER DBPASSWORD EXCLUDE SELECT READ FOLDER FOLDERID - IMPACT_LIMIT NOEXEC PREFIX REPORT UPDATE_RULE DELETE NOADD NODELDUP - NOUPDATE - LIBID LIBRARY LIBURI - TYPE DETAIL SUMMARY - ) - # from SAS 9.4 Language Interfaces to Metadata, Third Edition - - @proc_keywords["GCHART"] ||= Set.new %w( - DATA ANNOTATE GOUT IMAGEMAP BLOCK HBAR HBAR3D VBAR VBAR3D PIE PIE3D - DONUT STAR ANNO - BY NOTE FORMAT LABEL WHERE - BLOCKMAX CAXIS COUTLINE CTEXT LEGEND NOHEADING NOLEGEND PATTERNID - GROUP MIDPOINT SUBGROUP WOUTLINE DESCRIPTION NAME DISCRETE LEVELS - OLD MISSING HTML_LEGEND HTML URL FREQ G100 SUMVAR TYPE - CAUTOREF CERROR CFRAME CLM CREF FRAME NOFRAME GSPACE IFRAME - IMAGESTYLE TILE FIT LAUTOREF NOSYMBOL SHAPE SPACE - SUBOUTSIDE WAUTOREF WIDTH WREF - ASCENDING AUTOREF CLIPREF DESCENDING FRONTREF GAXIS MAXIS MINOR - NOAXIS NOBASEREF NOZERO RANGE AXIS REF CFREQ CFREQLABEL NONE CPERCENT - CPERCENTLABEL ERRORBAR BARS BOTH TOP FREQLABEL INSIDE MEAN MEANLABEL - NOSTATS OUTSIDE PERCENT PERCENTLABEL PERCENTSUM SUM - CFILL DETAIL_RADIUS EXPLODE FILL SOLID X INVISIBLE - RADIUS DETAIL_THRESHOLD DETAIL_PERCENT DETAIL_SLICE - DETAIL_VALUE DONUTPCT ACROSS DOWN NOGROUPHEADING - MATCHCOLOR OTHERCOLOR OTHERLABEL ARROW PLABEL PPERCENT SLICE - VALUE - ANGLE CLOCKWISE JSTYLE - NOCONNECT STARMAX STARMIN - ) - # from SAS GRAPH 9.4 Reference, Fourth Edition - - @proc_keywords["GPLOT"] ||= Set.new %w( - DATA ANNOTATE GOUT IMAGEMAP UNIFORM BUBBLE BUBBLE2 PLOT PLOT2 - BCOLOR BFILL BFONT BLABEL BSCALE AREA RADIUS BSIZE DESCRIPTION NAME - AUTOHREF CAUTOHREF CHREF HAXIS HMINOR HREF HREVERSE HZERO LAUTOHREF - LHREF WAUTOHREF WHREF HTML URL - CAXIS CFRAME CTEXT DATAORDER FRAME NOFRAME FRONTREF GRID IFRAME - IMAGESTYLE TILE FIT NOAXIS - AUTOVREF CAUTOVREF CVREF LAUTOVREF LVREF VAXIS VMINOR VREF VREVERSE - VZERO WAUTOVREF WVREF - CBASELINE COUTLINE - AREAS LEGEND NOLASTAREA NOLEGEND OVERLAY REGEQN SKIPMISS - ) - # from SAS GRAPH 9.4 Reference, Fourth Edition - - @proc_keywords["REG"] ||= Set.new %w( - MODEL BY FREQ ID VAR WEIGHT ADD CODE DELETE MTEST OUTPUT PAINT - PLOT PRINT REFIT RESTRICT REWEIGHT STORE TEST - ) - # from SAS/STAT 15.1 User's Guide - - @proc_keywords["SGPLOT"] ||= Set.new %w( - STYLEATTRS BAND X Y UPPER LOWER BLOCK BUBBLE DENSITY DOT DROPLINE - ELLIPSE ELLIPSEPARM FRINGE GRADLEGEND HBAR HBARBASIC HBARPARM - HBOX HEATMAP HEATMAPPARM HIGHLOW HISTOGRAM HLINE INSET KEYLEGEND - LINEPARM LOESS NEEDLE PBSPLINE POLYGON REFLINE REG SCATTER SERIES - SPLINE STEP SYMBOLCHAR SYMBOLIMAGE TEXT VBAR VBARBASIC VBARPARM - VBOX VECTOR VLINE WATERFALL XAXIS X2AXIS XAXISTABLE YAXIS Y2AXIS - YAXISTABLE - ) - - # from ODS Graphics: Procedures Guide, Sixth Edition - return @proc_keywords - end - - def self.sas_proc_names - # from SAS Procedures by Name - # http://support.sas.com/documentation/cdl/en/allprodsproc/68038/HTML/default/viewer.htm#procedures.htm - - @sas_proc_names ||= Set.new %w( - ACCESS ACECLUS ADAPTIVEREG ALLELE ANOM ANOVA APPEND APPSRV ARIMA - AUTHLIB AUTOREG BCHOICE BOM BOXPLOT BTL BUILD CALENDAR CALIS CALLRFC - CANCORR CANDISC CAPABILITY CASECONTROL CATALOG CATMOD CDISC - CHART CIMPORT CLP CLUSTER COMPARE COMPILE COMPUTAB CONTENTS CONVERT - COPULA COPY CORR CORRESP COUNTREG CPM CPORT CUSUM CV2VIEW DATEKEYS - DATASETS DATASOURCE DB2EXT DB2UTIL DBCSTAB DBF DBLOAD DELETE DIF - DISCRIM DISPLAY DISTANCE DMSRVADM DMSRVDATASVC DMSRVPROCESSSVC - DOCUMENT DOWNLOAD DQLOCLST DQMATCH DQSCHEME DS2 DTREE ENTROPY ESM - EXPAND EXPLODE EXPORT FACTEX FACTOR FAMILY FASTCLUS FCMP FEDSQL FMM - FONTREG FORECAST FORMAT FORMS FREQ FSBROWSE FSEDIT FSLETTER FSLIST - FSVIEW G3D G3GRID GA GAM GAMPL GANNO GANTT GAREABAR GBARLINE GCHART - GCONTOUR GDEVICE GEE GENESELECT GENMOD GEOCODE GFONT GINSIDE GIS GKPI - GLIMMIX GLM GLMMOD GLMPOWER GLMSELECT GMAP GOPTIONS GPLOT GPROJECT - GRADAR GREDUCE GREMOVE GREPLAY GROOVY GSLIDE GTILE HADOOP HAPLOTYPE - HDMD HPBIN HPCANDISC HPCDM HPCOPULA HPCORR HPCOUNTREG HPDMDB HPDS2 - HPFMM HPGENSELECT HPIMPUTE HPLMIXED HPLOGISTIC HPMIXED HPNLMOD - HPPANEL HPPLS HPPRINCOMP HPQUANTSELECT HPQLIM HPREG HPSAMPLE - HPSEVERITY HPSPLIT HPSUMMARY HTSNP HTTP ICLIFETEST ICPHREG IML IMPORT - IMSTAT IMXFER INBREED INFOMAPS INTPOINT IOMOPERATE IRT ISHIKAWA ITEMS - JAVAINFO JSON KDE KRIGE2D LASR LATTICE LIFEREG LIFETEST LOAN - LOCALEDATA LOESS LOGISTIC LP LUA MACONTROL MAPIMPORT MCMC MDC MDDB - MDS MEANS METADATA METALIB METAOPERATE MI MIANALYZE MIGRATE MIXED - MODECLUS MODEL MSCHART MULTTEST MVPDIAGNOSE MVPMODEL MVPMONITOR - NESTED NETDRAW NETFLOW NLIN NLMIXED NLP NPAR1WAY ODSLIST ODSTABLE - ODSTEXT OLAP OLAPCONTENTS OLAPOPERATE OPERATE OPTEX OPTGRAPH OPTIONS - OPTLOAD OPTLP OPTLSO OPTMILP OPTMODEL OPTNET OPTQP OPTSAVE ORTHOREG - PANEL PARETO PDLREG PDS PDSCOPY PHREG PLAN PLM PLOT PLS PM PMENU - POWER PRESENV PRINCOMP PRINQUAL PRINT PRINTTO PROBIT PROTO PRTDEF - PRTEXP PSMOOTH PWENCODE QDEVICE QLIM QUANTLIFE QUANTREG QUANTSELECT - QUEST RANK RAREEVENTS RDC RDPOOL RDSEC RECOMMEND REG REGISTRY RELEASE - RELIABILITY REPORT RISK ROBUSTREG RSREG SCAPROC SCORE SEQDESIGN - SEQTEST SERVER SEVERITY SGDESIGN SGPANEL SGPLOT SGRENDER SGSCATTER - SHEWHART SIM2D SIMILARITY SIMLIN SIMNORMAL SOAP SORT SOURCE SPECTRA - SPP SQL SQOOP SSM STANDARD STATESPACE STDIZE STDRATE STEPDISC STP - STREAM SUMMARY SURVEYFREQ SURVEYIMPUTE SURVEYLOGISTIC SURVEYMEANS - SURVEYPHREG SURVEYREG SURVEYSELECT SYSLIN TABULATE TAPECOPY TAPELABEL - TEMPLATE TIMEDATA TIMEID TIMEPLOT TIMESERIES TPSPLINE TRANSPOSE - TRANSREG TRANTAB TREE TSCSREG TTEST UCM UNIVARIATE UPLOAD VARCLUS - VARCOMP VARIOGRAM VARMAX VASMP X11 X12 X13 XSL - ) - end + lazy { require_relative 'sas/keywords' } state :basics do # Rules to be parsed before the keywords (which are different depending @@ -414,12 +52,12 @@ def self.sas_proc_names groups Keyword, Text, Punctuation push :datalines end + rule %r/\b(datalines4|cards4)(\s*)(;)/i do groups Keyword, Text, Punctuation push :datalines4 end - # operators (Language Reference p96) rule %r(\*\*|[\*/\+-]), Operator rule %r/[^¬~]?=:?|[<>]=?:?/, Operator @@ -441,16 +79,14 @@ def self.sas_proc_names rule %r/(%macro|%mend)(\s*)(\w+)/i do groups Keyword, Text, Name::Function end + rule %r/%mend/, Keyword - rule %r/%\w+/ do |m| - if self.class.sas_macro_statements.include? m[0].upcase - token Keyword - elsif self.class.sas_macro_functions.include? m[0].upcase - token Keyword - else - token Name - end + keywords %r/%\w+/ do + transform(&:upcase) + rule SAS_MACRO_STATEMENTS, Keyword + rule SAS_MACRO_FUNCTIONS, Keyword + default Name end end @@ -477,7 +113,7 @@ def self.sas_proc_names rule %r!(proc)(\s+)(\w+)!ix do |m| @proc_name = m[3].upcase puts " proc name: #{@proc_name}" if @debug - if self.class.sas_proc_names.include? @proc_name + if SAS_PROC_NAMES.include? @proc_name groups Keyword, Text, Keyword else groups Keyword, Text, Name @@ -495,20 +131,16 @@ def self.sas_proc_names groups Keyword, Text, Name::Variable end - rule %r/\w+/ do |m| - if self.class.data_step_statements.include? m[0].upcase - token Keyword - elsif self.class.sas_functions.include? m[0].upcase - token Keyword - else - token Name - end + keywords %r/\w+/ do + transform(&:upcase) + rule DATA_STEP_STATEMENTS, Keyword + rule SAS_FUNCTIONS, Keyword + default Name end - mixin :basics2 + mixin :basics2 end - state :single_string do rule %r/''/, Str::Escape rule %r/'/, Str::Single, :pop! @@ -536,27 +168,27 @@ def self.sas_proc_names rule %r/;{,3}/, Literal::String::Heredoc end - # PROCS state :proc do rule %r/(quit|run)/i, Keyword, :pop! mixin :basics - rule %r/\w+/ do |m| - if self.class.data_step_statements.include? m[0].upcase - token Keyword - elsif self.class.sas_functions.include? m[0].upcase - token Keyword - elsif self.class.proc_keywords.has_key?(@proc_name) and self.class.proc_keywords[@proc_name].include? m[0].upcase - token Keyword - else - token Name + + keywords %r/\w+/ do + rule DATA_STEP_STATEMENTS, Keyword + rule SAS_FUNCTIONS, Keyword + + default do |m| + if PROC_KEYWORDS[@proc_name]&.include?(m[0].upcase) + token Keyword + else + token Name + end end end - mixin :basics2 + mixin :basics2 end - - end #class SAS - end #module Lexers -end #module Rouge + end + end +end diff --git a/lib/rouge/lexers/sas/keywords.rb b/lib/rouge/lexers/sas/keywords.rb new file mode 100644 index 0000000000..7d4fc93edc --- /dev/null +++ b/lib/rouge/lexers/sas/keywords.rb @@ -0,0 +1,349 @@ +module Rouge + module Lexers + class SAS + # from Data step statements - SAS 9.4 Statements reference + # http://support.sas.com/documentation/cdl/en/lestmtsref/68024/PDF/default/lestmtsref.pdf + DATA_STEP_STATEMENTS = Set.new %w( + ABORT ARRAY ATTRIB BY CALL CARDS CARDS4 CATNAME CHECKPOINT + EXECUTE_ALWAYS CONTINUE DATA DATALINES DATALINES4 DELETE DESCRIBE + DISPLAY DM DO UNTIL WHILE DROP END ENDSAS ERROR EXECUTE FILE FILENAME + FOOTNOTE FORMAT GO TO IF THEN ELSE INFILE INFORMAT INPUT + KEEP LABEL LEAVE LENGTH LIBNAME LINK LIST LOCK LOSTCARD MERGE + MISSING MODIFY OPTIONS OUTPUT PAGE PUT PUTLOG REDIRECT REMOVE RENAME + REPLACE RESETLINE RETAIN RETURN RUN SASFILE SELECT SET SKIP STOP + SYSECHO TITLE UPDATE WHERE WINDOW X + ) + + # from SAS 9.4 Functions and CALL Routines reference + # http://support.sas.com/documentation/cdl/en/lefunctionsref/67960/PDF/default/lefunctionsref.pdf + SAS_FUNCTIONS = Set.new %w( + ABS ADDR ADDRLONG AIRY ALLCOMB ALLPERM ANYALNUM ANYALPHA ANYCNTRL + ANYDIGIT ANYFIRST ANYGRAPH ANYLOWER ANYNAME ANYPRINT ANYPUNCT + ANYSPACE ANYUPPER ANYXDIGIT ARCOS ARCOSH ARSIN ARSINH ARTANH ATAN + ATAN2 ATTRC ATTRN BAND BETA BETAINV BLACKCLPRC BLACKPTPRC + BLKSHCLPRC BLKSHPTPRC BLSHIFT BNOT BOR BRSHIFT BXOR BYTE CAT CATQ + CATS CATT CATX CDF CEIL CEILZ CEXIST CHAR CHOOSEC CHOOSEN CINV + CLOSE CMISS CNONCT COALESCE COALESCEC COLLATE COMB COMPARE COMPBL + COMPFUZZ COMPGED COMPLEV COMPOUND COMPRESS CONSTANT CONVX CONVXP + COS COSH COT COUNT COUNTC COUNTW CSC CSS CUMIPMT CUMPRINC CUROBS + CV DACCDB DACCDBSL DACCSL DACCSYD DACCTAB DAIRY DATDIF DATE + DATEJUL DATEPART DATETIME DAY DCLOSE DCREATE DEPDB DEPDBSL DEPSL + DEPSYD DEPTAB DEQUOTE DEVIANCE DHMS DIF DIGAMMA DIM DINFO DIVIDE + DNUM DOPEN DOPTNAME DOPTNUM DOSUBL DREAD DROPNOTE DSNAME + DSNCATLGD DUR DURP EFFRATE ENVLEN ERF ERFC EUCLID EXIST EXP FACT + FAPPEND FCLOSE FCOL FCOPY FDELETE FETCH FETCHOBS FEXIST FGET + FILEEXIST FILENAME FILEREF FINANCE FIND FINDC FINDW FINFO FINV + FIPNAME FIPNAMEL FIPSTATE FIRST FLOOR FLOORZ FMTINFO FNONCT FNOTE + FOPEN FOPTNAME FOPTNUM FPOINT FPOS FPUT FREAD FREWIND FRLEN FSEP + FUZZ FWRITE GAMINV GAMMA GARKHCLPRC GARKHPTPRC GCD GEODIST + GEOMEAN GEOMEANZ GETOPTION GETVARC GETVARN GRAYCODE HARMEAN + HARMEANZ HBOUND HMS HOLIDAY HOLIDAYCK HOLIDAYCOUNT HOLIDAYNAME + HOLIDAYNX HOLIDAYNY HOLIDAYTEST HOUR HTMLDECODE HTMLENCODE + IBESSEL IFC IFN INDEX INDEXC INDEXW INPUT INPUTC INPUTN INT + INTCINDEX INTCK INTCYCLE INTFIT INTFMT INTGET INTINDEX INTNX + INTRR INTSEAS INTSHIFT INTTEST INTZ IORCMSG IPMT IQR IRR JBESSEL + JULDATE JULDATE7 KURTOSIS LAG LARGEST LBOUND LCM LCOMB LEFT + LENGTH LENGTHC LENGTHM LENGTHN LEXCOMB LEXCOMBI LEXPERK LEXPERM + LFACT LGAMMA LIBNAME LIBREF LOG LOG1PX LOG10 LOG2 LOGBETA LOGCDF + LOGISTIC LOGPDF LOGSDF LOWCASE LPERM LPNORM MAD MARGRCLPRC + MARGRPTPRC MAX MD5 MDY MEAN MEDIAN MIN MINUTE MISSING MOD + MODEXIST MODULE MODULEC MODULEN MODZ MONTH MOPEN MORT MSPLINT + MVALID N NETPV NLITERAL NMISS NOMRATE NORMAL NOTALNUM NOTALPHA + NOTCNTRL NOTDIGIT NOTE NOTFIRST NOTGRAPH NOTLOWER NOTNAME + NOTPRINT NOTPUNCT NOTSPACE NOTUPPER NOTXDIGIT NPV NVALID NWKDOM + OPEN ORDINAL PATHNAME PCTL PDF PEEK PEEKC PEEKCLONG PEEKLONG PERM + PMT POINT POISSON PPMT PROBBETA PROBBNML PROBBNRM PROBCHI PROBF + PROBGAM PROBHYPR PROBIT PROBMC PROBNEGB PROBNORM PROBT PROPCASE + PRXCHANGE PRXMATCH PRXPAREN PRXPARSE PRXPOSN PTRLONGADD PUT PUTC + PUTN PVP QTR QUANTILE QUOTE RANBIN RANCAU RAND RANEXP RANGAM + RANGE RANK RANNOR RANPOI RANTBL RANTRI RANUNI RENAME REPEAT + RESOLVE REVERSE REWIND RIGHT RMS ROUND ROUNDE ROUNDZ SAVING + SAVINGS SCAN SDF SEC SECOND SHA256 SHA256HEX SHA256HMACHEX SIGN + SIN SINH SKEWNESS SLEEP SMALLEST SOAPWEB SOAPWEBMETA + SOAPWIPSERVICE SOAPWIPSRS SOAPWS SOAPWSMETA SOUNDEX SPEDIS SQRT + SQUANTILE STD STDERR STFIPS STNAME STNAMEL STRIP SUBPAD SUBSTR + SUBSTRN SUM SUMABS SYMEXIST SYMGET SYMGLOBL SYMLOCAL SYSEXIST + SYSGET SYSMSG SYSPARM SYSPROCESSID SYSPROCESSNAME SYSPROD SYSRC + SYSTEM TAN TANH TIME TIMEPART TIMEVALUE TINV TNONCT TODAY + TRANSLATE TRANSTRN TRANWRD TRIGAMMA TRIM TRIMN TRUNC TSO TYPEOF + TZONEID TZONENAME TZONEOFF TZONES2U TZONEU2S UNIFORM UPCASE + URLDECODE URLENCODE USS UUIDGEN VAR VARFMT VARINFMT VARLABEL + VARLEN VARNAME VARNUM VARRAY VARRAYX VARTYPE VERIFY VFORMAT + VFORMATD VFORMATDX VFORMATN VFORMATNX VFORMATW VFORMATWX VFORMATX + VINARRAY VINARRAYX VINFORMAT VINFORMATD VINFORMATDX VINFORMATN + VINFORMATNX VINFORMATW VINFORMATWX VINFORMATX VLABEL VLABELX + VLENGTH VLENGTHX VNAME VNAMEX VTYPE VTYPEX VVALUE VVALUEX WEEK + WEEKDAY WHICHC WHICHN WTO YEAR YIELDP YRDIF YYQ ZIPCITY + ZIPCITYDISTANCE ZIPFIPS ZIPNAME ZIPNAMEL ZIPSTATE + ) + + # from SAS 9.4 Macro Language Reference + # Chapter 12 + SAS_MACRO_STATEMENTS = Set.new %w( + %COPY %DISPLAY %GLOBAL %INPUT %LET %MACRO %PUT %SYMDEL %SYSCALL + %SYSEXEC %SYSLPUT %SYSMACDELETE %SYSMSTORECLEAR %SYSRPUT %WINDOW + %ABORT %DO %TO %UNTIL %WHILE %END %GOTO %IF %THEN %ELSE %LOCAL + %RETURN + %INCLUDE %LIST %RUN + ) + # Omitted: + # %label: Identifies the destination of a %GOTO statement. + # %MEND + + # from SAS 9.4 Macro Language Reference + # Chapter 12 + SAS_MACRO_FUNCTIONS = Set.new %w( + %BQUOTE %NRBQUOTE %EVAL %INDEX %LENGTH %QUOTE %NRQUOTE %SCAN + %QSCAN %STR %NRSTR %SUBSTR %QSUBSTR %SUPERQ %SYMEXIST %SYMGLOBL + %SYMLOCAL %SYSEVALF %SYSFUNC %QSYSFUNC %SYSGET %SYSMACEXEC + %SYSMACEXIST %SYSMEXECDEPTH %SYSMEXECNAME %SYSPROD %UNQUOTE + %UPCASE %QUPCASE + ) + + # from SAS 9.4 Macro Language Reference + # Chapter 12 + SAS_AUTO_MACRO_VARS = Set.new %w( + &SYSADDRBITS &SYSBUFFR &SYSCC &SYSCHARWIDTH &SYSCMD &SYSDATASTEPPHASE &SYSDATE + &SYSDATE9 &SYSDAY &SYSDEVIC &SYSDMG &SYSDSN &SYSENCODING &SYSENDIAN &SYSENV + &SYSERR &SYSERRORTEXT &SYSFILRC &SYSHOSTINFOLONG &SYSHOSTNAME &SYSINDEX + &SYSINFO &SYSJOBID &SYSLAST &SYSLCKRC &SYSLIBRC &SYSLOGAPPLNAME &SYSMACRONAME + &SYSMENV &SYSMSG &SYSNCPU &SYSNOBS &SYSODSESCAPECHAR &SYSODSPATH &SYSPARM + &SYSPBUFF &SYSPRINTTOLIST &SYSPRINTTOLOG &SYSPROCESSID &SYSPROCESSMODE + &SYSPROCESSNAME &SYSPROCNAME &SYSRC &SYSSCP &SYSSCPL &SYSSITE &SYSSIZEOFLONG + &SYSSIZEOFPTR &SYSSIZEOFUNICODE &SYSSTARTID &SYSSTARTNAME &SYSTCPIPHOSTNAME + &SYSTIME &SYSTIMEZONE &SYSTIMEZONEIDENT &SYSTIMEZONEOFFSET &SYSUSERID &SYSVER + &SYSVLONG &SYSVLONG4 &SYSWARNINGTEXT + ) + + # Create a hash with keywords for common PROCs, keyed by PROC name + PROC_KEYWORDS = { + + # from SAS 9.4 SQL Procedure User's Guide + "SQL" => Set.new(%w( + ALTER TABLE CONNECT CREATE INDEX VIEW DELETE DESCRIBE DISCONNECT DROP EXECUTE + INSERT RESET SELECT UPDATE VALIDATE ADD CONSTRAINT FOREIGN KEY PRIMARY + MODIFY LIKE AS ORDER BY USING FROM INTO SET VALUES DISTINCT UNIQUE + WHERE GROUP HAVING LEFT RIGHT INNER JOIN ON + )), + + # from BASE SAS 9.4 Procedures Guide, Fifth Edition + "MEANS" => Set.new(%w( + BY CLASS FREQ ID OUTPUT OUT TYPES VAR WAYS WEIGHT + ATTRIB FORMAT LABEL WHERE + DESCENDING NOTSORTED + NOTHREADS NOTRAP PCTLDEF SUMSIZE THREADS CLASSDATA COMPLETETYPES + EXCLUSIVE MISSING FW MAXDEC NONOBS NOPRINT ORDER FORMATTED + UNFORMATTED PRINT PRINTALLTYPES PRINTIDVARS STACKODSOUTPUT + CHARTYPE DESCENDTYPES IDMIN + ALPHA EXCLNPWGT QMARKERS QMETHOD QNTLDEF VARDEF + CLM CSS CV KURTOSIS KURT LCLM MAX MEAN MIN MODE N + NMISS RANGE SKEWNESS SKEW STDDEV STD STDERR SUM SUMWGT UCLM USS + MEDIAN P50 Q1 P25 Q3 P75 P1 P90 P5 P95 P10 P99 P20 P30 P40 P60 P70 + P80 QRANGE + PROBT PRT T + ASCENDING GROUPINTERNAL MLF PRELOADFMT + MAXID AUTOLABEL AUTONAME KEEPLEN LEVELS NOINHERIT + )), + + # from BASE SAS 9.4 Procedures Guide, Fifth Edition + "DATASETS" => Set.new(%w( + AGE APPEND ATTRIB AUDIT CHANGE CONTENTS COPY DELETE EXCHANGE + EXCLUDE FORMAT IC CREATE REACTIVATE INDEX CENTILES INFORMAT + INITIATE LABEL LOG MODIFY REBUILD RENAME REPAIR RESUME SAVE SELECT + SUSPEND TERMINATE USER_VAR XATTR ADD OPTIONS REMOVE SET + )), + + # from BASE SAS 9.4 Procedures Guide, Fifth Edition + "SORT" => Set.new(%w( + BY DESCENDING KEY ASCENDING ASC DESC DATECOPY FORCE OVERWRITE + PRESORTED SORTSIZE TAGSORT DUPOUT OUT UNIQUEOUT NODUPKEY NOUNIQUEKEY + NOTHREADS THREADS EQUALS NOEQUALS + ATTRIB FORMAT LABEL WHERE + )), + + # from BASE SAS 9.4 Procedures Guide, Fifth Edition + "PRINT" => Set.new(%w( + BY DESCENDING NOTSORTED PAGEBY SUMBY ID STYLE SUM VAR CONTENTS DATA + GRANDTOTAL_LABEL HEADING LABEL SPLIT SUMLABEL NOSUMLABEL + BLANKLINE COUNT DOUBLE N NOOBS OBS ROUND + ROWS UNIFORM WIDTH + ATTRIB FORMAT WHERE + )), + + # from BASE SAS 9.4 Procedures Guide, Fifth Edition + "APPEND" => Set.new(%w( + BASE APPENDVER DATA ENCRYPTKEY FORCE GETSORT NOWARN + ATTRIB FORMAT LABEL WHERE + )), + + # from BASE SAS 9.4 Procedures Guide, Fifth Edition + "TRANSPOSE" => Set.new(%w( + DELIMITER LABEL LET NAME OUT PREFIX SUFFIX BY DESCENDING NOTSORTED + COPY ID IDLABEL VAR INDB + ATTRIB FORMAT WHERE + )), + + # from Base SAS 9.4 Procedures Guide: Statistical Procedures, Fourth Edition + "FREQ" => Set.new(%w( + BY EXACT OUTPUT TABLES TEST WEIGHT + COMPRESS DATA FORMCHAR NLEVELS NOPRINT ORDER PAGE FORMATTED FREQ + INTERNAL + AGREE BARNARD BINOMIAL BIN CHISQ COMOR EQOR ZELEN FISHER JT KAPPA + KENTB TAUB LRCHI MCNEM MEASURES MHCHI OR ODDSRATIO PCHI PCORR RELRISK + RISKDIFF SCORR SMDCR SMDRC STUTC TAUC TREND WTKAP WTKAPPA + OUT AJCHI ALL BDCHI CMH CMH1 CMH2 CMHCOR CMHGA CMHRMS COCHQ CONTGY + CRAMV EQKAP EQWKP GAMMA GS GAILSIMON LAMCR LAMDAS LAMRC LGOR LGRRC1 + LGRRC2 MHOR MHRRC1 MHRRC2 N NMISS PHI PLCORR RDIF1 RDIF2 RISKDIFF1 + RISKDIFF2 RRC1 RELRISK1 RRC2 RELRISK2 RSK1 RISK1 RSK11 RISK11 RSK12 + RISK12 RSK21 RISK21 RSK22 RISK22 TSYMM BOWKER U UCR URC + CELLCHI2 CUMCOL DEVIATION EXPECTED MISSPRINT PEARSONREF PRINTWKTS + SCOROUT SPARSE STDRES TOTPCT + CONTENTS CROSSLIST FORMAT LIST MAXLEVELS NOCOL NOCUM NOFREQ NOPERCENT NOROW NOSPARSE NOWARN PLOTS OUTCUM OUTEXPECT OUTPCT + ZEROS + )), + + # from Base SAS 9.4 Procedures Guide: Statistical Procedures, Fourth Edition + "CORR" => Set.new(%w( + BY FREQ ID PARTIAL VAR WEIGHT WITH + DATA OUTH OUTK OUTP OUTPLC OUTPLS OUTS + EXCLNPWGHT FISHER HOEFFDING KENDALL NOMISS PEARSON POLYCHORIC + POLYSERIAL ALPHA COV CSSCP SINGULAR SSCP VARDEF PLOTS MATRIX SCATTER + BEST NOCORR NOPRINT NOPROB NOSIMPLE RANK + )), + + # from BASE SAS 9.4 Procedures Guide, Fifth Edition + "REPORT" => Set.new(%w( + BREAK BY DESCENDING NOTSORTED COLUMN COMPUTE STYLE LINE ENDCOMP + CALL DEFINE _ROW_ FREQ RBREAK WEIGHT + ATTRIB FORMAT LABEL WHERE + DATA NOALIAS NOCENTER NOCOMPLETECOLS NOCOMPLETEROWS NOTHREADS + NOWINDOWS OUT PCTLDEF THREADS WINDOWS COMPLETECOLS + COMPLETEROWS CONTENTS SPANROWS COMMAND HELP PROMPT + BOX BYPAGENO CENTER COLWIDTH FORMCHAR LS MISSING PANELS PS + PSPACE SHOWALL SPACING WRAP EXCLNPWGT QMARKERS QMETHOD QNTLDEF VARDEF + NAMED NOHEADER SPLIT HEADLINE HEADSKIP LIST NOEXEC OUTREPT PROFILE + REPORT + COLOR DOL DUL OL PAGE SKIP SUMMARIZE SUPPRESS UL + BLINK HIGHLIGHT RVSVIDEO MERGE REPLACE URL URLBP URLP + AFTER BEFORE _PAGE_ LEFT RIGHT CHARACTER LENGTH + EXCLUSIVE MLF ORDER FORMATTED INTERNAL PRELOADFMT + WIDTH + ACROSS ANALYSIS COMPUTED DISPLAY GROUP FLOW ID NOPRINT NOZERO + CSS CV MAX MEAN MIN MODE N NMISS PCTN PCTSUM RANGE STD STDERR SUM + SUMWGT USS VAR + MEDIAN P50 Q1 P25 Q3 P75 P1 P90 P5 P95 P10 P99 P20 P30 P40 P60 P70 + P80 QRANGE + PROBT PRT T + )), + + # from SAS 9.4 Language Interfaces to Metadata, Third Edition + "METALIB" => Set.new(%w( + OMR DBAUTH DBUSER DBPASSWORD EXCLUDE SELECT READ FOLDER FOLDERID + IMPACT_LIMIT NOEXEC PREFIX REPORT UPDATE_RULE DELETE NOADD NODELDUP + NOUPDATE + LIBID LIBRARY LIBURI + TYPE DETAIL SUMMARY + )), + + # from SAS GRAPH 9.4 Reference, Fourth Edition + "GCHART" => Set.new(%w( + DATA ANNOTATE GOUT IMAGEMAP BLOCK HBAR HBAR3D VBAR VBAR3D PIE PIE3D + DONUT STAR ANNO + BY NOTE FORMAT LABEL WHERE + BLOCKMAX CAXIS COUTLINE CTEXT LEGEND NOHEADING NOLEGEND PATTERNID + GROUP MIDPOINT SUBGROUP WOUTLINE DESCRIPTION NAME DISCRETE LEVELS + OLD MISSING HTML_LEGEND HTML URL FREQ G100 SUMVAR TYPE + CAUTOREF CERROR CFRAME CLM CREF FRAME NOFRAME GSPACE IFRAME + IMAGESTYLE TILE FIT LAUTOREF NOSYMBOL SHAPE SPACE + SUBOUTSIDE WAUTOREF WIDTH WREF + ASCENDING AUTOREF CLIPREF DESCENDING FRONTREF GAXIS MAXIS MINOR + NOAXIS NOBASEREF NOZERO RANGE AXIS REF CFREQ CFREQLABEL NONE CPERCENT + CPERCENTLABEL ERRORBAR BARS BOTH TOP FREQLABEL INSIDE MEAN MEANLABEL + NOSTATS OUTSIDE PERCENT PERCENTLABEL PERCENTSUM SUM + CFILL DETAIL_RADIUS EXPLODE FILL SOLID X INVISIBLE + RADIUS DETAIL_THRESHOLD DETAIL_PERCENT DETAIL_SLICE + DETAIL_VALUE DONUTPCT ACROSS DOWN NOGROUPHEADING + MATCHCOLOR OTHERCOLOR OTHERLABEL ARROW PLABEL PPERCENT SLICE + VALUE + ANGLE CLOCKWISE JSTYLE + NOCONNECT STARMAX STARMIN + )), + + # from SAS GRAPH 9.4 Reference, Fourth Edition + "GPLOT" => Set.new(%w( + DATA ANNOTATE GOUT IMAGEMAP UNIFORM BUBBLE BUBBLE2 PLOT PLOT2 + BCOLOR BFILL BFONT BLABEL BSCALE AREA RADIUS BSIZE DESCRIPTION NAME + AUTOHREF CAUTOHREF CHREF HAXIS HMINOR HREF HREVERSE HZERO LAUTOHREF + LHREF WAUTOHREF WHREF HTML URL + CAXIS CFRAME CTEXT DATAORDER FRAME NOFRAME FRONTREF GRID IFRAME + IMAGESTYLE TILE FIT NOAXIS + AUTOVREF CAUTOVREF CVREF LAUTOVREF LVREF VAXIS VMINOR VREF VREVERSE + VZERO WAUTOVREF WVREF + CBASELINE COUTLINE + AREAS LEGEND NOLASTAREA NOLEGEND OVERLAY REGEQN SKIPMISS + )), + + # from SAS/STAT 15.1 User's Guide + "REG" => Set.new(%w( + MODEL BY FREQ ID VAR WEIGHT ADD CODE DELETE MTEST OUTPUT PAINT + PLOT PRINT REFIT RESTRICT REWEIGHT STORE TEST + )), + + # from ODS Graphics: Procedures Guide, Sixth Edition + "SGPLOT" => Set.new(%w( + STYLEATTRS BAND X Y UPPER LOWER BLOCK BUBBLE DENSITY DOT DROPLINE + ELLIPSE ELLIPSEPARM FRINGE GRADLEGEND HBAR HBARBASIC HBARPARM + HBOX HEATMAP HEATMAPPARM HIGHLOW HISTOGRAM HLINE INSET KEYLEGEND + LINEPARM LOESS NEEDLE PBSPLINE POLYGON REFLINE REG SCATTER SERIES + SPLINE STEP SYMBOLCHAR SYMBOLIMAGE TEXT VBAR VBARBASIC VBARPARM + VBOX VECTOR VLINE WATERFALL XAXIS X2AXIS XAXISTABLE YAXIS Y2AXIS + YAXISTABLE + )), + }.freeze + + # from SAS Procedures by Name + # http://support.sas.com/documentation/cdl/en/allprodsproc/68038/HTML/default/viewer.htm#procedures.htm + SAS_PROC_NAMES = Set.new %w( + ACCESS ACECLUS ADAPTIVEREG ALLELE ANOM ANOVA APPEND APPSRV ARIMA + AUTHLIB AUTOREG BCHOICE BOM BOXPLOT BTL BUILD CALENDAR CALIS CALLRFC + CANCORR CANDISC CAPABILITY CASECONTROL CATALOG CATMOD CDISC + CHART CIMPORT CLP CLUSTER COMPARE COMPILE COMPUTAB CONTENTS CONVERT + COPULA COPY CORR CORRESP COUNTREG CPM CPORT CUSUM CV2VIEW DATEKEYS + DATASETS DATASOURCE DB2EXT DB2UTIL DBCSTAB DBF DBLOAD DELETE DIF + DISCRIM DISPLAY DISTANCE DMSRVADM DMSRVDATASVC DMSRVPROCESSSVC + DOCUMENT DOWNLOAD DQLOCLST DQMATCH DQSCHEME DS2 DTREE ENTROPY ESM + EXPAND EXPLODE EXPORT FACTEX FACTOR FAMILY FASTCLUS FCMP FEDSQL FMM + FONTREG FORECAST FORMAT FORMS FREQ FSBROWSE FSEDIT FSLETTER FSLIST + FSVIEW G3D G3GRID GA GAM GAMPL GANNO GANTT GAREABAR GBARLINE GCHART + GCONTOUR GDEVICE GEE GENESELECT GENMOD GEOCODE GFONT GINSIDE GIS GKPI + GLIMMIX GLM GLMMOD GLMPOWER GLMSELECT GMAP GOPTIONS GPLOT GPROJECT + GRADAR GREDUCE GREMOVE GREPLAY GROOVY GSLIDE GTILE HADOOP HAPLOTYPE + HDMD HPBIN HPCANDISC HPCDM HPCOPULA HPCORR HPCOUNTREG HPDMDB HPDS2 + HPFMM HPGENSELECT HPIMPUTE HPLMIXED HPLOGISTIC HPMIXED HPNLMOD + HPPANEL HPPLS HPPRINCOMP HPQUANTSELECT HPQLIM HPREG HPSAMPLE + HPSEVERITY HPSPLIT HPSUMMARY HTSNP HTTP ICLIFETEST ICPHREG IML IMPORT + IMSTAT IMXFER INBREED INFOMAPS INTPOINT IOMOPERATE IRT ISHIKAWA ITEMS + JAVAINFO JSON KDE KRIGE2D LASR LATTICE LIFEREG LIFETEST LOAN + LOCALEDATA LOESS LOGISTIC LP LUA MACONTROL MAPIMPORT MCMC MDC MDDB + MDS MEANS METADATA METALIB METAOPERATE MI MIANALYZE MIGRATE MIXED + MODECLUS MODEL MSCHART MULTTEST MVPDIAGNOSE MVPMODEL MVPMONITOR + NESTED NETDRAW NETFLOW NLIN NLMIXED NLP NPAR1WAY ODSLIST ODSTABLE + ODSTEXT OLAP OLAPCONTENTS OLAPOPERATE OPERATE OPTEX OPTGRAPH OPTIONS + OPTLOAD OPTLP OPTLSO OPTMILP OPTMODEL OPTNET OPTQP OPTSAVE ORTHOREG + PANEL PARETO PDLREG PDS PDSCOPY PHREG PLAN PLM PLOT PLS PM PMENU + POWER PRESENV PRINCOMP PRINQUAL PRINT PRINTTO PROBIT PROTO PRTDEF + PRTEXP PSMOOTH PWENCODE QDEVICE QLIM QUANTLIFE QUANTREG QUANTSELECT + QUEST RANK RAREEVENTS RDC RDPOOL RDSEC RECOMMEND REG REGISTRY RELEASE + RELIABILITY REPORT RISK ROBUSTREG RSREG SCAPROC SCORE SEQDESIGN + SEQTEST SERVER SEVERITY SGDESIGN SGPANEL SGPLOT SGRENDER SGSCATTER + SHEWHART SIM2D SIMILARITY SIMLIN SIMNORMAL SOAP SORT SOURCE SPECTRA + SPP SQL SQOOP SSM STANDARD STATESPACE STDIZE STDRATE STEPDISC STP + STREAM SUMMARY SURVEYFREQ SURVEYIMPUTE SURVEYLOGISTIC SURVEYMEANS + SURVEYPHREG SURVEYREG SURVEYSELECT SYSLIN TABULATE TAPECOPY TAPELABEL + TEMPLATE TIMEDATA TIMEID TIMEPLOT TIMESERIES TPSPLINE TRANSPOSE + TRANSREG TRANTAB TREE TSCSREG TTEST UCM UNIVARIATE UPLOAD VARCLUS + VARCOMP VARIOGRAM VARMAX VASMP X11 X12 X13 XSL + ) + end + end +end From 75e1cd03b793ef9d24887539e5adaccef18f100d Mon Sep 17 00:00:00 2001 From: jneen Date: Tue, 23 Jun 2026 09:51:36 -0400 Subject: [PATCH 141/154] use the keywords api more for SassCommon --- lib/rouge/lexers/sass/common.rb | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/lib/rouge/lexers/sass/common.rb b/lib/rouge/lexers/sass/common.rb index 6cb75b248c..bc4db6453b 100644 --- a/lib/rouge/lexers/sass/common.rb +++ b/lib/rouge/lexers/sass/common.rb @@ -66,14 +66,10 @@ class SassCommon < RegexLexer rule %r(//[^\n]*), Comment::Single # identifiers - rule(id) do |m| - if CSS::BUILTINS.include? m[0] - token Name::Builtin - elsif CSS::COLORS.include? m[0] - token Name::Constant - else - token Name - end + keywords id do + rule CSS::BUILTINS, Name::Builtin + rule CSS::COLORS, Name::Constant + default Name end end From 045884ed6d1e711488e92a71d405d2417c37267b Mon Sep 17 00:00:00 2001 From: jneen Date: Tue, 23 Jun 2026 09:56:41 -0400 Subject: [PATCH 142/154] use the keywords api for scala --- .rubocop_todo.yml | 1 - lib/rouge/lexers/scala.rb | 8 ++++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index 50cc8e5351..f5ab6f24b7 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -24,7 +24,6 @@ Naming/VariableName: # Offense count: 113 Rouge/NoBuildingAlternationPatternInRegexp: Exclude: - - 'lib/rouge/lexers/scala.rb' - 'lib/rouge/lexers/sml.rb' - 'lib/rouge/lexers/stata.rb' - 'lib/rouge/lexers/vala.rb' diff --git a/lib/rouge/lexers/scala.rb b/lib/rouge/lexers/scala.rb index 0758699b60..55152ab7c3 100644 --- a/lib/rouge/lexers/scala.rb +++ b/lib/rouge/lexers/scala.rb @@ -32,7 +32,7 @@ class Scala < RegexLexer idrest = %r(#{letter}(?:#{letter}|#{digits})*(?:(?<=_)#{op}+)?)x - keywords = %w( + KEYWORDS = Set.new %w( abstract case catch def do else extends final finally for forSome if implicit lazy match new override private protected requires return sealed super this throw try val var while with yield @@ -77,10 +77,14 @@ class Scala < RegexLexer groups Operator, Name::Property end + keywords %r/[a-z]+/ do + rule KEYWORDS, Keyword + end + rule %r( - (#{keywords.join("|")})\b| (<[%:-]|=>|>:|[#=@_\u21D2\u2190])(\b|(?=\s)|$) )x, Keyword + rule %r/:(?!#{op})/, Keyword, :type rule %r/(true|false|null)\b/, Keyword::Constant rule %r/(import|package)(\s+)/ do From 7090495361377cb6f369ad90a5adc71418168f62 Mon Sep 17 00:00:00 2001 From: jneen Date: Tue, 23 Jun 2026 09:59:04 -0400 Subject: [PATCH 143/154] use the keywords api for scheme --- lib/rouge/lexers/scheme.rb | 101 +++++++++++++++++-------------------- 1 file changed, 45 insertions(+), 56 deletions(-) diff --git a/lib/rouge/lexers/scheme.rb b/lib/rouge/lexers/scheme.rb index 4386ffc7d1..c161091219 100644 --- a/lib/rouge/lexers/scheme.rb +++ b/lib/rouge/lexers/scheme.rb @@ -11,51 +11,47 @@ class Scheme < RegexLexer filenames '*.scm', '*.ss' mimetypes 'text/x-scheme', 'application/x-scheme' - def self.keywords - @keywords ||= Set.new %w( - lambda define if else cond and or case let let* letrec begin - do delay set! => quote quasiquote unquote unquote-splicing - define-syntax let-syntax letrec-syntax syntax-rules - ) - end - - def self.builtins - @builtins ||= Set.new %w( - * + - / < <= = > >= abs acos angle append apply asin - assoc assq assv atan boolean? caaaar caaadr caaar caadar - caaddr caadr caar cadaar cadadr cadar caddar cadddr caddr - cadr call-with-current-continuation call-with-input-file - call-with-output-file call-with-values call/cc car cdaaar cdaadr - cdaar cdadar cdaddr cdadr cdar cddaar cddadr cddar cdddar cddddr - cdddr cddr cdr ceiling char->integer char-alphabetic? char-ci<=? - char-ci=? char-ci>? char-downcase - char-lower-case? char-numeric? char-ready? char-upcase - char-upper-case? char-whitespace? char<=? char=? - char>? char? close-input-port close-output-port complex? cons - cos current-input-port current-output-port denominator - display dynamic-wind eof-object? eq? equal? eqv? eval - even? exact->inexact exact? exp expt floor for-each force gcd - imag-part inexact->exact inexact? input-port? integer->char - integer? interaction-environment lcm length list list->string - list->vector list-ref list-tail list? load log magnitude - make-polar make-rectangular make-string make-vector map - max member memq memv min modulo negative? newline not - null-environment null? number->string number? numerator odd? - open-input-file open-output-file output-port? pair? peek-char - port? positive? procedure? quotient rational? rationalize - read read-char real-part real? remainder reverse round - scheme-report-environment set-car! set-cdr! sin sqrt string - string->list string->number string->symbol string-append - string-ci<=? string-ci=? string-ci>? - string-copy string-fill! string-length string-ref - string-set! string<=? string=? - string>? string? substring symbol->string symbol? - tan transcript-off transcript-on truncate values vector - vector->list vector-fill! vector-length vector-ref - vector-set! vector? with-input-from-file with-output-to-file - write write-char zero? - ) - end + KEYWORDS = Set.new %w( + lambda define if else cond and or case let let* letrec begin + do delay set! => quote quasiquote unquote unquote-splicing + define-syntax let-syntax letrec-syntax syntax-rules + ) + + BUILTINS = Set.new %w( + * + - / < <= = > >= abs acos angle append apply asin + assoc assq assv atan boolean? caaaar caaadr caaar caadar + caaddr caadr caar cadaar cadadr cadar caddar cadddr caddr + cadr call-with-current-continuation call-with-input-file + call-with-output-file call-with-values call/cc car cdaaar cdaadr + cdaar cdadar cdaddr cdadr cdar cddaar cddadr cddar cdddar cddddr + cdddr cddr cdr ceiling char->integer char-alphabetic? char-ci<=? + char-ci=? char-ci>? char-downcase + char-lower-case? char-numeric? char-ready? char-upcase + char-upper-case? char-whitespace? char<=? char=? + char>? char? close-input-port close-output-port complex? cons + cos current-input-port current-output-port denominator + display dynamic-wind eof-object? eq? equal? eqv? eval + even? exact->inexact exact? exp expt floor for-each force gcd + imag-part inexact->exact inexact? input-port? integer->char + integer? interaction-environment lcm length list list->string + list->vector list-ref list-tail list? load log magnitude + make-polar make-rectangular make-string make-vector map + max member memq memv min modulo negative? newline not + null-environment null? number->string number? numerator odd? + open-input-file open-output-file output-port? pair? peek-char + port? positive? procedure? quotient rational? rationalize + read read-char real-part real? remainder reverse round + scheme-report-environment set-car! set-cdr! sin sqrt string + string->list string->number string->symbol string-append + string-ci<=? string-ci=? string-ci>? + string-copy string-fill! string-length string-ref + string-set! string<=? string=? + string>? string? substring symbol->string symbol? + tan transcript-off transcript-on truncate values vector + vector->list vector-fill! vector-length vector-ref + vector-set! vector? with-input-from-file with-output-to-file + write write-char zero? + ) id = /[a-z0-9!$\%&*+,\/:<=>?@^_~|-]+/i @@ -93,21 +89,14 @@ def self.builtins end state :command do - rule id do |m| - if self.class.keywords.include? m[0] - token Keyword - elsif self.class.builtins.include? m[0] - token Name::Builtin - else - token Name::Function - end - - pop! + keywords id do + rule KEYWORDS, Keyword, :pop! + rule BUILTINS, Name::Builtin, :pop! + default Name::Function, :pop! end rule(//) { pop! } end - end end end From b8ca3e04e841d5f549b9c2740b78165eaed2e2d5 Mon Sep 17 00:00:00 2001 From: jneen Date: Tue, 23 Jun 2026 23:01:13 -0400 Subject: [PATCH 144/154] use the keywords api for sieve --- lib/rouge/lexers/sieve.rb | 94 +++++++++++++++++---------------------- 1 file changed, 42 insertions(+), 52 deletions(-) diff --git a/lib/rouge/lexers/sieve.rb b/lib/rouge/lexers/sieve.rb index 08d60327e5..c67e01728e 100644 --- a/lib/rouge/lexers/sieve.rb +++ b/lib/rouge/lexers/sieve.rb @@ -13,48 +13,42 @@ class Sieve < RegexLexer id = /:?[a-zA-Z_][a-zA-Z0-9_]*/ # control commands (rfc5228 § 3) - def self.controls - @controls ||= %w(if elsif else require stop) - end + CONTROLS = Set.new %w(if elsif else require stop) - def self.actions - @actions ||= Set.new( - # action commands (rfc5228 § 2.9) - %w(keep fileinto redirect discard) + - # Editheader Extension (rfc5293) - %w(addheader deleteheader) + - # Reject and Extended Reject Extensions (rfc5429) - %w(reject ereject) + - # Extension for Notifications (rfc5435) - %w(notify) + - # Imap4flags Extension (rfc5232) - %w(setflag addflag removeflag) + - # Vacation Extension (rfc5230) - %w(vacation) + - # MIME Part Tests, Iteration, Extraction, Replacement, and Enclosure (rfc5703) - %w(replace enclose extracttext) - ) - end + ACTIONS = Set.new( + # action commands (rfc5228 § 2.9) + %w(keep fileinto redirect discard) + + # Editheader Extension (rfc5293) + %w(addheader deleteheader) + + # Reject and Extended Reject Extensions (rfc5429) + %w(reject ereject) + + # Extension for Notifications (rfc5435) + %w(notify) + + # Imap4flags Extension (rfc5232) + %w(setflag addflag removeflag) + + # Vacation Extension (rfc5230) + %w(vacation) + + # MIME Part Tests, Iteration, Extraction, Replacement, and Enclosure (rfc5703) + %w(replace enclose extracttext) + ) - def self.tests - @tests ||= Set.new( - # test commands (rfc5228 § 5) - %w(address allof anyof exists false header not size true) + - # Body Extension (rfc5173) - %w(body) + - # Imap4flags Extension (rfc5232) - %w(hasflag) + - # Spamtest and Virustest Extensions (rfc5235) - %w(spamtest virustest) + - # Date and Index Extensions (rfc5260) - %w(date currentdate) + - # Extension for Notifications (rfc5435) - %w(valid_notify_method notify_method_capability) + - # Extensions for Checking Mailbox Status and Accessing Mailbox - # Metadata (rfc5490) - %w(mailboxexists metadata metadataexists servermetadata servermetadataexists) - ) - end + TESTS = Set.new( + # test commands (rfc5228 § 5) + %w(address allof anyof exists false header not size true) + + # Body Extension (rfc5173) + %w(body) + + # Imap4flags Extension (rfc5232) + %w(hasflag) + + # Spamtest and Virustest Extensions (rfc5235) + %w(spamtest virustest) + + # Date and Index Extensions (rfc5260) + %w(date currentdate) + + # Extension for Notifications (rfc5435) + %w(valid_notify_method notify_method_capability) + + # Extensions for Checking Mailbox Status and Accessing Mailbox + # Metadata (rfc5490) + %w(mailboxexists metadata metadataexists servermetadata servermetadataexists) + ) state :comments_and_whitespace do rule %r/\s+/, Text @@ -75,18 +69,14 @@ def self.tests rule %r/[\[\](),;{}]/, Punctuation - rule id do |m| - if self.class.controls.include? m[0] - token Keyword - elsif self.class.tests.include? m[0] - token Name::Variable - elsif self.class.actions.include? m[0] - token Name::Function - elsif m[0] =~ /^:/ # tags like :contains, :matches etc. - token Operator - else - token Name::Other - end + # tags like :contains, :matches, etc. + rule %r/:[a-z][a-z0-9_]*/, Operator + + keywords id do + rule CONTROLS, Keyword + rule TESTS, Name::Variable + rule ACTIONS, Name::Function + default Name::Other end rule %r/"/, Str::Double, :string From 06b097748fb6e74eda6371561ec9a46d293c0399 Mon Sep 17 00:00:00 2001 From: jneen Date: Wed, 24 Jun 2026 13:29:21 -0400 Subject: [PATCH 145/154] use the keywords api for smarty --- lib/rouge/lexers/smarty.rb | 40 ++++++++++++++++---------------------- 1 file changed, 17 insertions(+), 23 deletions(-) diff --git a/lib/rouge/lexers/smarty.rb b/lib/rouge/lexers/smarty.rb index 9870e8b1f5..2e5c37ed62 100644 --- a/lib/rouge/lexers/smarty.rb +++ b/lib/rouge/lexers/smarty.rb @@ -11,23 +11,20 @@ class Smarty < TemplateLexer filenames '*.tpl', '*.smarty' mimetypes 'application/x-smarty', 'text/x-smarty' - def self.builtins - @builtins ||= %w( - append assign block call capture config_load debug extends - for foreach foreachelse break continue function if elseif - else include include_php insert ldelim rdelim literal nocache - php section sectionelse setfilter strip while - counter cycle eval fetch html_checkboxes html_image html_options - html_radios html_select_date html_select_time html_table - mailto math textformat - capitalize cat count_characters count_paragraphs - count_sentences count_words date_format default escape - from_charset indent lower nl2br regex_replace replace spacify - string_format strip strip_tags to_charset truncate unescape - upper wordwrap - ) - end - + BUILTINS = Set.new %w( + append assign block call capture config_load debug extends + for foreach foreachelse break continue function if elseif + else include include_php insert ldelim rdelim literal nocache + php section sectionelse setfilter strip while + counter cycle eval fetch html_checkboxes html_image html_options + html_radios html_select_date html_select_time html_table + mailto math textformat + capitalize cat count_characters count_paragraphs + count_sentences count_words date_format default escape + from_charset indent lower nl2br regex_replace replace spacify + string_format strip_tags to_charset truncate unescape + upper wordwrap + ) state :root do rule(/\{\s+/) { delegate parent } @@ -68,12 +65,9 @@ def self.builtins rule %r/"(\\.|.)*?"/, Str::Double rule %r/'(\\.|.)*?'/, Str::Single - rule %r/([a-zA-Z_]\w*)/ do |m| - if self.class.builtins.include? m[0] - token Name::Builtin - else - token Name::Attribute - end + keywords %r/([a-zA-Z_]\w*)/ do + rule BUILTINS, Name::Builtin + default Name::Attribute end end end From 975f1d0446c719e0fc352a3dd5edcf0204b486dd Mon Sep 17 00:00:00 2001 From: jneen Date: Wed, 24 Jun 2026 13:29:36 -0400 Subject: [PATCH 146/154] use the keywords api for SML --- .rubocop_todo.yml | 1 - lib/rouge/lexers/sml.rb | 39 ++++++++++++++------------------------- 2 files changed, 14 insertions(+), 26 deletions(-) diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index f5ab6f24b7..d98c65e617 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -24,7 +24,6 @@ Naming/VariableName: # Offense count: 113 Rouge/NoBuildingAlternationPatternInRegexp: Exclude: - - 'lib/rouge/lexers/sml.rb' - 'lib/rouge/lexers/stata.rb' - 'lib/rouge/lexers/vala.rb' - 'lib/rouge/lexers/wollok.rb' diff --git a/lib/rouge/lexers/sml.rb b/lib/rouge/lexers/sml.rb index 08ed1bf200..024bdf07d9 100644 --- a/lib/rouge/lexers/sml.rb +++ b/lib/rouge/lexers/sml.rb @@ -12,19 +12,15 @@ class SML < RegexLexer mimetypes 'text/x-standardml', 'application/x-standardml' - def self.keywords - @keywords ||= Set.new %w( - abstype and andalso as case datatype do else end exception - fn fun handle if in infix infixr let local nonfix of op open - orelse raise rec then type val with withtype while - eqtype functor include sharing sig signature struct structure - where - ) - end + KEYWORDS = Set.new %w( + abstype and andalso as case datatype do else end exception + fn fun handle if in infix infixr let local nonfix of op open + orelse raise rec then type val with withtype while + eqtype functor include sharing sig signature struct structure + where + ) - def self.symbolic_reserved - @symbolic_reserved ||= Set.new %w(: | = => -> # :>) - end + SYMBOLIC_RESERVED = Set.new %w(: | = => -> # :>) id = /[\w']+/i symbol = %r([!%&$#/:<=>?@\\~`^|*+-]+) @@ -52,25 +48,17 @@ def self.symbolic_reserved end def token_for_id_with_dot(id) - if self.class.keywords.include? id + if KEYWORDS.include? id Error else Name::Namespace end end - def token_for_final_id(id) - if self.class.keywords.include? id or self.class.symbolic_reserved.include? id - Error - else - Name - end - end - def token_for_id(id) - if self.class.keywords.include? id + if KEYWORDS.include? id Keyword::Reserved - elsif self.class.symbolic_reserved.include? id + elsif SYMBOLIC_RESERVED.include? id Punctuation else Name @@ -188,8 +176,9 @@ def token_for_id(id) end state :breakout do - rule %r/(?=\b(#{SML.keywords.to_a.join('|')})\b(?!'))/ do - pop! + keywords %r/(?=(#{id}))/ do + group 1 + rule KEYWORDS, Keyword, :pop! end end From b48488bb3c426c10ebcdc8d0d9caa9c6ade0fdb2 Mon Sep 17 00:00:00 2001 From: jneen Date: Tue, 23 Jun 2026 23:57:38 -0400 Subject: [PATCH 147/154] use the keywords api for SQF --- lib/rouge/lexers/sparql.rb | 53 ++++++++++----------- lib/rouge/lexers/sqf.rb | 98 +++++++++++++++----------------------- 2 files changed, 63 insertions(+), 88 deletions(-) diff --git a/lib/rouge/lexers/sparql.rb b/lib/rouge/lexers/sparql.rb index c3b9fb2d5a..9a36317a73 100644 --- a/lib/rouge/lexers/sparql.rb +++ b/lib/rouge/lexers/sparql.rb @@ -10,27 +10,23 @@ class SPARQL < RegexLexer filenames '*.rq' mimetypes 'application/sparql-query' - def self.builtins - @builtins = Set.new %w[ - ABS AVG BNODE BOUND CEIL COALESCE CONCAT CONTAINS COUNT DATATYPE DAY - ENCODE_FOR_URI FLOOR GROUP_CONCAT HOURS IF IRI isBLANK isIRI - isLITERAL isNUMERIC isURI LANG LANGMATCHES LCASE MAX MD5 MIN MINUTES - MONTH NOW RAND REGEX REPLACE ROUND SAMETERM SAMPLE SECONDS SEPARATOR - SHA1 SHA256 SHA384 SHA512 STR STRAFTER STRBEFORE STRDT STRENDS - STRLANG STRLEN STRSTARTS STRUUID SUBSTR SUM TIMEZONE TZ UCASE URI - UUID YEAR - ] - end - - def self.keywords - @keywords = Set.new %w[ - ADD ALL AS ASC ASK BASE BIND BINDINGS BY CLEAR CONSTRUCT COPY CREATE - DATA DEFAULT DELETE DESC DESCRIBE DISTINCT DROP EXISTS FILTER FROM - GRAPH GROUP HAVING IN INSERT LIMIT LOAD MINUS MOVE NAMED NOT - OFFSET OPTIONAL ORDER PREFIX SELECT REDUCED SERVICE SILENT TO UNDEF - UNION USING VALUES WHERE WITH - ] - end + BUILTINS = Set.new %w[ + ABS AVG BNODE BOUND CEIL COALESCE CONCAT CONTAINS COUNT DATATYPE DAY + ENCODE_FOR_URI FLOOR GROUP_CONCAT HOURS IF IRI isBLANK isIRI + isLITERAL isNUMERIC isURI LANG LANGMATCHES LCASE MAX MD5 MIN MINUTES + MONTH NOW RAND REGEX REPLACE ROUND SAMETERM SAMPLE SECONDS SEPARATOR + SHA1 SHA256 SHA384 SHA512 STR STRAFTER STRBEFORE STRDT STRENDS + STRLANG STRLEN STRSTARTS STRUUID SUBSTR SUM TIMEZONE TZ UCASE URI + UUID YEAR + ] + + KEYWORDS = Set.new %w[ + ADD ALL AS ASC ASK BASE BIND BINDINGS BY CLEAR CONSTRUCT COPY CREATE + DATA DEFAULT DELETE DESC DESCRIBE DISTINCT DROP EXISTS FILTER FROM + GRAPH GROUP HAVING IN INSERT LIMIT LOAD MINUS MOVE NAMED NOT + OFFSET OPTIONAL ORDER PREFIX SELECT REDUCED SERVICE SILENT TO UNDEF + UNION USING VALUES WHERE WITH + ] state :root do rule %r(\s+)m, Text::Whitespace @@ -51,14 +47,13 @@ def self.keywords rule %r(true|false)i, Keyword::Constant rule %r/a\b/, Keyword - rule %r([A-Z][[:word:]]+\b)i do |m| - if self.class.builtins.include? m[0].upcase - token Name::Builtin - elsif self.class.keywords.include? m[0].upcase - token Keyword - else - token Error - end + keywords %r([A-Z][[:word:]]+\b)i do + transform(&:upcase) + + rule BUILTINS, Name::Builtin + rule KEYWORDS, Keyword + + default Error end rule %r([+\-]?(?:\d+\.\d*|\.\d+)(?:[e][+\-]?[0-9]+)?)i, Num::Float diff --git a/lib/rouge/lexers/sqf.rb b/lib/rouge/lexers/sqf.rb index d4e910a5f9..1ba16cc26e 100644 --- a/lib/rouge/lexers/sqf.rb +++ b/lib/rouge/lexers/sqf.rb @@ -10,49 +10,37 @@ class SQF < RegexLexer title "SQF" desc "Status Quo Function, a Real Virtuality engine scripting language" - def self.wordoperators - @wordoperators ||= Set.new %w( - and or not - ) - end + WORDOPERATORS = Set.new %w( + and or not + ) - def self.initializers - @initializers ||= Set.new %w( - private param params - ) - end + INITIALIZERS = Set.new %w( + private param params + ) - def self.controlflow - @controlflow ||= Set.new %w( - if then else exitwith switch do case default while for from to step - foreach - ) - end + CONTROLFLOW = Set.new %w( + if then else exitwith switch do case default while for from to step + foreach + ) - def self.constants - @constants ||= Set.new %w( - true false player confignull controlnull displaynull grpnull - locationnull netobjnull objnull scriptnull tasknull teammembernull - ) - end + CONSTANTS = Set.new %w( + true false player confignull controlnull displaynull grpnull + locationnull netobjnull objnull scriptnull tasknull teammembernull + ) - def self.namespaces - @namespaces ||= Set.new %w( - currentnamespace missionnamespace parsingnamespace profilenamespace - uinamespace - ) - end + NAMESPACES = Set.new %w( + currentnamespace missionnamespace parsingnamespace profilenamespace + uinamespace + ) - def self.diag_commands - @diag_commands ||= Set.new %w( - diag_activemissionfsms diag_activesqfscripts diag_activesqsscripts - diag_activescripts diag_captureframe diag_captureframetofile - diag_captureslowframe diag_codeperformance diag_drawmode diag_enable - diag_enabled diag_fps diag_fpsmin diag_frameno diag_lightnewload - diag_list diag_log diag_logslowframe diag_mergeconfigfile - diag_recordturretlimits diag_setlightnew diag_ticktime diag_toggle - ) - end + DIAG_COMMANDS = Set.new %w( + diag_activemissionfsms diag_activesqfscripts diag_activesqsscripts + diag_activescripts diag_captureframe diag_captureframetofile + diag_captureslowframe diag_codeperformance diag_drawmode diag_enable + diag_enabled diag_fps diag_fpsmin diag_frameno diag_lightnewload + diag_list diag_log diag_logslowframe diag_mergeconfigfile + diag_recordturretlimits diag_setlightnew diag_ticktime diag_toggle + ) lazy do require_relative 'sqf/keywords.rb' @@ -79,28 +67,20 @@ def self.diag_commands rule %r"[\!\%\&\*\+\-\/\<\=\>\^\|\#]", Operator rule %r"[\(\)\{\}\[\]\,\:\;]", Punctuation + rule %r/_[a-z0-9]+/i, Name::Variable + # Identifiers (variables and functions) - rule %r"[a-zA-Z0-9_]+" do |m| - name = m[0].downcase - if self.class.wordoperators.include? name - token Operator::Word - elsif self.class.initializers.include? name - token Keyword::Declaration - elsif self.class.controlflow.include? name - token Keyword::Reserved - elsif self.class.constants.include? name - token Keyword::Constant - elsif self.class.namespaces.include? name - token Keyword::Namespace - elsif self.class.diag_commands.include? name - token Name::Function - elsif COMMANDS.include? name - token Name::Function - elsif %r"_.+" =~ name - token Name::Variable - else - token Name::Variable::Global - end + keywords %r"[a-z0-9_]+"i do + transform(&:downcase) + + rule WORDOPERATORS, Operator::Word + rule INITIALIZERS, Keyword::Declaration + rule CONTROLFLOW, Keyword::Reserved + rule CONSTANTS, Keyword::Constant + rule NAMESPACES, Keyword::Namespace + rule DIAG_COMMANDS, Name::Function + rule COMMANDS, Name::Function + default Name end end end From ad9f5c006161c92703b42921bcfa5effbeceac26 Mon Sep 17 00:00:00 2001 From: jneen Date: Wed, 24 Jun 2026 13:26:01 -0400 Subject: [PATCH 148/154] use the keywords api (mostly) for stan --- lib/rouge/lexers/stan.rb | 339 ++---------------------------- lib/rouge/lexers/stan/keywords.rb | 281 +++++++++++++++++++++++++ 2 files changed, 303 insertions(+), 317 deletions(-) create mode 100644 lib/rouge/lexers/stan/keywords.rb diff --git a/lib/rouge/lexers/stan.rb b/lib/rouge/lexers/stan.rb index 8724306f88..c175b59cb8 100644 --- a/lib/rouge/lexers/stan.rb +++ b/lib/rouge/lexers/stan.rb @@ -9,331 +9,43 @@ class Stan < RegexLexer tag 'stan' filenames '*.stan', '*.stanfunctions' + lazy { require_relative 'stan/keywords' } + # optional comment or whitespace WS = %r((?:\s|//.*?\n|/[*].*?[*]/)+) ID = /[a-zA-Z_][a-zA-Z0-9_]*/ RT = /(?:(?:[a-z_]\s*(?:\[[0-9, ]\])?)\s+)*/ - OP = Regexp.new([ + OP = %r( # Assigment operators - "=", + = # Comparison operators - "<", "<=", ">", ">=", "==", "!=", + | < | <= | > | >= | == | != # Boolean operators - "!", "&&", "\\|\\|", + | ! | [&][&] | [|][|] # Real-valued arithmetic operators - "\\+", "-", "\\*", "/", "\\^", + | [-+*/^] # Transposition operator - "'", + | ['] # Elementwise functions - "\\.\\+", "\\.-", "\\.\\*", "\\./", "\\.\\^", + | [.][-+*/^] # Matrix division operators - "\\\\", + | \\ # Compound assigment operators - "\\+=", "-=", "\\*=", "/=", "\\.\\*=", "\\./=", + | (?:[-+*/] | [.][*/])= # Sampling - "~", + | [~] # Conditional operator - "\\?", ":" - ].join("|")) - - def self.keywords - @keywords ||= Set.new %w( - if else while for break continue print reject return - ) - end - - def self.types - @types ||= Set.new %w( - int real vector ordered positive_ordered simplex unit_vector - row_vector matrix cholesky_factor_corr cholesky_factor_cov corr_matrix - cov_matrix data void complex array - ) - end - - def self.reserved - @reserved ||= Set.new [ - # Reserved words from Stan language - "for", "in", "while", "repeat", "until", "if", "then", "else", "true", - "false", "target", "functions", "model", "data", "parameters", - "quantities", "transformed", "generated", - - # Reserved names from Stan implementation - "var", "fvar", "STAN_MAJOR", "STAN_MINOR", "STAN_PATCH", - "STAN_MATH_MAJOR", "STAN_MATH_MINOR", "STAN_MATH_PATCH", - - # Reserved names from C++ - "alignas", "alignof", "and", "and_eq", "asm", "auto", "bitand", - "bitor", "bool", "break", "case", "catch", "char", "char16_t", - "char32_t", "class", "compl", "const", "constexpr", "const_cast", - "continue", "decltype", "default", "delete", "do", "double", - "dynamic_cast", "enum", "explicit", "export", "extern", "float", "friend", "goto", "inline", "int", - "long", "mutable", "namespace", "new", "noexcept", "not", "not_eq", - "nullptr", "operator", "or", "or_eq", "private", "protected", - "public", "register", "reinterpret_cast", "return", "short", "signed", - "sizeof", "static", "static_assert", "static_cast", "struct", - "switch", "template", "this", "thread_local", "throw", "try", - "typedef", "typeid", "typename", "union", "unsigned", "using", - "virtual", "void", "volatile", "wchar_t", "xor", "xor_eq" - ] - end - - def self.builtin_functions - @builtin_functions ||= Set.new [ - # Integer-Valued Basic Functions - - ## Absolute functions - "abs", "int_step", - - ## Bound functions - "min", "max", - - ## Size functions - "size", - - # Real-Valued Basic Functions - - ## Log probability function - "target", "get_lp", - - ## Logical functions - "step", "is_inf", "is_nan", - - ## Step-like functions - "fabs", "fdim", "fmin", "fmax", "fmod", "floor", "ceil", "round", - "trunc", - - ## Power and logarithm functions - "sqrt", "cbrt", "square", "exp", "exp2", "log", "log2", "log10", - "pow", "inv", "inv_sqrt", "inv_square", - - ## Trigonometric functions - "hypot", "cos", "sin", "tan", "acos", "asin", "atan", "atan2", - - ## Hyperbolic trigonometric functions - "cosh", "sinh", "tanh", "acosh", "asinh", "atanh", - - ## Link functions - "logit", "inv_logit", "inv_cloglog", - - ## Probability-related functions - "erf", "erfc", "Phi", "inv_Phi", "Phi_approx", "binary_log_loss", - "owens_t", - - ## Combinatorial functions - "beta", "inc_beta", "lbeta", "tgamma", "lgamma", "digamma", - "trigamma", "lmgamma", "gamma_p", "gamma_q", - "binomial_coefficient_log", "choose", "bessel_first_kind", - "bessel_second_kind", "modified_bessel_first_kind", - "log_modified_bessel_first_kind", "modified_bessel_second_kind", - "falling_factorial", "lchoose", "log_falling_factorial", - "rising_factorial", "log_rising_factorial", - - ## Composed functions - "expm1", "fma", "multiply_log", "ldexp", "lmultiply", "log1p", - "log1m", "log1p_exp", "log1m_exp", "log_diff_exp", "log_mix", - "log_sum_exp", "log_inv_logit", "log_inv_logit_diff", - "log1m_inv_logit", - - ## Special functions - "lambert_w0", "lambert_wm1", - - # Complex-Valued Basic Functions - - ## Complex constructors and accessors - "to_complex", "get_real", "get_imag", - - ## Complex special functions - "arg", "norm", "conj", "proj", "polar", - - # Array Operations - - ## Reductions - "sum", "prod", "mean", "variance", "sd", "distance", - "squared_distance", "quantile", - - ## Array size and dimension function - "dims", "num_elements", - - ## Array broadcasting - "rep_array", - - ## Array concatenation - "append_array", - - ## Sorting functions - "sort_asc", "sort_desc", "sort_indices_asc", "sort_indices_desc", - "rank", - - ## Reversing functions - "reverse", "rows", "cols", - - ## Dot products and specialized products - "dot_product", "columns_dot_product", "rows_dot_product", "dot_self", - "columns_dot_self", "rows_dot_self", "tcrossprod", "crossprod", - "quad_form", "quad_form_diag", "quad_form_sym", "trace_quad_form", - "trace_gen_quad_form", "multiply_lower_tri_self_transpose", - "diag_pre_multiply", "diag_post_multiply", - - ## Broadcast functions - "rep_vector", "rep_row_vector", "rep_matrix", - "symmetrize_from_lower_tri", - - ## Diagonal matrix functions - "add_diag", "diagonal", "diag_matrix", "identity_matrix", - - ## Container construction functions - "linspaced_array", "linspaced_int_array", "linspaced_vector", - "linspaced_row_vector", "one_hot_int_array", "one_hot_array", - "one_hot_vector", "one_hot_row_vector", "ones_int_array", - "ones_array", "ones_vector", "ones_row_vector", "zeros_int_array", - "zeros_array", "zeros_vector", "zeros_row_vector", "uniform_simplex", - - ## Slicing and blocking functions - "col", "row", "block", "sub_col", "sub_row", "head", "tail", - "segment", - - ## Matrix concatenation - "append_col", "append_row", - - ## Special matrix functions - "softmax", "log_softmax", "cumulative_sum", - - ## Covariance functions - "cov_exp_quad", - - ## Linear algebra functions and solvers - "mdivide_left_tri_low", "mdivide_right_tri_low", "mdivide_left_spd", - "mdivide_right_spd", "matrix_exp", "matrix_exp_multiply", - "scale_matrix_exp_multiply", "matrix_power", "trace", "determinant", - "log_determinant", "inverse", "inverse_spd", "chol2inv", - "generalized_inverse", "eigenvalues_sym", "eigenvectors_sym", - "qr_thin_Q", "qr_thin_R", "qr_Q", "qr_R", "cholseky_decompose", - "singular_values", "svd_U", "svd_V", - - # Sparse Matrix Operations - - ## Conversion functions - "csr_extract_w", "csr_extract_v", "csr_extract_u", - "csr_to_dense_matrix", - - ## Sparse matrix arithmetic - "csr_matrix_times_vector", - - # Mixed Operations - "to_matrix", "to_vector", "to_row_vector", "to_array_2d", - "to_array_1d", - - # Higher-Order Functions - - ## Algebraic equation solver - "algebra_solver", "algebra_solver_newton", - - ## Ordinary differential equation - "ode_rk45", "ode_rk45_tol", "ode_ckrk", "ode_ckrk_tol", "ode_adams", - "ode_adams_tol", "ode_bdf", "ode_bdf_tol", "ode_adjoint_tol_ctl", - - ## 1D integrator - "integrate_1d", - - ## Reduce-sum function - "reduce_sum", "reduce_sum_static", - - ## Map-rect function - "map_rect", - - # Deprecated Functions - "integrate_ode_rk45", "integrate_ode", "integrate_ode_adams", - "integrate_ode_bdf", - - # Hidden Markov Models - "hmm_marginal", "hmm_latent_rng", "hmm_hidden_state_prob" - ] - end - - def self.distributions - @distributions ||= Set.new( - [ - # Discrete Distributions - - ## Binary Distributions - "bernoulli", "bernoulli_logit", "bernoulli_logit_glm", - - ## Bounded Discrete Distributions - "binomial", "binomial_logit", "beta_binomial", "hypergeometric", - "categorical", "categorical_logit_glm", "discrete_range", - "ordered_logistic", "ordered_logistic_glm", "ordered_probit", - - ## Unbounded Discrete Distributions - "neg_binomial", "neg_binomial_2", "neg_binomial_2_log", - "neg_binomial_2_log_glm", "poisson", "poisson_log", - "poisson_log_glm", - - ## Multivariate Discrete Distributions - "multinomial", "multinomial_logit", - - # Continuous Distributions - - ## Unbounded Continuous Distributions - "normal", "std_normal", "normal_id_glm", "exp_mod_normal", - "skew_normal", "student_t", "cauchy", "double_exponential", - "logistic", "gumbel", "skew_double_exponential", - - ## Positive Continuous Distributions - "lognormal", "chi_square", "inv_chi_square", - "scaled_inv_chi_square", "exponential", "gamma", "inv_gamma", - "weibull", "frechet", "rayleigh", - - ## Positive Lower-Bounded Distributions - "pareto", "pareto_type_2", "wiener", - - ## Continuous Distributions on [0, 1] - "beta", "beta_proportion", - - ## Circular Distributions - "von_mises", - - ## Bounded Continuous Distributions - "uniform", - - ## Distributions over Unbounded Vectors - "multi_normal", "multi_normal_prec", "multi_normal_cholesky", - "multi_gp", "multi_gp_cholesky", "multi_student_t", - "gaussian_dlm_obs", - - ## Simplex Distributions - "dirichlet", - - ## Correlation Matrix Distributions - "lkj_corr", "lkj_corr_cholesky", - - ## Covariance Matrix Distributions - "wishart", "inv_wishart" - ].product([ - "", "_lpmf", "_lupmf", "_lpdf", "_lcdf", "_lccdf", "_rng", "_log", - "_cdf_log", "_ccdf_log" - ]).map {|s| "#{s[0]}#{s[1]}"} - ) - end - - def self.constants - @constants ||= Set.new [ - # Mathematical constants - "pi", "e", "sqrt2", "log2", "log10", - - # Special values - "not_a_number", "positive_infinity", "negative_infinity", - "machine_precision" - ] - end + | [?:] + )x state :root do mixin :whitespace @@ -391,18 +103,11 @@ def self.constants rule %r([|](?![|])), Punctuation rule %r(T\b), Keyword::Reserved rule %r((lower|upper)\b), Name::Attribute - rule ID do |m| - name = m[0] - - if self.class.keywords.include? name - token Keyword - elsif self.class.types.include? name - token Keyword::Type - elsif self.class.reserved.include? name - token Keyword::Reserved - else - token Name::Variable - end + keywords ID do + rule KEYWORDS, Keyword + rule TYPES, Keyword::Type + rule RESERVED, Keyword::Reserved + default Name end end @@ -416,11 +121,11 @@ def self.constants recurse m[1] name = m[2] - if self.class.builtin_functions.include? name + if BUILTIN_FUNCTIONS.include? name token Name::Builtin, name - elsif self.class.distributions.include? name + elsif DISTRIBUTIONS.include? name token Name::Builtin, name - elsif self.class.constants.include? name + elsif CONSTANTS.include? name token Keyword::Constant else token Name::Function, name diff --git a/lib/rouge/lexers/stan/keywords.rb b/lib/rouge/lexers/stan/keywords.rb new file mode 100644 index 0000000000..c841295897 --- /dev/null +++ b/lib/rouge/lexers/stan/keywords.rb @@ -0,0 +1,281 @@ +module Rouge + module Lexers + class Stan + KEYWORDS = Set.new %w( + if else while for break continue print reject return + ) + + TYPES = Set.new %w( + int real vector ordered positive_ordered simplex unit_vector + row_vector matrix cholesky_factor_corr cholesky_factor_cov corr_matrix + cov_matrix data void complex array + ) + + RESERVED = Set.new [ + # Reserved words from Stan language + "for", "in", "while", "repeat", "until", "if", "then", "else", "true", + "false", "target", "functions", "model", "data", "parameters", + "quantities", "transformed", "generated", + + # Reserved names from Stan implementation + "var", "fvar", "STAN_MAJOR", "STAN_MINOR", "STAN_PATCH", + "STAN_MATH_MAJOR", "STAN_MATH_MINOR", "STAN_MATH_PATCH", + + # Reserved names from C++ + "alignas", "alignof", "and", "and_eq", "asm", "auto", "bitand", + "bitor", "bool", "break", "case", "catch", "char", "char16_t", + "char32_t", "class", "compl", "const", "constexpr", "const_cast", + "continue", "decltype", "default", "delete", "do", "double", + "dynamic_cast", "enum", "explicit", "export", "extern", "float", "friend", "goto", "inline", "int", + "long", "mutable", "namespace", "new", "noexcept", "not", "not_eq", + "nullptr", "operator", "or", "or_eq", "private", "protected", + "public", "register", "reinterpret_cast", "return", "short", "signed", + "sizeof", "static", "static_assert", "static_cast", "struct", + "switch", "template", "this", "thread_local", "throw", "try", + "typedef", "typeid", "typename", "union", "unsigned", "using", + "virtual", "void", "volatile", "wchar_t", "xor", "xor_eq" + ] + + BUILTIN_FUNCTIONS = Set.new [ + # Integer-Valued Basic Functions + + ## Absolute functions + "abs", "int_step", + + ## Bound functions + "min", "max", + + ## Size functions + "size", + + # Real-Valued Basic Functions + + ## Log probability function + "target", "get_lp", + + ## Logical functions + "step", "is_inf", "is_nan", + + ## Step-like functions + "fabs", "fdim", "fmin", "fmax", "fmod", "floor", "ceil", "round", + "trunc", + + ## Power and logarithm functions + "sqrt", "cbrt", "square", "exp", "exp2", "log", "log2", "log10", + "pow", "inv", "inv_sqrt", "inv_square", + + ## Trigonometric functions + "hypot", "cos", "sin", "tan", "acos", "asin", "atan", "atan2", + + ## Hyperbolic trigonometric functions + "cosh", "sinh", "tanh", "acosh", "asinh", "atanh", + + ## Link functions + "logit", "inv_logit", "inv_cloglog", + + ## Probability-related functions + "erf", "erfc", "Phi", "inv_Phi", "Phi_approx", "binary_log_loss", + "owens_t", + + ## Combinatorial functions + "beta", "inc_beta", "lbeta", "tgamma", "lgamma", "digamma", + "trigamma", "lmgamma", "gamma_p", "gamma_q", + "binomial_coefficient_log", "choose", "bessel_first_kind", + "bessel_second_kind", "modified_bessel_first_kind", + "log_modified_bessel_first_kind", "modified_bessel_second_kind", + "falling_factorial", "lchoose", "log_falling_factorial", + "rising_factorial", "log_rising_factorial", + + ## Composed functions + "expm1", "fma", "multiply_log", "ldexp", "lmultiply", "log1p", + "log1m", "log1p_exp", "log1m_exp", "log_diff_exp", "log_mix", + "log_sum_exp", "log_inv_logit", "log_inv_logit_diff", + "log1m_inv_logit", + + ## Special functions + "lambert_w0", "lambert_wm1", + + # Complex-Valued Basic Functions + + ## Complex constructors and accessors + "to_complex", "get_real", "get_imag", + + ## Complex special functions + "arg", "norm", "conj", "proj", "polar", + + # Array Operations + + ## Reductions + "sum", "prod", "mean", "variance", "sd", "distance", + "squared_distance", "quantile", + + ## Array size and dimension function + "dims", "num_elements", + + ## Array broadcasting + "rep_array", + + ## Array concatenation + "append_array", + + ## Sorting functions + "sort_asc", "sort_desc", "sort_indices_asc", "sort_indices_desc", + "rank", + + ## Reversing functions + "reverse", "rows", "cols", + + ## Dot products and specialized products + "dot_product", "columns_dot_product", "rows_dot_product", "dot_self", + "columns_dot_self", "rows_dot_self", "tcrossprod", "crossprod", + "quad_form", "quad_form_diag", "quad_form_sym", "trace_quad_form", + "trace_gen_quad_form", "multiply_lower_tri_self_transpose", + "diag_pre_multiply", "diag_post_multiply", + + ## Broadcast functions + "rep_vector", "rep_row_vector", "rep_matrix", + "symmetrize_from_lower_tri", + + ## Diagonal matrix functions + "add_diag", "diagonal", "diag_matrix", "identity_matrix", + + ## Container construction functions + "linspaced_array", "linspaced_int_array", "linspaced_vector", + "linspaced_row_vector", "one_hot_int_array", "one_hot_array", + "one_hot_vector", "one_hot_row_vector", "ones_int_array", + "ones_array", "ones_vector", "ones_row_vector", "zeros_int_array", + "zeros_array", "zeros_vector", "zeros_row_vector", "uniform_simplex", + + ## Slicing and blocking functions + "col", "row", "block", "sub_col", "sub_row", "head", "tail", + "segment", + + ## Matrix concatenation + "append_col", "append_row", + + ## Special matrix functions + "softmax", "log_softmax", "cumulative_sum", + + ## Covariance functions + "cov_exp_quad", + + ## Linear algebra functions and solvers + "mdivide_left_tri_low", "mdivide_right_tri_low", "mdivide_left_spd", + "mdivide_right_spd", "matrix_exp", "matrix_exp_multiply", + "scale_matrix_exp_multiply", "matrix_power", "trace", "determinant", + "log_determinant", "inverse", "inverse_spd", "chol2inv", + "generalized_inverse", "eigenvalues_sym", "eigenvectors_sym", + "qr_thin_Q", "qr_thin_R", "qr_Q", "qr_R", "cholseky_decompose", + "singular_values", "svd_U", "svd_V", + + # Sparse Matrix Operations + + ## Conversion functions + "csr_extract_w", "csr_extract_v", "csr_extract_u", + "csr_to_dense_matrix", + + ## Sparse matrix arithmetic + "csr_matrix_times_vector", + + # Mixed Operations + "to_matrix", "to_vector", "to_row_vector", "to_array_2d", + "to_array_1d", + + # Higher-Order Functions + + ## Algebraic equation solver + "algebra_solver", "algebra_solver_newton", + + ## Ordinary differential equation + "ode_rk45", "ode_rk45_tol", "ode_ckrk", "ode_ckrk_tol", "ode_adams", + "ode_adams_tol", "ode_bdf", "ode_bdf_tol", "ode_adjoint_tol_ctl", + + ## 1D integrator + "integrate_1d", + + ## Reduce-sum function + "reduce_sum", "reduce_sum_static", + + ## Map-rect function + "map_rect", + + # Deprecated Functions + "integrate_ode_rk45", "integrate_ode", "integrate_ode_adams", + "integrate_ode_bdf", + + # Hidden Markov Models + "hmm_marginal", "hmm_latent_rng", "hmm_hidden_state_prob" + ] + + DISTRIBUTIONS = Set.new([ + # Discrete Distributions + + ## Binary Distributions + "bernoulli", "bernoulli_logit", "bernoulli_logit_glm", + + ## Bounded Discrete Distributions + "binomial", "binomial_logit", "beta_binomial", "hypergeometric", + "categorical", "categorical_logit_glm", "discrete_range", + "ordered_logistic", "ordered_logistic_glm", "ordered_probit", + + ## Unbounded Discrete Distributions + "neg_binomial", "neg_binomial_2", "neg_binomial_2_log", + "neg_binomial_2_log_glm", "poisson", "poisson_log", + "poisson_log_glm", + + ## Multivariate Discrete Distributions + "multinomial", "multinomial_logit", + + # Continuous Distributions + + ## Unbounded Continuous Distributions + "normal", "std_normal", "normal_id_glm", "exp_mod_normal", + "skew_normal", "student_t", "cauchy", "double_exponential", + "logistic", "gumbel", "skew_double_exponential", + + ## Positive Continuous Distributions + "lognormal", "chi_square", "inv_chi_square", + "scaled_inv_chi_square", "exponential", "gamma", "inv_gamma", + "weibull", "frechet", "rayleigh", + + ## Positive Lower-Bounded Distributions + "pareto", "pareto_type_2", "wiener", + + ## Continuous Distributions on [0, 1] + "beta", "beta_proportion", + + ## Circular Distributions + "von_mises", + + ## Bounded Continuous Distributions + "uniform", + + ## Distributions over Unbounded Vectors + "multi_normal", "multi_normal_prec", "multi_normal_cholesky", + "multi_gp", "multi_gp_cholesky", "multi_student_t", + "gaussian_dlm_obs", + + ## Simplex Distributions + "dirichlet", + + ## Correlation Matrix Distributions + "lkj_corr", "lkj_corr_cholesky", + + ## Covariance Matrix Distributions + "wishart", "inv_wishart" + ].product([ + "", "_lpmf", "_lupmf", "_lpdf", "_lcdf", "_lccdf", "_rng", "_log", + "_cdf_log", "_ccdf_log" + ]).map { |(a, b)| "#{a}#{b}" }) + + CONSTANTS = Set.new [ + # Mathematical constants + "pi", "e", "sqrt2", "log2", "log10", + + # Special values + "not_a_number", "positive_infinity", "negative_infinity", + "machine_precision" + ] + end + end +end From aa992f5619b249ab74078c9d6cfb001c7756f253 Mon Sep 17 00:00:00 2001 From: jneen Date: Wed, 24 Jun 2026 13:27:07 -0400 Subject: [PATCH 149/154] use the keywords api for stata --- .rubocop_todo.yml | 6 -- lib/rouge/lexers/stata.rb | 137 +++++++---------------------- lib/rouge/lexers/stata/builtins.rb | 88 ++++++++++++++++++ 3 files changed, 120 insertions(+), 111 deletions(-) create mode 100644 lib/rouge/lexers/stata/builtins.rb diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index d98c65e617..0887512b76 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -24,12 +24,6 @@ Naming/VariableName: # Offense count: 113 Rouge/NoBuildingAlternationPatternInRegexp: Exclude: - - 'lib/rouge/lexers/stata.rb' - 'lib/rouge/lexers/vala.rb' - 'lib/rouge/lexers/wollok.rb' - 'lib/rouge/lexers/xojo.rb' - -# Offense count: 20 -Rouge/NoHugeCollections: - Exclude: - - 'lib/rouge/lexers/stata.rb' diff --git a/lib/rouge/lexers/stata.rb b/lib/rouge/lexers/stata.rb index d002316204..d31d0b6f98 100644 --- a/lib/rouge/lexers/stata.rb +++ b/lib/rouge/lexers/stata.rb @@ -10,113 +10,42 @@ class Stata < RegexLexer filenames '*.do', '*.ado' mimetypes 'application/x-stata', 'text/x-stata' - ### - # Stata reference manual is available online at: https://www.stata.com/features/documentation/ - ### + lazy { require_relative 'stata/builtins' } - # Partial list of common programming and estimation commands, as of Stata 16 - # Note: not all abbreviations are included - KEYWORDS = %w( - do run include clear assert set mata log - by bys bysort cap capt capture char class classutil which cdir confirm new existence creturn - _datasignature discard di dis disp displ displa display ereturn error _estimates exit file open read write seek close query findfile fvexpand - gettoken java home heapmax java_heapmax icd9 icd9p icd10 icd10cm icd10pcs initialize javacall levelsof - tempvar tempname tempfile macro shift uniq dups retokenize clean sizeof posof - makecns matcproc marksample mark markout markin svymarkout matlist - accum define dissimilarity eigenvalues get rowjoinbyname rownames score svd symeigen dir list ren rename - more pause plugin call postfile _predict preserve restore program define drop end python qui quietly noi noisily _return return _rmcoll rmsg _robust - serset locale_functions locale_ui signestimationsample checkestimationsample sleep syntax sysdir adopath adosize - tabdisp timer tokenize trace unab unabcmd varabbrev version viewsource - window fopen fsave manage menu push stopbox - net from cd link search install sj stb ado update uninstall pwd ssc ls - using insheet outsheet mkmat svmat sum summ summarize - graph gr_edit twoway histogram kdensity spikeplot - mi miss missing var varname order compress append - gen gene gener genera generat generate egen replace duplicates - estimates nlcom lincom test testnl predict suest - _regress reg regr regre regres regress probit logit ivregress logistic svy gmm ivprobit ivtobit - bsample assert codebook collapse compare contract copy count cross datasignature d ds desc describe destring tostring - drawnorm edit encode decode erase expand export filefilter fillin format frame frget frlink gsort - import dbase delimited excel fred haver sas sasxport5 sasxport8 spss infile infix input insobs inspect ipolate isid - joinby label language labelbook lookfor memory mem merge mkdir mvencode notes obs odbc order outfile - pctile xtile _pctile putmata range recast recode rename group reshape rm rmdir sample save saveold separate shell snapshot sort split splitsample stack statsby sysuse - type unicode use varmanage vl webuse xpose zipfile - number keep tab table tabulate stset stcox tsset xtset - ) - - # Complete list of functions by name, as of Stata 16 - PRIMITIVE_FUNCTIONS = %w( - abbrev abs acos acosh age age_frac asin asinh atan atan2 atanh autocode - betaden binomial binomialp binomialtail binormal birthday bofd byteorder - c _caller cauchy cauchyden cauchytail Cdhms ceil char chi2 chi2den chi2tail Chms - chop cholesky clip Clock clock clockdiff cloglog Cmdyhms Cofc cofC Cofd cofd coleqnumb - collatorlocale collatorversion colnfreeparms colnumb colsof comb cond corr cos cosh - daily date datediff datediff_frac day det dgammapda dgammapdada dgammapdadx dgammapdxdx dhms - diag diag0cnt digamma dofb dofC dofc dofh dofm dofq dofw dofy dow doy dunnettprob e el epsdouble - epsfloat exp expm1 exponential exponentialden exponentialtail - F Fden fileexists fileread filereaderror filewrite float floor fmtwidth frval _frval Ftail - fammaden gammap gammaptail get hadamard halfyear halfyearly has_eprop hh hhC hms hofd hours - hypergeometric hypergeometricp - I ibeta ibetatail igaussian igaussianden igaussiantail indexnot inlist inrange int inv invbinomial invbinomialtail - invcauchy invcauchytail invchi2 invchi2tail invcloglog invdunnettprob invexponential invexponentialtail invF - invFtail invgammap invgammaptail invibeta invibetatail invigaussian invigaussiantail invlaplace invlaplacetail - invlogistic invlogistictail invlogit invnbinomial invnbinomialtail invnchi2 invnchi2tail invnF invnFtail invnibeta invnormal invnt invnttail - invpoisson invpoissontail invsym invt invttail invtukeyprob invweibull invweibullph invweibullphtail invweibulltail irecode islepyear issymmetric - J laplace laplaceden laplacetail ln ln1m ln1p lncauchyden lnfactorial lngamma lnigammaden lnigaussianden lniwishartden lnlaplaceden lnmvnormalden - lnnormal lnnormalden lnnormalden lnnormalden lnwishartden log log10 log1m log1p logistic logisticden logistictail logit - matmissing matrix matuniform max maxbyte maxdouble maxfloat maxint maxlong mdy mdyhms mi min minbyte mindouble minfloat minint minlong minutes - missing mm mmC mod mofd month monthly mreldif msofhours msofminutes msofseconds - nbetaden nbinomial nbinomialp nbinomialtail nchi2 nchi2den nchi2tail nextbirthday nextleapyear nF nFden nFtail nibeta - normal normalden npnchi2 npnF npnt nt ntden nttail nullmat - plural poisson poissonp poissontail previousbirthday previousleapyear qofd quarter quarterly r rbeta rbinomial rcauchy rchi2 recode - real regexm regexr regexs reldif replay return rexponential rgamma rhypergeometric rigaussian rlaplace rlogistic rnormal - round roweqnumb rownfreeparms rownumb rowsof rpoisson rt runiform runiformint rweibull rweibullph - s scalar seconds sign sin sinh smallestdouble soundex soundex_nara sqrt ss ssC strcat strdup string stritrim strlen strlower - strltrim strmatch strofreal strpos strproper strreverse strrpos strrtrim strtoname strtrim strupper subinstr subinword substr sum sweep - t tan tanh tC tc td tden th tin tm tobytes tq trace trigamma trunc ttail tukeyprob tw twithin - uchar udstrlen udsubstr uisdigit uisletter uniform ustrcompare ustrcompareex ustrfix ustrfrom ustrinvalidcnt ustrleft ustrlen ustrlower - ustrltrim ustrnormalize ustrpos ustrregexm ustrregexra ustrregexrf ustrregexs ustrreverse ustrright ustrrpos ustrrtrim ustrsortkey - ustrsortkeyex ustrtitle ustrto ustrtohex ustrtoname ustrtrim ustrunescape ustrupper ustrword ustrwordcount usubinstr usubstr - vec vecdiag week weekly weibull weibullden weibullph weibullphden weibullphtail weibulltail wofd word wordbreaklocale wordcount - year yearly yh ym yofd yq yw - ) - - # Note: types `str1-str2045` handled separately below - def self.type_keywords - @type_keywords ||= Set.new %w(byte int long float double str strL numeric string integer scalar matrix local global numlist varlist newlist) - end + start { push :bol } - # Stata commands used with braces. Includes all valid abbreviations for 'forvalues'. - def self.reserved_keywords - @reserved_keywords ||= Set.new %w(if else foreach forv forva forval forvalu forvalue forvalues to while in of continue break nobreak) + state :inline_whitespace do + rule %r/[ \t]+/, Text::Whitespace end - ### - # Lexer state and rules - ### - state :root do + state :bol do + mixin :inline_whitespace - # Pre-processor commands: # - rule %r/^\s*#.*$/, Comment::Preproc + # Multi-line comment: /* and */ + rule %r(/(\\\n)?[*].*?[*](\\\n)?/)m, Comment::Multiline + + keywords %r/[a-z_]+/ do + rule KEYWORDS, Keyword, :pop! + end - # Hashbang comments: *! - rule %r/^\*!.*$/, Comment::Hashbang + # Pre-processor commands: # + rule %r/#.*\n?/, Comment::Preproc - # Single-line comment: * - rule %r/^\s*\*.*$/, Comment::Single + rule %r/[*]!.*\n?/, Comment::Hashbang + rule %r/[*].*\n?/, Comment::Single - # Keywords: recognize only when they are the first word - rule %r/^\s*(#{KEYWORDS.join('|')})\b/, Keyword + rule(//) { pop! } + end - # Whitespace. Classify `\n` as `Text` to avoid interference with `Comment` and `Keyword` above - rule(/[ \t]+/, Text::Whitespace) - rule(/[\n\r]+/, Text) + ### + # Lexer state and rules + ### + state :root do + mixin :inline_whitespace # In-line comment: // - rule %r/\/\/.*?$/, Comment::Single - - # Multi-line comment: /* and */ - rule %r(/(\\\n)?[*].*?[*](\\\n)?/)m, Comment::Multiline + rule %r(//.*\n?), Comment::Single, :bol + rule %r([\r\n]+), Text, :bol # Strings indicated by compound double-quotes (`""') and double-quotes ("") rule %r/`"(\\.|.)*?"'/, Str::Double @@ -134,7 +63,9 @@ def self.reserved_keywords # Only recognize primitive functions when they are actually used as a function call, i.e. followed by an opening parenthesis # `Name::Builtin` would be more logical, but is not usually highlighted, so use `Name::Function` instead - rule %r/\b(#{PRIMITIVE_FUNCTIONS.join('|')})(?=\()/, Name::Function + keywords %r/\w+(?=[(])/ do + rule PRIMITIVE_FUNCTIONS, Name::Function + end # Matrix operator `..` (declare here instead of with other operators, in order to avoid conflict with numbers below) rule %r/\.\.(?=.*\])/, Operator @@ -146,14 +77,10 @@ def self.reserved_keywords rule %r/\b[ICOicoLFDSlfds]\w*\./, Operator rule %r/\b[ICOicoLFDSlfds]\w*(?=\(.*\)\.)/, Operator - rule %r/\w+/ do |m| - if self.class.reserved_keywords.include? m[0] - token Keyword::Reserved - elsif self.class.type_keywords.include? m[0] - token Keyword::Type - else - token Name - end + keywords %r/\w+/ do + rule RESERVED_KEYWORDS, Keyword::Reserved + rule TYPE_KEYWORDS, Keyword::Type + default Name end rule %r/[\[\]{}();,]/, Punctuation diff --git a/lib/rouge/lexers/stata/builtins.rb b/lib/rouge/lexers/stata/builtins.rb new file mode 100644 index 0000000000..50da626c33 --- /dev/null +++ b/lib/rouge/lexers/stata/builtins.rb @@ -0,0 +1,88 @@ +module Rouge + module Lexers + class Stata + ### + # Stata reference manual is available online at: https://www.stata.com/features/documentation/ + ### + + # Partial list of common programming and estimation commands, as of Stata 16 + # Note: not all abbreviations are included + KEYWORDS = Set.new %w( + do run include clear assert set mata log + by bys bysort cap capt capture char class classutil which cdir confirm new existence creturn + _datasignature discard di dis disp displ displa display ereturn error _estimates exit file open read write seek close query findfile fvexpand + gettoken java home heapmax java_heapmax icd9 icd9p icd10 icd10cm icd10pcs initialize javacall levelsof + tempvar tempname tempfile macro shift uniq dups retokenize clean sizeof posof + makecns matcproc marksample mark markout markin svymarkout matlist + accum define dissimilarity eigenvalues get rowjoinbyname rownames score svd symeigen dir list ren rename + more pause plugin call postfile _predict preserve restore program define drop end python qui quietly noi noisily _return return _rmcoll rmsg _robust + serset locale_functions locale_ui signestimationsample checkestimationsample sleep syntax sysdir adopath adosize + tabdisp timer tokenize trace unab unabcmd varabbrev version viewsource + window fopen fsave manage menu push stopbox + net from cd link search install sj stb ado update uninstall pwd ssc ls + using insheet outsheet mkmat svmat sum summ summarize + graph gr_edit twoway histogram kdensity spikeplot + mi miss missing var varname order compress append + gen gene gener genera generat generate egen replace duplicates + estimates nlcom lincom test testnl predict suest + _regress reg regr regre regres regress probit logit ivregress logistic svy gmm ivprobit ivtobit + bsample assert codebook collapse compare contract copy count cross datasignature d ds desc describe destring tostring + drawnorm edit encode decode erase expand export filefilter fillin format frame frget frlink gsort + import dbase delimited excel fred haver sas sasxport5 sasxport8 spss infile infix input insobs inspect ipolate isid + joinby label language labelbook lookfor memory mem merge mkdir mvencode notes obs odbc order outfile + pctile xtile _pctile putmata range recast recode rename group reshape rm rmdir sample save saveold separate shell snapshot sort split splitsample stack statsby sysuse + type unicode use varmanage vl webuse xpose zipfile + number keep tab table tabulate stset stcox tsset xtset + ) + + # Complete list of functions by name, as of Stata 16 + PRIMITIVE_FUNCTIONS = Set.new %w( + abbrev abs acos acosh age age_frac asin asinh atan atan2 atanh autocode + betaden binomial binomialp binomialtail binormal birthday bofd byteorder + c _caller cauchy cauchyden cauchytail Cdhms ceil char chi2 chi2den chi2tail Chms + chop cholesky clip Clock clock clockdiff cloglog Cmdyhms Cofc cofC Cofd cofd coleqnumb + collatorlocale collatorversion colnfreeparms colnumb colsof comb cond corr cos cosh + daily date datediff datediff_frac day det dgammapda dgammapdada dgammapdadx dgammapdxdx dhms + diag diag0cnt digamma dofb dofC dofc dofh dofm dofq dofw dofy dow doy dunnettprob e el epsdouble + epsfloat exp expm1 exponential exponentialden exponentialtail + F Fden fileexists fileread filereaderror filewrite float floor fmtwidth frval _frval Ftail + fammaden gammap gammaptail get hadamard halfyear halfyearly has_eprop hh hhC hms hofd hours + hypergeometric hypergeometricp + I ibeta ibetatail igaussian igaussianden igaussiantail indexnot inlist inrange int inv invbinomial invbinomialtail + invcauchy invcauchytail invchi2 invchi2tail invcloglog invdunnettprob invexponential invexponentialtail invF + invFtail invgammap invgammaptail invibeta invibetatail invigaussian invigaussiantail invlaplace invlaplacetail + invlogistic invlogistictail invlogit invnbinomial invnbinomialtail invnchi2 invnchi2tail invnF invnFtail invnibeta invnormal invnt invnttail + invpoisson invpoissontail invsym invt invttail invtukeyprob invweibull invweibullph invweibullphtail invweibulltail irecode islepyear issymmetric + J laplace laplaceden laplacetail ln ln1m ln1p lncauchyden lnfactorial lngamma lnigammaden lnigaussianden lniwishartden lnlaplaceden lnmvnormalden + lnnormal lnnormalden lnnormalden lnnormalden lnwishartden log log10 log1m log1p logistic logisticden logistictail logit + matmissing matrix matuniform max maxbyte maxdouble maxfloat maxint maxlong mdy mdyhms mi min minbyte mindouble minfloat minint minlong minutes + missing mm mmC mod mofd month monthly mreldif msofhours msofminutes msofseconds + nbetaden nbinomial nbinomialp nbinomialtail nchi2 nchi2den nchi2tail nextbirthday nextleapyear nF nFden nFtail nibeta + normal normalden npnchi2 npnF npnt nt ntden nttail nullmat + plural poisson poissonp poissontail previousbirthday previousleapyear qofd quarter quarterly r rbeta rbinomial rcauchy rchi2 recode + real regexm regexr regexs reldif replay return rexponential rgamma rhypergeometric rigaussian rlaplace rlogistic rnormal + round roweqnumb rownfreeparms rownumb rowsof rpoisson rt runiform runiformint rweibull rweibullph + s scalar seconds sign sin sinh smallestdouble soundex soundex_nara sqrt ss ssC strcat strdup string stritrim strlen strlower + strltrim strmatch strofreal strpos strproper strreverse strrpos strrtrim strtoname strtrim strupper subinstr subinword substr sum sweep + t tan tanh tC tc td tden th tin tm tobytes tq trace trigamma trunc ttail tukeyprob tw twithin + uchar udstrlen udsubstr uisdigit uisletter uniform ustrcompare ustrcompareex ustrfix ustrfrom ustrinvalidcnt ustrleft ustrlen ustrlower + ustrltrim ustrnormalize ustrpos ustrregexm ustrregexra ustrregexrf ustrregexs ustrreverse ustrright ustrrpos ustrrtrim ustrsortkey + ustrsortkeyex ustrtitle ustrto ustrtohex ustrtoname ustrtrim ustrunescape ustrupper ustrword ustrwordcount usubinstr usubstr + vec vecdiag week weekly weibull weibullden weibullph weibullphden weibullphtail weibulltail wofd word wordbreaklocale wordcount + year yearly yh ym yofd yq yw + ) + + # Note: types `str1-str2045` handled separately below + TYPE_KEYWORDS = Set.new %w( + byte int long float double str strL numeric string integer scalar matrix + local global numlist varlist newlist + ) + + # Stata commands used with braces. Includes all valid abbreviations for 'forvalues'. + RESERVED_KEYWORDS = Set.new %w( + if else foreach forv forva forval forvalu forvalue forvalues to while + in of continue break nobreak + ) + end + end +end From 52db194c166f50e534b1c3cd45786d0797c5d3d4 Mon Sep 17 00:00:00 2001 From: jneen Date: Wed, 24 Jun 2026 13:31:26 -0400 Subject: [PATCH 150/154] use the keywords api for supercollider --- lib/rouge/lexers/supercollider.rb | 45 ++++++++++++------------------- 1 file changed, 17 insertions(+), 28 deletions(-) diff --git a/lib/rouge/lexers/supercollider.rb b/lib/rouge/lexers/supercollider.rb index b2831a3a9a..b1109b572c 100644 --- a/lib/rouge/lexers/supercollider.rb +++ b/lib/rouge/lexers/supercollider.rb @@ -10,29 +10,23 @@ class SuperCollider < RegexLexer title "SuperCollider" desc 'A cross-platform interpreted programming language for sound synthesis, algorithmic composition, and realtime performance' - def self.keywords - @keywords ||= Set.new %w( - var arg classvar const super this - ) - end + KEYWORDS = Set.new %w( + var arg classvar const super this + ) # these aren't technically keywords, but we treat # them as such because it makes things clearer 99% # of the time - def self.reserved - @reserved ||= Set.new %w( - case do for forBy loop if while new newCopyArgs - ) - end + RESERVED = Set.new %w( + case do for forBy loop if while new newCopyArgs + ) - def self.constants - @constants ||= Set.new %w( - true false nil inf thisThread - thisMethod thisFunction thisProcess - thisFunctionDef currentEnvironment - topEnvironment - ) - end + CONSTANTS = Set.new %w( + true false nil inf thisThread + thisMethod thisFunction thisProcess + thisFunctionDef currentEnvironment + topEnvironment + ) state :whitespace do rule %r/\s+/m, Text @@ -86,16 +80,11 @@ def self.constants rule %r/_\w+/, Name::Function # main identifiers section - rule %r/[a-z]\w*/ do |m| - if self.class.keywords.include? m[0] - token Keyword - elsif self.class.constants.include? m[0] - token Keyword::Constant - elsif self.class.reserved.include? m[0] - token Keyword::Reserved - else - token Name - end + keywords %r/[a-z]\w*/ do + rule KEYWORDS, Keyword + rule CONSTANTS, Keyword::Constant + rule RESERVED, Keyword::Reserved + default Name end # environment variables From 7abf34b467e00bfad30ec9218a044b6cc6d46955 Mon Sep 17 00:00:00 2001 From: jneen Date: Wed, 24 Jun 2026 13:34:19 -0400 Subject: [PATCH 151/154] use the keywords api for swift --- lib/rouge/lexers/swift.rb | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/lib/rouge/lexers/swift.rb b/lib/rouge/lexers/swift.rb index b757f5b3ff..8ee8a11468 100644 --- a/lib/rouge/lexers/swift.rb +++ b/lib/rouge/lexers/swift.rb @@ -14,7 +14,7 @@ class Swift < RegexLexer id_rest = /[\p{Alnum}_]|[^\u0000-\uFFFF]/ id = /#{id_head}#{id_rest}*/ - keywords = Set.new %w( + KEYWORDS = Set.new %w( autoreleasepool await break case catch consume continue default defer discard do each else fallthrough guard if in for repeat return switch throw try where while as dynamicType is new super self Self Type @@ -22,11 +22,11 @@ class Swift < RegexLexer associativity async didSet get infix inout isolated left mutating none nonmutating operator override postfix precedence precedencegroup prefix rethrows right set throws unowned weak willSet ) - declarations = Set.new %w( + DECLARATIONS = Set.new %w( actor any associatedtype borrowing class consuming deinit distributed dynamic enum convenience extension fileprivate final func import indirect init internal lazy let macro nonisolated open optional package private protocol public required some static struct subscript typealias var ) - constants = Set.new %w( + CONSTANTS = Set.new %w( true false nil ) @@ -128,17 +128,17 @@ class Swift < RegexLexer groups Name::Variable, Text, Punctuation end - rule id do |m| - if keywords.include? m[0] - token Keyword - elsif declarations.include? m[0] - token Keyword::Declaration - elsif constants.include? m[0] - token Keyword::Constant - elsif m[0] =~ /^[[:upper:]]/ - token Keyword::Type - else - token Name + keywords id do + rule KEYWORDS, Keyword + rule DECLARATIONS, Keyword::Declaration + rule CONSTANTS, Keyword::Constant + + default do |m| + if m[0][0].match?(/[[:upper:]]/) + token Keyword::Type + else + token Name + end end end From 9c8b1121153453d25d030399c54987b6b51a6ab8 Mon Sep 17 00:00:00 2001 From: jneen Date: Wed, 24 Jun 2026 13:35:27 -0400 Subject: [PATCH 152/154] use the keywords api for syzlang --- lib/rouge/lexers/syzlang.rb | 40 +++++++++++++++---------------------- 1 file changed, 16 insertions(+), 24 deletions(-) diff --git a/lib/rouge/lexers/syzlang.rb b/lib/rouge/lexers/syzlang.rb index dbaea03e51..d5bf0a8851 100644 --- a/lib/rouge/lexers/syzlang.rb +++ b/lib/rouge/lexers/syzlang.rb @@ -8,22 +8,18 @@ class Syzlang < RegexLexer desc "Syscall description language used by syzkaller" tag 'syzlang' - def self.keywords - @keywords ||= Set.new %w( - align breaks_returns dec define disabled hex ignore_return in incdir - include inet inout oct opt out packed parent prog_timeout pseudo - resource size syscall timeout type varlen - ) - end - - def self.keywords_type - @keywords_type ||= Set.new %w( - array bitsize bool16 bool32 bool64 bool8 boolptr buffer bytesize - bytesize2 bytesize4 bytesize8 const csum filename fileoff flags fmt - int16 int16be int32 int32be int64 int64be int8 int8be intptr len - offsetof optional proc ptr ptr64 string stringnoz text vma vma64 void - ) - end + KEYWORDS = Set.new %w( + align breaks_returns dec define disabled hex ignore_return in incdir + include inet inout oct opt out packed parent prog_timeout pseudo + resource size syscall timeout type varlen + ) + + KEYWORDS_TYPE = Set.new %w( + array bitsize bool16 bool32 bool64 bool8 boolptr buffer bytesize + bytesize2 bytesize4 bytesize8 const csum filename fileoff flags fmt + int16 int16be int32 int32be int64 int64be int8 int8be intptr len + offsetof optional proc ptr ptr64 string stringnoz text vma vma64 void + ) comment = /#.*$/ inline_spaces = /[ \t]+/ @@ -63,14 +59,10 @@ def self.keywords_type mixin :mixin_string # Keywords. - rule id do |m| - if self.class.keywords.include?(m[0]) - token Keyword - elsif self.class.keywords_type.include?(m[0]) - token Keyword::Type - else - token Name - end + keywords id do + rule KEYWORDS, Keyword + rule KEYWORDS_TYPE, Keyword::Type + default Name end # Ranges. From 3d6a13c39e76e5b223d3652094670e30832410fa Mon Sep 17 00:00:00 2001 From: jneen Date: Wed, 24 Jun 2026 13:36:34 -0400 Subject: [PATCH 153/154] use the keywords api for syzprog --- lib/rouge/lexers/syzprog.rb | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/lib/rouge/lexers/syzprog.rb b/lib/rouge/lexers/syzprog.rb index 8cbcace5b7..869631ca29 100644 --- a/lib/rouge/lexers/syzprog.rb +++ b/lib/rouge/lexers/syzprog.rb @@ -8,13 +8,11 @@ class Syzprog < RegexLexer desc "Program description language used by syzkaller" tag 'syzprog' - def self.keywords - @keywords ||= Set.new %w( - ANY ANYBLOB ANYPTR ANYPTR64 ANYPTRS ANYRES16 ANYRES32 ANYRES64 - ANYRESDEC ANYRESHEX ANYRESOCT ANYUNION AUTO false nil true void - async fail_nth rerun - ) - end + KEYWORDS = Set.new %w( + ANY ANYBLOB ANYPTR ANYPTR64 ANYPTRS ANYRES16 ANYRES32 ANYRES64 + ANYRESDEC ANYRESHEX ANYRESOCT ANYUNION AUTO false nil true void + async fail_nth rerun + ) comment = /#.*$/ inline_spaces = /[ \t]+/ @@ -58,12 +56,9 @@ def self.keywords mixin :mixin_string rule %r/#{res_id}/, Keyword::Pseudo - rule id do |m| - if self.class.keywords.include?(m[0]) - token Keyword - else - token Name - end + keywords id do + rule KEYWORDS, Keyword + default Name end end From 8198ef3ba1459b8244678f47020a2b0966cf565e Mon Sep 17 00:00:00 2001 From: jneen Date: Wed, 24 Jun 2026 13:39:13 -0400 Subject: [PATCH 154/154] use the keywords api for tcl --- lib/rouge/lexers/tcl.rb | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/lib/rouge/lexers/tcl.rb b/lib/rouge/lexers/tcl.rb index 74e45deecc..533e3085c6 100644 --- a/lib/rouge/lexers/tcl.rb +++ b/lib/rouge/lexers/tcl.rb @@ -16,14 +16,14 @@ def self.detect?(text) return true if text.shebang? 'jimsh' end - KEYWORDS = %w( + KEYWORDS = Set.new %w( after apply array break catch continue elseif else error eval expr for foreach global if namespace proc rename return set switch then trace unset update uplevel upvar variable vwait while ) - BUILTINS = %w( + BUILTINS = Set.new %w( append bgerror binary cd chan clock close concat dde dict encoding eof exec exit fblocked fconfigure fcopy file fileevent flush format gets glob history http incr info interp @@ -65,14 +65,10 @@ def self.gen_command_state(name='') push :"params#{name}" end - rule %r/#{NOT_CHARS[END_WORD]}+/ do |m| - if KEYWORDS.include? m[0] - token Keyword - elsif BUILTINS.include? m[0] - token Name::Builtin - else - token Text - end + keywords %r/#{NOT_CHARS[END_WORD]}+/ do |m| + rule KEYWORDS, Keyword + rule BUILTINS, Name::Builtin + default Text end mixin :whitespace