Class: Ripper::Lexer

Inherits:
Ripper show all
Defined in:
lib/ripper/lexer.rb

Overview

:nodoc: internal use only

Defined Under Namespace

Classes: Elem, State

Constant Summary

Constants inherited from Ripper

EVENTS, PARSER_EVENTS, SCANNER_EVENTS

Instance Attribute Summary collapse

Instance Method Summary collapse

Methods inherited from Ripper

lex, parse, sexp, sexp_raw, slice, token_match, tokenize

Instance Attribute Details

#errors ⇒ Object (readonly)

Returns the value of attribute errors.



103
104
105
 
# File 'lib/ripper/lexer.rb', line 103

def errors
  @errors
end

Instance Method Details

#lex(**kw) ⇒ Object 



109
110
111
 
# File 'lib/ripper/lexer.rb', line 109

def lex(**kw)
  parse(**kw).sort_by(&:pos).map(&:to_a)
end

#parse(raise_errors: false) ⇒ Object 



128
129
130
131
132
133
134
135
136
137
138
139
140
141
 
# File 'lib/ripper/lexer.rb', line 128

def parse(raise_errors: false)
  @errors = []
  @buf = []
  @stack = []
  super()
  if raise_errors and !@errors.empty?
    raise SyntaxError, @errors.map(&:message).join(' ;')
  end
  @buf.flatten!
  unless (result = @buf).empty?
    result.concat(@buf) until (@buf = []; super(); @buf.empty?)
  end
  result
end

#scan(**kw) ⇒ Object

parse the code and returns elements including errors.



114
115
116
117
118
119
120
121
122
123
124
125
126
 
# File 'lib/ripper/lexer.rb', line 114

def scan(**kw)
  result = (parse(**kw) + errors + @stack.flatten).uniq.sort_by {|e| [*e.pos, (e.message ? -1 : 0)]}
  result.each_with_index do |e, i|
    if e.event == :on_parse_error and e.tok.empty? and (pre = result[i-1]) and
      pre.pos[0] == e.pos[0] and (pre.pos[1] + pre.tok.size) == e.pos[1]
      e.tok = pre.tok
      e.pos[1] = pre.pos[1]
      result[i-1] = e
      result[i] = pre
    end
  end
  result
end

#tokenize(**kw) ⇒ Object 



105
106
107
 
# File 'lib/ripper/lexer.rb', line 105

def tokenize(**kw)
  parse(**kw).sort_by(&:pos).map(&:tok)
end