Module: Chronic::Tokenizer

Defined in:
lib/chronic/tokenizer.rb

Class Method Summary collapse

Class Method Details

.char_type(char) ⇒ Object 



3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
 
# File 'lib/chronic/tokenizer.rb', line 3

def self.char_type(char)
  case char
  when '.'
    :period
  when /[[:alpha:]]/
    :letter
  when /[[:digit:]]/
    :digit
  when /[[:space:]]/
    :space
  when /[[:punct:]]/
    :punct
  else
    :other
  end
end

.tokenize(text) ⇒ Object

Proccess text to tokens



21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
 
# File 'lib/chronic/tokenizer.rb', line 21

def self.tokenize(text)
  tokens = []
  index = 0
  previos_index = 0
  text.each_char do |char|
    type = char_type(char)
    if type == :space
      tokens << Token.new(text[previos_index...index], text, previos_index)
      previos_index = index+1
    end
    index += 1
  end
  tokens << Token.new(text[previos_index...index], text, previos_index)
  tokens
end