3 input = STDIN.read.split("^").reject { |tok| tok == "" }
6 token = token.sub(/[ ]*([^$]*)[$][ ]*/, '\1') # Strip spaces and the $ symbol
7 m = /([^<]*)(<([^>]*)>)?(.*)?/.match(token)
8 lemma, pos, tags = m[1], m[3], m[4]
14 tags = tags.split(">").collect { |s| s[1..s.length-1] }.join("|")
17 print "#{lemma}\t#{pos}\t#{tags}\n"