token.go 986 B

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788
  1. package lexer
  2. import "fmt"
  3. type TokenType int
  4. const (
  5. EOF TokenType = iota
  6. Error
  7. Text
  8. Char
  9. Any
  10. Super
  11. Single
  12. Not
  13. Separator
  14. RangeOpen
  15. RangeClose
  16. RangeLo
  17. RangeHi
  18. RangeBetween
  19. TermsOpen
  20. TermsClose
  21. )
  22. func (tt TokenType) String() string {
  23. switch tt {
  24. case EOF:
  25. return "eof"
  26. case Error:
  27. return "error"
  28. case Text:
  29. return "text"
  30. case Char:
  31. return "char"
  32. case Any:
  33. return "any"
  34. case Super:
  35. return "super"
  36. case Single:
  37. return "single"
  38. case Not:
  39. return "not"
  40. case Separator:
  41. return "separator"
  42. case RangeOpen:
  43. return "range_open"
  44. case RangeClose:
  45. return "range_close"
  46. case RangeLo:
  47. return "range_lo"
  48. case RangeHi:
  49. return "range_hi"
  50. case RangeBetween:
  51. return "range_between"
  52. case TermsOpen:
  53. return "terms_open"
  54. case TermsClose:
  55. return "terms_close"
  56. default:
  57. return "undef"
  58. }
  59. }
  60. type Token struct {
  61. Type TokenType
  62. Raw string
  63. }
  64. func (t Token) String() string {
  65. return fmt.Sprintf("%v<%q>", t.Type, t.Raw)
  66. }