Expanding on definitions, starting the tokenizer implementation.

This commit is contained in:
Pat Garrity 2024-02-18 21:41:43 -06:00
parent bad991337d
commit c5ea827944
Signed by: pfm
GPG key ID: 5CA5D21BAB7F3A76
6 changed files with 153 additions and 23 deletions

View file

@ -33,4 +33,38 @@ object Keyword:
case object True extends Keyword("true")
case object Type extends Keyword("type")
val All: List[Keyword] = List(
Alias,
Case,
Class,
Const,
Defn,
Do,
Else,
End,
Enum,
Export,
False,
Fn,
Given,
If,
Import,
Infix,
Lambda,
Let,
Match,
Mut,
Namespace,
Object,
Private,
Record,
Return,
Then,
True,
Type
)
def isKeyword(candidate: String): Boolean =
All.contains(candidate)
end Keyword

View file

@ -0,0 +1,15 @@
package ava.parser
sealed trait Name:
def value: String
object Name:
case class UserDef(value: String) extends Name
case object AnonValue extends Name:
val value: String = "_"
case object AnonType extends Name:
val value: String = "*"
end Name

View file

@ -0,0 +1,33 @@
package ava.parser
sealed abstract class Operator(val value: String)
object Operator:
case object Hole extends Operator("???")
case object ImportSplat extends Operator("*")
case object Member extends Operator(".")
case object Union extends Operator("|")
case object ListPrepend extends Operator(":-")
case object BindType extends Operator(":")
case object BindValue extends Operator(":=")
case object FnReturn extends Operator("->")
case object BindDo extends Operator("<-")
case object ClassMember extends Operator("::")
case object Case extends Operator("=>")
val All: List[Operator] = List(
Hole,
ImportSplat,
Member,
Union,
ListPrepend,
BindType,
BindValue,
FnReturn,
BindDo,
ClassMember,
Case
)
end Operator

View file

@ -1,6 +1,52 @@
package ava.parser
case class Token(
value: String,
tokenType: TokenType
)
sealed trait Token
object Token:
/** Most tokens are generic tokens. They represent some arbitrary grouping of
* characters that will be refined later.
*
* @param value
* The token value.
*/
case class Generic(value: String) extends Token
/** Comments are detected at time of tokenization, and are arbitrary strings.
*
* @param value
* The comment value.
*/
case class Comment(value: String) extends Token
/** The '(' character.
*/
case object OpenParen extends Token
/** The ')' character.
*/
case object CloseParen extends Token
/** The '.' character.
*/
case object Dot extends Token
/** The ',' character.
*/
case object Comma extends Token
/** The '"' character.
*/
case object DoubleQuote extends Token
/** The ''' character.
*/
case object SingleQuote extends Token
/** The ':' character.
*/
case object Colon extends Token
/** The '#' character.
*/
case object Tuple extends Token
end Token

View file

@ -1,18 +0,0 @@
package ava.parser
sealed trait TokenType
object TokenType:
case object Keyword extends TokenType
case object ReservedOperator extends TokenType
case object Name extends TokenType
case object OpenParen extends TokenType
case object CloseParen extends TokenType
case object Dot extends TokenType
case object Comma extends TokenType
case object DoubleQuote extends TokenType
case object Hole extends TokenType
case object Literal extends TokenType
end TokenType

View file

@ -1,6 +1,26 @@
package ava.parser
class Tokenizer(private val reader: CharacterReader):
def next(): Option[String] = None
import Tokenizer.*
private var state: State = State.Initial
def next(): Option[Token] = None
def close(): Unit = reader.close()
object Tokenizer:
sealed trait State
object State:
case object Initial extends State
case object PotentialComment extends State
case object InComment extends State
case object InQuote extends State
case object InGeneric extends State
end State
end Tokenizer