Add string literals

src/token/errors.cpp

@@ -0,0 +1,15 @@
+#include <token/errors.hpp>
+
+namespace Token
+{
+	TokenizationError::TokenizationError(Token const & token, std::string const & msg)
+		: errorToken(token),
+		errorMsg(msg)
+	{
+	}
+
+	MissingEndOfString::MissingEndOfString(Token const & token)
+		: TokenizationError(token, "Missing string terminator (\")")
+	{
+	}
+}

src/token/token.cpp

@@ -103,6 +103,11 @@ namespace Token
 		return Token(TokenType::Memory, value, isValid, lineNumber, lineColumn);
 	}
 
+	Token Token::CreateStringLiteralToken(std::string const & value, int const lineNumber, int const lineColumn)
+	{
+		return Token(TokenType::String, value.substr(1, value.size() - 2), true, lineNumber, lineColumn);
+	}
+
 	void Token::DebugPrint() const
 	{
 		std::putc(' ', stdout);
@@ -188,6 +193,10 @@ namespace Token
 			}
 			break;
 
+			case TokenType::String:
+			std::printf("STRING=\"%s\"", std::get<std::string>(data).c_str());
+			break;
+
 			case TokenType::Unknown:
 			default:
 			std::printf("UNKNOWN_TOKEN");

src/token/tokenizer.cpp

@@ -1,5 +1,6 @@
 #include <map>
 #include <stdexcept>
+#include <token/errors.hpp>
 #include <token/tokenizer.hpp>
 
 namespace Token
@@ -9,7 +10,7 @@ namespace Token
 		return c == '\n' || c == ' ' || c == '\t' || c == '\r';
 	}
 
-	std::tuple<int, bool> TryParse(std::string const & string)
+	std::tuple<int, bool> TryParseInt(std::string const & string)
 	{
 		try
 		{
@@ -30,20 +31,22 @@ namespace Token
 		}
 
 		char const prefix = string[0];
-		if (prefix == '$')
+		switch(prefix)
 		{
-			auto const result = TryParse(string.substr(1, string.size()));
-			return Token::CreateImmediateValueToken(std::get<0>(result), std::get<1>(result), lineNumber, lineColumn);
-		}
+			case '$':
+			{
+				auto const result = TryParseInt(string.substr(1, string.size()));
+				return Token::CreateImmediateValueToken(std::get<0>(result), std::get<1>(result), lineNumber, lineColumn);
+			}
 
-		if (prefix == '%')
-		{
+			case '%':
 			return Token::CreateRegisterToken(GetRegisterType(string.substr(1, string.size())), lineNumber, lineColumn);
-		}
 
-		if (prefix == ';')
-		{
+			case ';':
 			return Token::CreateStatementEndToken(lineNumber, lineColumn);
+
+			default:
+			break;
 		}
 
 		char const postfix = string[string.size() - 1];
@@ -64,7 +67,7 @@ namespace Token
 			std::string const valueString = string.substr(2, string.size() - 3u);
 			if (memoryPrefix == '$')
 			{
-				auto const result = TryParse(valueString);
+				auto const result = TryParseInt(valueString);
 				return Token::CreateMemoryToken(std::get<0>(result), std::get<1>(result), lineNumber, lineColumn);
 			}
 			else if (memoryPrefix == '%')
@@ -97,8 +100,10 @@ namespace Token
 		enum class TokenizerState
 		{
 			LookForNextToken,
+			LookForStringEnd,
 			LookForTokenEnd,
 		};
+
 		TokenizerState state = TokenizerState::LookForNextToken;
 		unsigned columnTokenStart = 0;
 		for(unsigned column = 0u; column < line.size(); ++column)
@@ -115,7 +120,21 @@ namespace Token
 					}
 
 					columnTokenStart = column;
-					state = TokenizerState::LookForTokenEnd;
+
+					switch(line[column])
+					{
+						case '"':
+						state = TokenizerState::LookForStringEnd;
+						break;
+
+						case ';':
+						tokens.push_back(ExtractToken(line.substr(column, 1), lineNumber, column));
+						break;
+
+						default:
+						state = TokenizerState::LookForTokenEnd;
+						break;
+					}
 				}
 				break;
 
@@ -130,11 +149,29 @@ namespace Token
 					state = TokenizerState::LookForNextToken;
 				}
 				break;
+
+				case TokenizerState::LookForStringEnd:
+				if (line[column] == '"' && line[column - 1] != '\\')
+				{
+					tokens.push_back(
+						Token::CreateStringLiteralToken(line.substr(columnTokenStart, column - columnTokenStart), lineNumber, columnTokenStart));
+					state = TokenizerState::LookForNextToken;
+				}
 			}
 		}
-		if (state == TokenizerState::LookForTokenEnd)
+
+		switch(state)
 		{
+			case TokenizerState::LookForTokenEnd:
 			tokens.push_back(ExtractToken(line.substr(columnTokenStart, line.size()), lineNumber, columnTokenStart));
+			break;
+
+			case TokenizerState::LookForStringEnd:
+			throw MissingEndOfString(ExtractToken(line.substr(columnTokenStart, line.size()), lineNumber, columnTokenStart));
+
+			case TokenizerState::LookForNextToken:
+			default:
+			break;
 		}
 	}
 }