Refactor tokenization

src/token/token.cpp

@@ -108,7 +108,12 @@ namespace Token
 
 	Token Token::CreateMemoryToken(RegisterType const registerType, int const lineNumber, int const lineColumn)
 	{
-		return Token(TokenType::Memory, registerType, registerType != RegisterType::Unknown, lineNumber, lineColumn);
+		if (registerType == RegisterType::Unknown)
+		{
+			return CreateErrorToken("Unknown register used", TokenType::Register, lineNumber, lineColumn);
+		}
+
+		return Token(TokenType::Memory, registerType, true, lineNumber, lineColumn);
 	}
 
 	Token Token::CreateMemoryToken(int const value, int const lineNumber, int const lineColumn)
@@ -116,7 +121,7 @@ namespace Token
 		return Token(TokenType::Memory, value, true, lineNumber, lineColumn);
 	}
 
-	void Token::DebugPrint() const
+	void Token::Print() const
 	{
 		std::putc(' ', stdout);
 		switch(type)
@@ -207,28 +212,4 @@ namespace Token
 			break;
 		}
 	}
-
-	void PrintTokens(std::vector<Token> const & tokens)
-	{
-		std::puts("*** Tokenization result ***");
-		unsigned statementNumber = 0u;
-		std::printf("%02u - ", statementNumber);
-		for(unsigned i = 0u; i < tokens.size(); ++i)
-		{
-			auto const & token = tokens[i];
-			token.DebugPrint();
-			if (token.type == TokenType::StatementEnd)
-			{
-				++statementNumber;
-				if (i + 1 < tokens.size())
-				{
-					std::printf("\n%02u - ", statementNumber);
-				}
-				else
-				{
-					std::puts("");
-				}
-			}
-		}
-	}
 }