Refactor tokenization

src/wassembler.cpp

@@ -21,7 +21,6 @@ void PrintBadToken(Token::Token const & token, std::vector<std::string> const &
 
 void PrintTokenError(Interpret::InterpretationError const & err, std::vector<std::string> const & lines)
 {
-	std::printf("%s ", err.errorMsg.c_str());
 	PrintBadToken(err.errorToken, lines);
 }
 
@@ -66,6 +65,24 @@ bool Wassembler::LoadTokens(std::vector<std::string> const & lines, std::vector<
 		}
 	}
 
+	if (printTokens && tokens.size() > 0)
+	{
+		int previousLine = tokens[0].lineNumber;
+		std::printf("Line %04i: ", previousLine);
+		for(auto const & token : tokens)
+		{
+			if (token.lineNumber != previousLine)
+			{
+				std::putc('\n', stdout);
+				previousLine = token.lineNumber;
+				std::printf("Line %04i: ", previousLine);
+			}
+
+			token.Print();
+		}
+		std::putc('\n', stdout);
+	}
+
 	// Validate the syntax
 	bool syntaxError = false;
 	for(auto const & token : tokens)
@@ -91,6 +108,11 @@ void Wassembler::EnableSubstitutionsLogging()
 	printSubstitutions = true;
 }
 
+void Wassembler::EnableTokensLogging()
+{
+	printTokens = true;
+}
+
 bool Wassembler::LoadFromFile(std::string const & filePath)
 {
 	std::vector<std::string> lines;
@@ -122,6 +144,7 @@ bool Wassembler::LoadFromFile(std::string const & filePath)
 	}
 	catch(Interpret::InterpretationError & e)
 	{
+		std::printf("Semantic error ");
 		PrintBadToken(e.errorToken, lines);
 		std::puts("Aborting due to semantic error(s)");
 		return false;