Refactor tokenization

include/interpret/errors.hpp

@@ -8,7 +8,6 @@ namespace Interpret
 	struct InterpretationError : public std::exception
 	{
 		Token::Token errorToken;
-		std::string errorMsg;
 		InterpretationError(Token::Token const & token, std::string const & msg);
 	};
 

include/token/token.hpp

@@ -46,8 +46,6 @@ namespace Token
 		static Token CreateMemoryToken(RegisterType const registerType, int const lineNumber, int const lineColumn);
 		static Token CreateMemoryToken(int const value, int const lineNumber, int const lineColumn);
 
-		void DebugPrint() const;
+		void Print() const;
 	};
-
-	void PrintTokens(std::vector<Token> const & tokens);
 }

include/token/tokenizer.hpp

@@ -8,20 +8,16 @@ namespace Token
 	class Tokenizer
 	{
 	private:
-		std::vector<std::pair<std::string, std::string>> substitutions;
-
+		// argument for string should never be of length zero
 		Token ExtractToken(
-			std::string string,
-			int const lineNumber,
-			int const lineColumn) const;
-
-		void ParseCharacterLiteral(
-			std::string const & line,
-			int const lineNumber,
-			unsigned & lineColumn,
-			std::vector<Token> & tokens) const;
+			std::string const & string,
+			std::size_t const lineNumber,
+			std::size_t const lineColumn) const;
 
 	public:
-		void Tokenize(std::string const & line, int const lineNumber, std::vector<Token> & tokens);
+		void Tokenize(
+			std::string const & line,
+			std::size_t const lineNumber,
+			std::vector<Token> & tokens);
 	};
 }

include/utils.hpp

@@ -1,6 +1,18 @@
 #pragma once
+#include <optional>
+#include <string>
 
 namespace Utils
 {
 	bool isWhitespaceCharacter(char const c);
+
+	// Returns nullopt in case the value is missing its terminator character
+	std::optional<std::string> getValueSurroundedBy(
+		std::string const & src,
+		std::size_t const pos,
+		char const surroundingCharacter);
+
+	std::string getValueSurroundedByWhitespace(
+		std::string const & src,
+		std::size_t const pos);
 }

include/wassembler.hpp

@@ -10,13 +10,16 @@ private:
 	Configuration config;
 	Execute::VirtualMachine vm;
 	bool printSubstitutions;
+	bool printTokens;
 
 	bool LoadLinesFromFile(std::string const & filePath, std::vector<std::string> & lines) const;
 	bool LoadTokens(std::vector<std::string> const & lines, std::vector<Token::Token> & tokens) const;
 
 public:
 	void SetMemorySize(unsigned const size);
+
 	void EnableSubstitutionsLogging();
+	void EnableTokensLogging();
 
 	bool LoadFromFile(std::string const & filePath);