Refactor tokenization

include/token/token.hpp

@@ -46,8 +46,6 @@ namespace Token
 		static Token CreateMemoryToken(RegisterType const registerType, int const lineNumber, int const lineColumn);
 		static Token CreateMemoryToken(int const value, int const lineNumber, int const lineColumn);
 
-		void DebugPrint() const;
+		void Print() const;
 	};
-
-	void PrintTokens(std::vector<Token> const & tokens);
 }

include/token/tokenizer.hpp

@@ -8,20 +8,16 @@ namespace Token
 	class Tokenizer
 	{
 	private:
-		std::vector<std::pair<std::string, std::string>> substitutions;
-
+		// argument for string should never be of length zero
 		Token ExtractToken(
-			std::string string,
-			int const lineNumber,
-			int const lineColumn) const;
-
-		void ParseCharacterLiteral(
-			std::string const & line,
-			int const lineNumber,
-			unsigned & lineColumn,
-			std::vector<Token> & tokens) const;
+			std::string const & string,
+			std::size_t const lineNumber,
+			std::size_t const lineColumn) const;
 
 	public:
-		void Tokenize(std::string const & line, int const lineNumber, std::vector<Token> & tokens);
+		void Tokenize(
+			std::string const & line,
+			std::size_t const lineNumber,
+			std::vector<Token> & tokens);
 	};
 }