#pragma once
#include <string>
#include <token/token.hpp>
#include <vector>

namespace Token
{
	class Tokenizer
	{
	private:
		std::vector<std::pair<std::string, std::string>> substitutions;

		Token ExtractToken(std::string string, int const lineNumber, int const lineColumn) const;
		void ParseComment(std::string const & string, int const lineNumber, int const lineColumn);

	public:
		void Tokenize(std::string const & line, int const lineNumber, std::vector<Token> & tokens);
	};
}