#pragma once
#include <string>
#include <token/token.hpp>
#include <vector>

namespace Token
{
	class Tokenizer
	{
	private:
		std::vector<std::pair<std::string, std::string>> substitutions;

		Token ExtractToken(
			std::string string,
			int const lineNumber,
			int const lineColumn) const;

		void ParseComment(
			std::string const & string,
			int const lineNumber,
			int const lineColumn);

		void ParseCharacterLiteral(
			std::string const & line,
			int const lineNumber,
			unsigned & lineColumn,
			std::vector<Token> & tokens) const;

	public:
		void Tokenize(std::string const & line, int const lineNumber, std::vector<Token> & tokens);
	};
}