#pragma once
#include <string>
#include <token/token.hpp>
#include <vector>

namespace Token
{
	class Tokenizer
	{
	private:
		// Argument for string should never be of length zero
		Token ExtractToken(
			std::string const & string,
			std::size_t const lineNumber,
			std::size_t const lineColumn) const;

	public:
		void Tokenize(
			std::string const & line,
			std::size_t const lineNumber,
			std::vector<Token> & tokens);
	};
}