Basic arithmetic and jump labels

src/execute/state.cpp

@@ -0,0 +1,11 @@
+#include <execute/state.hpp>
+
+namespace Execute
+{
+	State::State(std::unordered_map<std::string, unsigned> const & _labelStatementIndice)
+		: currentStatement(0u),
+		nextStatement(1u),
+		labelStatementIndice(_labelStatementIndice)
+	{
+	}
+}

src/execute/virtualmachine.cpp

@@ -0,0 +1,52 @@
+#include <execute/virtualmachine.hpp>
+
+namespace Execute
+{
+	void VirtualMachine::Step()
+	{
+		state.nextStatement = state.currentStatement + 1u;
+		
+		code.statements[state.currentStatement]->Execute(flags, state, registers);
+		
+		state.currentStatement = state.nextStatement;
+		if (state.currentStatement >= code.statements.size())
+		{
+			terminated = true;
+		}
+	}
+
+	void VirtualMachine::Run()
+	{
+		while(!IsTerminated())
+		{
+			Step();
+		}
+	}
+
+	void VirtualMachine::SingleStep()
+	{
+		if(!IsTerminated())
+		{
+			Step();
+		}
+	}
+
+	Flags const & VirtualMachine::GetFlags() const { return flags; }
+	Registers const & VirtualMachine::GetRegisters() const { return registers; }
+	State const & VirtualMachine::GetState() const { return state; }
+	Interpret::Statement const * const VirtualMachine::GetCurrentStatement() const
+	{
+		return code.statements[state.currentStatement].get();
+	}
+
+	bool VirtualMachine::IsTerminated() const { return terminated; } 
+
+	VirtualMachine::VirtualMachine(Interpret::Code const & _code)
+		: flags(),
+		registers(),
+		state(_code.labelStatementIndice),
+		terminated(false),
+		code(_code)
+	{
+	}
+}

src/interpret/error.cpp

@@ -0,0 +1,58 @@
+#include <interpret/errors.hpp>
+
+namespace Interpret
+{
+	TokenError::TokenError(Token::Token const & token, std::string const & msg)
+		: errorToken(token),
+		errorMsg(msg)
+	{
+	}
+
+	ExpectedArgument::ExpectedArgument(Token::Token const & token)
+		: TokenError(token, "Expected an argument")
+	{
+	}
+
+	ExpectedLabel::ExpectedLabel(Token::Token const & token)
+		: TokenError(token, "Expected a label")
+	{
+	}
+
+	ExpectedImmediate::ExpectedImmediate(Token::Token const & token)
+		: TokenError(token, "Expected an immediate value")
+	{
+	}
+
+	ExpectedRegister::ExpectedRegister(Token::Token const & token)
+		: TokenError(token, "Expected a register")
+	{
+	}
+
+	ExpectedOperand::ExpectedOperand(Token::Token const & token)
+		: TokenError(token, "Expected an operand")
+	{
+	}
+
+	TooManyArguments::TooManyArguments(Token::Token const & token)
+		: TokenError(token, "Too many arguments for operand")
+	{
+	}
+
+	TooFewArguments::TooFewArguments(Token::Token const & token)
+		: TokenError(token, "Too few arguments for operand")
+	{
+	}
+
+	MissingEndOfStatment::MissingEndOfStatment(Token::Token const & token)
+		: TokenError(token, "Missing end of line terminator (;)")
+	{
+	}
+
+	namespace Internal
+	{
+		BadTokenForValue::BadTokenForValue(Token::Token const & token)
+			: TokenError(token, "Internal error when converting token to value")
+		{
+		}
+	}
+}

src/interpret/interpreter.cpp

@@ -0,0 +1,77 @@
+#include <interpret/errors.hpp>
+#include <interpret/interpreter.hpp>
+#include <interpret/operanddefinitions.hpp>
+
+namespace Interpret
+{
+	void Interpreter::Interpret(std::vector<Token::Token> const & tokens, Code & code)
+	{
+		enum class InterpreterState
+		{
+			FindOperand,
+			FindArguments,
+			FindStatementEnd
+		};
+
+		InterpreterState state = InterpreterState::FindOperand;
+		unsigned operatorTokenIndex = 0u;
+		int expectedNumberOfArguments = 0;
+		for(unsigned i = 0u; i < tokens.size(); ++i)
+		{
+			auto const & token = tokens[i];
+			switch(state)
+			{
+				case InterpreterState::FindOperand:
+				if (token.type == Token::TokenType::Operand)
+				{
+					operatorTokenIndex = i;
+					expectedNumberOfArguments = GetRequiredNumberOfArguments(std::get<Token::OperandType>(token.data));
+					if (expectedNumberOfArguments < 1)
+					{
+						state = InterpreterState::FindStatementEnd;
+					}
+					else
+					{
+						state = InterpreterState::FindArguments;
+					}
+				}
+				else if (token.type == Token::TokenType::Label)
+				{
+					code.labelStatementIndice[std::get<std::string>(token.data)] = code.statements.size();
+				}
+				else if (token.type != Token::TokenType::StatementEnd)
+				{
+					throw ExpectedOperand(token);
+				}
+				break;
+
+				case InterpreterState::FindArguments:
+				if (token.type == Token::TokenType::ImmediateInteger || token.type == Token::TokenType::Register || token.type == Token::TokenType::Label)
+				{
+					expectedNumberOfArguments -= 1;
+					if (expectedNumberOfArguments < 1)
+					{
+						state = InterpreterState::FindStatementEnd;
+					}
+				}
+				else
+				{
+					throw ExpectedArgument(token);
+				}
+				break;
+
+				case InterpreterState::FindStatementEnd:
+				if (token.type != Token::TokenType::StatementEnd)
+				{
+					throw MissingEndOfStatment(token);
+				}
+				else
+				{
+					code.statements.emplace_back(ExtractStatement(operatorTokenIndex, tokens));
+					state = InterpreterState::FindOperand;
+				}
+				break;
+			}
+		}
+	}
+}

src/interpret/operanddefinitions.cpp

@@ -0,0 +1,165 @@
+#include <interpret/operanddefinitions.hpp>
+#include <interpret/errors.hpp>
+
+namespace Interpret
+{
+	Value GetImmediateArgument(unsigned const index, std::vector<Token::Token> const & tokens)
+	{
+		auto const & token = tokens[index];
+		if (token.type == Token::TokenType::ImmediateInteger)
+		{
+			Value v;
+			v.CreateFromToken(token);
+
+			return v;
+		}
+
+		throw ExpectedImmediate(token);
+	}
+
+	Value GetRegisterArgument(unsigned const index, std::vector<Token::Token> const & tokens)
+	{
+		auto const & token = tokens[index];
+		if (token.type == Token::TokenType::Register)
+		{
+			Value v;
+			v.CreateFromToken(token);
+
+			return v;
+		}
+
+		throw ExpectedRegister(token);
+	}
+
+	Value GetImmediateOrRegisterArgument(unsigned const index, std::vector<Token::Token> const & tokens)
+	{
+		auto const & token = tokens[index];
+		if (token.type == Token::TokenType::ImmediateInteger || token.type == Token::TokenType::Register)
+		{
+			Value v;
+			v.CreateFromToken(token);
+
+			return v;
+		}
+
+		throw ExpectedRegister(token);
+	}
+
+	void AddArithmeticArguments(ThreeArgumentStatement & statement, unsigned const operandIndex, std::vector<Token::Token> const & tokens)
+	{
+		statement.firstArgument = GetImmediateOrRegisterArgument(operandIndex + 1u, tokens);
+		statement.secondArgument = GetImmediateOrRegisterArgument(operandIndex + 2u, tokens);
+		statement.thirdArgument = GetRegisterArgument(operandIndex + 3u, tokens);
+	}
+
+	std::unique_ptr<Statement> ExtractStatement(unsigned const operandIndex, std::vector<Token::Token> const & tokens)
+	{
+		auto const & token = tokens[operandIndex];
+		switch(std::get<Token::OperandType>(token.data))
+		{
+			case Token::OperandType::AddInteger:
+			{
+				auto statement = std::make_unique<ThreeArgumentStatement>();
+				statement->function = [](Execute::Flags & flags, int argument1, int argument2, int & argument3)
+				{
+					argument3 = argument1 + argument2;
+				};
+				AddArithmeticArguments(*statement, operandIndex, tokens);
+				return statement;
+			}
+
+			case Token::OperandType::SubtractInteger:
+			{
+				auto statement = std::make_unique<ThreeArgumentStatement>();
+				statement->function = [](Execute::Flags & flags, int argument1, int argument2, int & argument3)
+				{
+					argument3 = argument1 - argument2;
+				};
+				AddArithmeticArguments(*statement, operandIndex, tokens);
+				return statement;
+			}
+
+			case Token::OperandType::DivideInteger:
+			{
+				auto statement = std::make_unique<ThreeArgumentStatement>();
+				statement->function = [](Execute::Flags & flags, int argument1, int argument2, int & argument3)
+				{
+					argument3 = argument1 / argument2;
+				};
+				AddArithmeticArguments(*statement, operandIndex, tokens);
+				return statement;
+			}
+
+			case Token::OperandType::MultiplyInteger:
+			{
+				auto statement = std::make_unique<ThreeArgumentStatement>();
+				statement->function = [](Execute::Flags & flags, int argument1, int argument2, int & argument3)
+				{
+					argument3 = argument1 * argument2;
+				};
+				AddArithmeticArguments(*statement, operandIndex, tokens);
+				return statement;
+			}
+
+			case Token::OperandType::ShiftIntegerLeft:
+			{
+				auto statement = std::make_unique<ThreeArgumentStatement>();
+				statement->function = [](Execute::Flags & flags, int argument1, int argument2, int & argument3)
+				{
+					argument3 = argument1 << argument2;
+				};
+				AddArithmeticArguments(*statement, operandIndex, tokens);
+				return statement;
+			}
+
+			case Token::OperandType::ShiftIntegerRight:
+			{
+				auto statement = std::make_unique<ThreeArgumentStatement>();
+				statement->function = [](Execute::Flags & flags, int argument1, int argument2, int & argument3)
+				{
+					argument3 = argument1 >> argument2;
+				};
+				AddArithmeticArguments(*statement, operandIndex, tokens);
+				return statement;
+			}
+
+			case Token::OperandType::Jump:
+			{
+				auto labelToken = tokens[operandIndex + 1u];
+				if (labelToken.type != Token::TokenType::Label)
+				{
+					throw ExpectedLabel(labelToken);
+				}
+				return std::make_unique<JumpStatement>(std::get<std::string>(labelToken.data));
+			}
+
+			default:
+			{
+				auto statement = std::make_unique<NoArgumentStatement>();
+				// TODO throw error?
+				statement->function = [](Execute::Flags & flags, Execute::Registers & registers) { std::puts("ExtractStatement: Extracted unhandled operator type"); };
+				return statement;
+			}
+		}
+	}
+
+	int GetRequiredNumberOfArguments(Token::OperandType const type)
+	{
+		switch (type)
+		{
+			case Token::OperandType::AddInteger:
+			case Token::OperandType::SubtractInteger:
+			case Token::OperandType::DivideInteger:
+			case Token::OperandType::MultiplyInteger:
+			case Token::OperandType::ShiftIntegerLeft:
+			case Token::OperandType::ShiftIntegerRight:
+			return 3;
+
+			case Token::OperandType::Jump:
+			return 1;
+
+			default:
+			return 0;
+		}
+	}
+}

src/interpret/statement.cpp

@@ -0,0 +1,42 @@
+#include <interpret/statement.hpp>
+
+namespace Interpret
+{
+	void NoArgumentStatement::Execute(Execute::Flags & flags, Execute::State & state, Execute::Registers & registers)
+	{
+		function(flags, registers);
+	}
+
+	void OneArgumentStatement::Execute(Execute::Flags & flags, Execute::State & state, Execute::Registers & registers)
+	{
+		function(flags, firstArgument.GetValue(registers));
+	}
+
+	void TwoArgumentStatement::Execute(Execute::Flags & flags, Execute::State & state, Execute::Registers & registers)
+	{
+		function(flags, firstArgument.GetValue(registers), secondArgument.GetValue(registers));
+	}
+
+	void ThreeArgumentStatement::Execute(Execute::Flags & flags, Execute::State & state, Execute::Registers & registers)
+	{
+		function(flags, firstArgument.GetValue(registers), secondArgument.GetValue(registers), thirdArgument.GetValue(registers));
+	}
+
+	void JumpStatement::Execute(Execute::Flags & flags, Execute::State & state, Execute::Registers & registers)
+	{
+		auto const & elem = state.labelStatementIndice.find(label);
+		if (elem != state.labelStatementIndice.end())
+		{
+			state.nextStatement = elem->second;
+		}
+		else
+		{
+			throw std::runtime_error("Attempted jump to nonexisting label");
+		}
+	}
+
+	JumpStatement::JumpStatement(std::string const & _label)
+		: label(_label)
+	{
+	}
+}

src/interpret/value.cpp

@@ -0,0 +1,33 @@
+#include <interpret/errors.hpp>
+#include <interpret/value.hpp>
+
+namespace Interpret
+{
+	int & Value::GetValue(Execute::Registers & registers)
+	{
+		if (type == ValueType::ImmediateInteger)
+		{
+			return integer;
+		}
+
+		return registers.registers[registerIndex];
+	}
+
+	void Value::CreateFromToken(Token::Token const & token)
+	{
+		if (token.type == Token::TokenType::ImmediateInteger)
+		{
+			type = ValueType::ImmediateInteger;
+			integer = std::get<int>(token.data);
+		}
+		else if (token.type == Token::TokenType::Register)
+		{
+			type = ValueType::Register;
+			registerIndex = static_cast<int>(std::get<Token::RegisterType>(token.data));
+		}
+		else
+		{
+			throw Internal::BadTokenForValue(token);
+		}
+	}
+}

src/main.cpp

@@ -0,0 +1,133 @@
+#include <cstdio>
+#include <execute/virtualmachine.hpp>
+#include <fstream>
+#include <interpret/errors.hpp>
+#include <interpret/interpreter.hpp>
+#include <token/tokenizer.hpp>
+
+void PrintBadToken(Token::Token const & token, std::vector<std::string> const & lines)
+{
+	std::printf("at line number %i, column %i:\n",
+		token.lineNumber + 1,
+		token.lineColumn + 1);
+	std::printf("%s\n", lines[token.lineNumber].c_str());
+	for(int i = 0; i < token.lineColumn; ++i)
+	{
+		std::putc(' ', stdout);
+	}
+	std::puts("^");
+}
+
+void PrintTokenError(Interpret::TokenError const & err, std::vector<std::string> const & lines)
+{
+	std::printf("%s ", err.errorMsg.c_str());
+	PrintBadToken(err.errorToken, lines);
+}
+
+int main(int argc, char ** argv)
+{
+	if (argc != 2)
+	{
+		std::puts("Usage: wassembly [filename.wasm]");
+		return 1;
+	}
+
+	std::ifstream input(argv[1]);
+	if (!input.is_open())
+	{
+		std::printf("Error: Cannot open file %s for reading", argv[1]);
+		return 1;
+	}
+
+	std::vector<Token::Token> tokens;
+	Token::Tokenizer tokenizer;
+	std::vector<std::string> lines; // DEBUG
+	std::string line;
+	unsigned lineNumber = 0;
+	while(std::getline(input, line))
+	{
+		tokenizer.Tokenize(line, lineNumber, tokens);
+		++lineNumber;
+		lines.push_back(line); // DEBUG
+	}
+	input.close();
+
+	// DEBUG
+	std::puts("*** Tokenization result ***");
+	unsigned statementNumber = 0u;
+	std::printf("%02u - ", statementNumber);
+	for(unsigned i = 0u; i < tokens.size(); ++i)
+	{
+		auto const & token = tokens[i];
+		token.DebugPrint();
+		if (token.type == Token::TokenType::StatementEnd)
+		{
+			++statementNumber;
+			if (i + 1 < tokens.size())
+			{
+				std::printf("\n%02u - ", statementNumber);
+			}
+			else
+			{
+				std::puts("");
+			}
+		}
+	}
+	// END DEBUG
+
+	// Validate the syntax
+	bool syntaxOk = true;
+	for(auto const & token : tokens)
+	{
+		if (!token.isValid)
+		{
+			std::printf("Syntax error ");
+			PrintBadToken(token, lines);
+			syntaxOk = false;
+		}
+	}
+
+	if (!syntaxOk)
+	{
+		std::puts("Aborting due to syntax error(s)");
+		return 1;
+	}
+
+	Interpret::Interpreter interpreter;
+	Interpret::Code code;
+	try
+	{
+		interpreter.Interpret(tokens, code);
+	}
+	catch(Interpret::TokenError & e)
+	{
+		PrintTokenError(e, lines);
+		return 1;
+	}
+
+	// DEBUG
+	std::puts("\n*** Labels ***");
+	for(auto const & labelIndice : code.labelStatementIndice)
+	{
+		std::printf("Label %s points to statement %u\n", labelIndice.first.c_str(), labelIndice.second);
+	}
+
+	std::puts("\n*** Execution ***");
+	Execute::VirtualMachine vm(code);
+	for(unsigned i = 0u; i < 900000; ++i)
+	{
+		vm.SingleStep();
+		auto const & registers = vm.GetRegisters();
+		std::printf("A=%i B=%i C=%i D=%i\n", registers.A, registers.B, registers.C, registers.D);
+		auto const & flags = vm.GetFlags();
+		std::printf("-\n"); // TODO
+		auto const & state = vm.GetState();
+		std::printf("current_statement=%i\n", state.currentStatement);
+
+		std::puts("Press any key to step...");
+		std::getchar();
+	}
+	// END DEBUG
+	
+	return 0;
+}

src/token/operandtype.cpp

@@ -0,0 +1,27 @@
+#include <map>
+#include <token/operandtype.hpp>
+
+namespace Token
+{
+	OperandType GetOperandType(std::string const & op)
+	{
+		static std::map<std::string, OperandType, std::less<>> const operations =
+		{
+			{ "addi", OperandType::AddInteger },
+			{ "subi", OperandType::SubtractInteger },
+			{ "divi", OperandType::DivideInteger },
+			{ "muli", OperandType::MultiplyInteger },
+			{ "shri", OperandType::ShiftIntegerRight },
+			{ "shli", OperandType::ShiftIntegerLeft },
+			{ "jmp", OperandType::Jump }
+		};
+
+		auto const & result = operations.find(op);
+		if (result != operations.end())
+		{
+			return result->second;
+		}
+
+		return OperandType::Unknown;
+	}
+}

src/token/registertype.cpp

@@ -0,0 +1,24 @@
+#include <token/registertype.hpp>
+#include <map>
+
+namespace Token
+{
+	RegisterType GetRegisterType(std::string const & reg)
+	{
+		static std::map<std::string, RegisterType, std::less<>> const registers =
+		{
+			{ "A", RegisterType::A },
+			{ "B", RegisterType::B },
+			{ "C", RegisterType::C },
+			{ "D", RegisterType::D }
+		};
+
+		auto const & result = registers.find(reg);
+		if (result != registers.end())
+		{
+			return result->second;
+		}
+
+		return RegisterType::Unknown;
+	}
+}

src/token/token.cpp

@@ -0,0 +1,165 @@
+#include <cstdio>
+#include <token/token.hpp>
+
+namespace Token
+{
+	Token::Token(int const _lineNumber, int const _lineColumn)
+		: lineNumber(_lineNumber),
+		lineColumn(_lineColumn),
+		type(TokenType::Unknown),
+		isValid(false)
+	{
+	}
+
+	Token::Token(int const _lineNumber, int const _lineColumn, OperandType _operatorType, bool validness)
+		: lineNumber(_lineNumber),
+		lineColumn(_lineColumn),
+		type(TokenType::Operand),
+		isValid(validness),
+		data(_operatorType)
+	{
+	}
+
+	Token::Token(int const _lineNumber, int const _lineColumn, RegisterType _registerType, bool validness)
+		: lineNumber(_lineNumber),
+		lineColumn(_lineColumn),
+		type(TokenType::Register),
+		isValid(validness),
+		data(_registerType)
+	{
+	}
+
+	Token::Token(int const _lineNumber, int const _lineColumn, int value, bool validness)
+		: lineNumber(_lineNumber),
+		lineColumn(_lineColumn),
+		type(TokenType::ImmediateInteger),
+		isValid(validness),
+		data(value)
+	{
+	}
+
+	Token::Token(int const _lineNumber, int const _lineColumn, std::string const & value, bool validness)
+		: lineNumber(_lineNumber),
+		lineColumn(_lineColumn),
+		type(TokenType::Label),
+		isValid(validness),
+		data(value)
+	{
+	}
+
+	Token::Token(Token const & other)
+		: lineNumber(other.lineNumber),
+		lineColumn(other.lineColumn),
+		type(other.type),
+		isValid(other.isValid),
+		data(other.data)
+	{
+	}
+
+	void Token::DebugPrint() const
+	{
+		std::putc(' ', stdout);
+		switch(type)
+		{
+			case TokenType::ImmediateInteger:
+			if (isValid)
+			{
+				std::printf("$int=%i", std::get<int>(data));
+			}
+			else
+			{
+				std::printf("BAD_IMM_INT");
+			}
+			break;
+
+			case TokenType::Operand:
+			if (isValid)
+			{
+				switch(std::get<OperandType>(data))
+				{
+					case OperandType::AddInteger:
+					std::printf("addi");
+					break;
+
+					case OperandType::MultiplyInteger:
+					std::printf("muli");
+					break;
+
+					case OperandType::SubtractInteger:
+					std::printf("subi");
+					break;
+
+					case OperandType::DivideInteger:
+					std::printf("divi");
+					break;
+
+					case OperandType::ShiftIntegerLeft:
+					std::printf("shli");
+					break;
+
+					case OperandType::ShiftIntegerRight:
+					std::printf("shri");
+					break;
+
+					case OperandType::Jump:
+					std::printf("jump");
+					break;
+
+					default:
+					std::printf("unknown_op");
+					break;
+				}
+			}
+			else
+			{
+				std::printf("BAD_OP");
+			}
+			break;
+
+			case TokenType::Register:
+			if (isValid)
+			{
+				switch(std::get<RegisterType>(data))
+				{
+					case RegisterType::A:
+					std::printf("%%A");
+					break;
+
+					case RegisterType::B:
+					std::printf("%%B");
+					break;
+
+					case RegisterType::C:
+					std::printf("%%C");
+					break;
+
+					case RegisterType::D:
+					std::printf("%%D");
+					break;
+
+					default:
+					std::printf("%%unknown_reg");
+					break;
+				}
+			}
+			else
+			{
+				std::printf("BAD_REG");
+			}
+			break;
+
+			case TokenType::StatementEnd:
+			std::printf("EOS");
+			break;
+
+			case TokenType::Label:
+			std::printf("label=%s", std::get<std::string>(data).c_str());
+			break;
+
+			case TokenType::Unknown:
+			default:
+			std::printf("UNKNOWN_TOKEN");
+			break;
+		}
+	}
+}

src/token/tokenizer.cpp

@@ -0,0 +1,109 @@
+#include <map>
+#include <token/tokenizer.hpp>
+
+namespace Token
+{
+	bool IsWhiteSpace(char const c)
+	{
+		return c == '\n' || c == ' ' || c == '\t' || c == '\r';
+	}
+
+	Token ExtractToken(std::string const & string, int const lineNumber, int const lineColumn)
+	{
+		if (string.size() == 0)
+		{
+			return Token(lineNumber, lineColumn);
+		}
+
+		char const prefix = string[0];
+		if (prefix == '$')
+		{
+			int value = 0;
+			try
+			{
+				value = std::stoi(string.substr(1, string.size()));
+			}
+			catch(std::invalid_argument &)
+			{
+				return Token(lineNumber, lineColumn, 0, false);
+			}
+
+			return Token(lineNumber, lineColumn, value, true);
+		}
+
+		if (prefix == '%')
+		{
+			RegisterType const rtype = GetRegisterType(string.substr(1, string.size()));
+			return Token(lineNumber, lineColumn, rtype, rtype != RegisterType::Unknown);
+		}
+
+		if (prefix == ';')
+		{
+			Token token(lineNumber, lineColumn);
+			token.type = TokenType::StatementEnd;
+			token.isValid = true;
+			return token;
+		}
+
+		char const postfix = string[string.size() - 1];
+		if (postfix == ':')
+		{
+			return Token(lineNumber, lineColumn, string.substr(0, string.size() - 1), true);
+		}
+
+		OperandType const opType = GetOperandType(string);
+		if (opType != OperandType::Unknown)
+		{
+			return Token(lineNumber, lineColumn, opType, true);
+		}
+
+		// Last resort: it must be a label
+		return Token(lineNumber, lineColumn, string, true);
+	}
+
+	void Tokenizer::Tokenize(std::string const & line, int const lineNumber, std::vector<Token> & tokens)
+	{
+		enum class TokenizerState
+		{
+			LookForNextToken,
+			LookForTokenEnd,
+		};
+		TokenizerState state = TokenizerState::LookForNextToken;
+		unsigned tokenStart = 0;
+		for(unsigned i = 0u; i < line.size(); ++i)
+		{
+			switch(state)
+			{
+				case TokenizerState::LookForNextToken:
+				if (!IsWhiteSpace(line[i]))
+				{
+					if (line[i] == '#')
+					{
+						// Ignore comments
+						return;
+					}
+
+					tokenStart = i;
+					state = TokenizerState::LookForTokenEnd;
+				}
+				break;
+
+				case TokenizerState::LookForTokenEnd:
+				if (IsWhiteSpace(line[i]) || line[i] == ';')
+				{
+					tokens.push_back(ExtractToken(line.substr(tokenStart, i - tokenStart), lineNumber, tokenStart));
+					if (line[i] == ';')
+					{
+						tokens.push_back(ExtractToken(line.substr(i, 1), lineNumber, tokenStart));
+					}
+					state = TokenizerState::LookForNextToken;
+				}
+				break;
+			}
+		}
+		if (state == TokenizerState::LookForTokenEnd)
+		{
+			tokens.push_back(ExtractToken(line.substr(tokenStart, line.size()), lineNumber, tokenStart));
+		}
+	}
+}