Basic arithmetic and jump labels

src/token/tokenizer.cpp

@@ -0,0 +1,109 @@
+#include <map>
+#include <token/tokenizer.hpp>
+
+namespace Token
+{
+	bool IsWhiteSpace(char const c)
+	{
+		return c == '\n' || c == ' ' || c == '\t' || c == '\r';
+	}
+
+	Token ExtractToken(std::string const & string, int const lineNumber, int const lineColumn)
+	{
+		if (string.size() == 0)
+		{
+			return Token(lineNumber, lineColumn);
+		}
+
+		char const prefix = string[0];
+		if (prefix == '$')
+		{
+			int value = 0;
+			try
+			{
+				value = std::stoi(string.substr(1, string.size()));
+			}
+			catch(std::invalid_argument &)
+			{
+				return Token(lineNumber, lineColumn, 0, false);
+			}
+
+			return Token(lineNumber, lineColumn, value, true);
+		}
+
+		if (prefix == '%')
+		{
+			RegisterType const rtype = GetRegisterType(string.substr(1, string.size()));
+			return Token(lineNumber, lineColumn, rtype, rtype != RegisterType::Unknown);
+		}
+
+		if (prefix == ';')
+		{
+			Token token(lineNumber, lineColumn);
+			token.type = TokenType::StatementEnd;
+			token.isValid = true;
+			return token;
+		}
+
+		char const postfix = string[string.size() - 1];
+		if (postfix == ':')
+		{
+			return Token(lineNumber, lineColumn, string.substr(0, string.size() - 1), true);
+		}
+
+		OperandType const opType = GetOperandType(string);
+		if (opType != OperandType::Unknown)
+		{
+			return Token(lineNumber, lineColumn, opType, true);
+		}
+
+		// Last resort: it must be a label
+		return Token(lineNumber, lineColumn, string, true);
+	}
+
+	void Tokenizer::Tokenize(std::string const & line, int const lineNumber, std::vector<Token> & tokens)
+	{
+		enum class TokenizerState
+		{
+			LookForNextToken,
+			LookForTokenEnd,
+		};
+		TokenizerState state = TokenizerState::LookForNextToken;
+		unsigned tokenStart = 0;
+		for(unsigned i = 0u; i < line.size(); ++i)
+		{
+			switch(state)
+			{
+				case TokenizerState::LookForNextToken:
+				if (!IsWhiteSpace(line[i]))
+				{
+					if (line[i] == '#')
+					{
+						// Ignore comments
+						return;
+					}
+
+					tokenStart = i;
+					state = TokenizerState::LookForTokenEnd;
+				}
+				break;
+
+				case TokenizerState::LookForTokenEnd:
+				if (IsWhiteSpace(line[i]) || line[i] == ';')
+				{
+					tokens.push_back(ExtractToken(line.substr(tokenStart, i - tokenStart), lineNumber, tokenStart));
+					if (line[i] == ';')
+					{
+						tokens.push_back(ExtractToken(line.substr(i, 1), lineNumber, tokenStart));
+					}
+					state = TokenizerState::LookForNextToken;
+				}
+				break;
+			}
+		}
+		if (state == TokenizerState::LookForTokenEnd)
+		{
+			tokens.push_back(ExtractToken(line.substr(tokenStart, line.size()), lineNumber, tokenStart));
+		}
+	}
+}