Improve syntax error reporting

src/token/tokenizer.cpp

@@ -25,7 +25,8 @@ namespace Token
 	{
 		if (string.size() == 0)
 		{
-			return Token::CreateUnknownToken(lineNumber, lineColumn);
+			// TODO Should this become an error token?
+			return Token::CreateEmptyToken(lineNumber, lineColumn);
 		}
 
 		for(std::size_t i = 0; i < substitutions.size(); ++i)
@@ -48,14 +49,13 @@ namespace Token
 				{
 					return Token::CreateImmediateValueToken(
 						result.value(),
-						true,
 						lineNumber,
 						lineColumn);
 				}
 
-				return Token::CreateImmediateValueToken(
-					0,
-					false,
+				return Token::CreateErrorToken(
+					"Immediate cannot be parsed as an integer",
+					TokenType::ImmediateInteger,
 					lineNumber,
 					lineColumn);
 			}
@@ -79,7 +79,6 @@ namespace Token
 			// TODO check if label is an Operand?
 			return Token::CreateLabelToken(
 				string.substr(0, string.size() - 1),
-				true,
 				lineNumber,
 				lineColumn);
 		}
@@ -88,7 +87,11 @@ namespace Token
 		{
 			if(string.size() < 4)
 			{
-				return Token::CreateMemoryToken(0, false, lineNumber, lineColumn + 2u);
+				return Token::CreateErrorToken(
+					"Memory address statement is empty",
+					TokenType::Memory,
+					lineNumber,
+					lineColumn);
 			}
 
 			char const memoryPrefix = string[1];
@@ -101,14 +104,13 @@ namespace Token
 				{
 					return Token::CreateMemoryToken(
 						result.value(),
-						true,
 						lineNumber,
 						lineColumn);
 				}
 
-				return Token::CreateMemoryToken(
-					0,
-					false,
+				return Token::CreateErrorToken(
+					"Memory immediate address cannot be parsed as an integer",
+					TokenType::Memory,
 					lineNumber,
 					lineColumn);
 			}
@@ -121,13 +123,21 @@ namespace Token
 			}
 			else
 			{
-				return Token::CreateMemoryToken(0, false, lineNumber, lineColumn + 1u);
+				return Token::CreateErrorToken(
+					"Memory immediate address contains an unexpected value",
+					TokenType::Memory,
+					lineNumber,
+					lineColumn + 1u);
 			}
 		}
 		else if (prefix == '[' || postfix == ']')
 		{
 			int const errorLineColumn = (prefix == '[') ? lineColumn : (lineColumn + string.size() - 1u);
-			return Token::CreateMemoryToken(0, false, lineNumber, errorLineColumn);
+			return Token::CreateErrorToken(
+				"Non terminated memory address brackets",
+				TokenType::Memory,
+				lineNumber,
+				errorLineColumn);
 		}
 
 		OperandType const opType = GetOperandType(string);
@@ -137,7 +147,7 @@ namespace Token
 		}
 
 		// Last resort: it must be a label
-		return Token::CreateLabelToken(string, true, lineNumber, lineColumn);
+		return Token::CreateLabelToken(string, lineNumber, lineColumn);
 	}
 
 	bool IsWhiteSpace(char const c)
@@ -256,23 +266,20 @@ end_state_loop:
 		{
 			if (line[i] == '\'')
 			{
-				// Character literal must be exactly length 3 for now (2x ' + 1
-				// character in between)
 				if (lineColumn + 2u != i)
 				{
-					tokens.emplace_back(Token::CreateImmediateValueToken(
-						0,
-						false,
+					tokens.push_back(Token::CreateErrorToken(
+						"Character literal must be exactly 1 character long between single quotes",
+						TokenType::ImmediateInteger,
 						lineNumber,
-						lineColumn));
+						lineColumn + 1u));
 				}
 				else
 				{
-					tokens.emplace_back(Token::CreateImmediateValueToken(
+					tokens.push_back(Token::CreateImmediateValueToken(
 						line[i - 1],
-						true,
 						lineNumber,
-						lineColumn));
+						lineColumn + 1));
 				}
 
 				lineColumn = i;
@@ -280,10 +287,9 @@ end_state_loop:
 			}
 		}
 
-		// Non terminated character literal!
-		tokens.emplace_back(Token::CreateImmediateValueToken(
-			0,
-			false,
+		tokens.push_back(Token::CreateErrorToken(
+			"Non terminated character literal",
+			TokenType::ImmediateInteger,
 			lineNumber,
 			lineColumn));