Remove declare and string types to prepare for preprocessor

src/token/tokenizer.cpp

@@ -100,7 +100,6 @@ namespace Token
 		enum class TokenizerState
 		{
 			LookForNextToken,
-			LookForStringEnd,
 			LookForTokenEnd,
 		};
 
@@ -123,10 +122,6 @@ namespace Token
 
 					switch(line[column])
 					{
-						case '"':
-						state = TokenizerState::LookForStringEnd;
-						break;
-
 						case ';':
 						tokens.push_back(ExtractToken(line.substr(column, 1), lineNumber, column));
 						break;
@@ -150,13 +145,9 @@ namespace Token
 				}
 				break;
 
-				case TokenizerState::LookForStringEnd:
-				if (line[column] == '"' && line[column - 1] != '\\')
-				{
-					tokens.push_back(
-						Token::CreateStringLiteralToken(line.substr(columnTokenStart, column - columnTokenStart), lineNumber, columnTokenStart));
-					state = TokenizerState::LookForNextToken;
-				}
+				default:
+				std::puts("DEBUG: Unhandled TokenizerState value");
+				break;
 			}
 		}
 
@@ -166,9 +157,6 @@ namespace Token
 			tokens.push_back(ExtractToken(line.substr(columnTokenStart, line.size()), lineNumber, columnTokenStart));
 			break;
 
-			case TokenizerState::LookForStringEnd:
-			throw MissingEndOfString(ExtractToken(line.substr(columnTokenStart, line.size()), lineNumber, columnTokenStart));
-
 			case TokenizerState::LookForNextToken:
 			default:
 			break;