Renamed tokenizing module for clarity.

.travis.yml

@@ -42,5 +42,5 @@ script:
   - git submodule update --recursive --init
   - mkdir -p build/debug
   - cd build/debug
-  - cmake ../.. -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_COMPILER=$_CXX -DPLASP_BUILD_TESTS=ON -DPARSEBASE_BUILD_TESTS=ON
+  - cmake ../.. -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_COMPILER=$_CXX -DPLASP_BUILD_TESTS=ON -DTOKENIZE_BUILD_TESTS=ON
-  - make -j3 plasp-app && make -j3 run-parsebase-tests && make -j3 run-tests
+  - make -j3 plasp-app && make -j3 run-tokenize-tests && make -j3 run-tests

CMakeLists.txt

@@ -24,7 +24,7 @@ if (CMAKE_GENERATOR STREQUAL "Ninja" AND
 	set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fdiagnostics-color=always")
 endif()
 
-add_subdirectory(lib/parsebase)
+add_subdirectory(lib/tokenize)
 add_subdirectory(lib/pddlparse)
 add_subdirectory(src)
 add_subdirectory(app)

app/CMakeLists.txt

@@ -6,7 +6,7 @@ file(GLOB core_headers "*.h")
 set(includes
 	${Boost_INCLUDE_DIRS}
 	${PROJECT_SOURCE_DIR}/include
-	${PROJECT_SOURCE_DIR}/lib/parsebase/include
+	${PROJECT_SOURCE_DIR}/lib/tokenize/include
 )
 
 set(sources

app/main.cpp

@@ -109,7 +109,7 @@ int main(int argc, char **argv)
 
 	try
 	{
-		parsebase::Parser<parsebase::CaseInsensitiveParserPolicy> parser;
+		tokenize::Tokenizer<tokenize::CaseInsensitiveTokenizerPolicy> tokenizer;
 
 		if (variablesMap.count("input"))
 		{
@@ -118,11 +118,11 @@ int main(int argc, char **argv)
 			std::for_each(inputFiles.cbegin(), inputFiles.cend(),
 				[&](const auto &inputFile)
 				{
-					parser.read(inputFile);
+					tokenizer.read(inputFile);
 				});
 		}
 		else
-			parser.read("std::cin", std::cin);
+			tokenizer.read("std::cin", std::cin);
 
 		const auto detectLanguage =
 			[&]()
@@ -131,7 +131,7 @@ int main(int argc, char **argv)
 				const auto language = plasp::Language::fromString(languageName);
 
 				if (language == plasp::Language::Type::Automatic)
-					return plasp::detectLanguage(parser);
+					return plasp::detectLanguage(tokenizer);
 
 				return language;
 			};
@@ -148,19 +148,19 @@ int main(int argc, char **argv)
 
 		if (language == plasp::Language::Type::PDDL)
 		{
-			auto context = plasp::pddl::Context(std::move(parser), logger);
+			auto context = plasp::pddl::Context(std::move(tokenizer), logger);
 			auto description = plasp::pddl::Description::fromContext(context);
 			const auto translator = plasp::pddl::TranslatorASP(description, logger.outputStream());
 			translator.translate();
 		}
 		else if (language == plasp::Language::Type::SAS)
 		{
-			const auto description = plasp::sas::Description::fromParser(std::move(parser));
+			const auto description = plasp::sas::Description::fromTokenizer(std::move(tokenizer));
 			const auto translator = plasp::sas::TranslatorASP(description, logger.outputStream());
 			translator.translate();
 		}
 	}
-	catch (const parsebase::ParserException &e)
+	catch (const tokenize::TokenizerException &e)
 	{
 		logger.log(plasp::output::Priority::Error, e.location(), e.message().c_str());
 		return EXIT_FAILURE;

include/plasp/LanguageDetection.h

@@ -3,7 +3,7 @@
 
 #include <plasp/Language.h>
 
-#include <parsebase/Parser.h>
+#include <tokenize/Tokenizer.h>
 
 namespace plasp
 {
@@ -14,32 +14,32 @@ namespace plasp
 //
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-Language::Type detectLanguage(parsebase::Parser<parsebase::CaseInsensitiveParserPolicy> &parser)
+Language::Type detectLanguage(tokenize::Tokenizer<tokenize::CaseInsensitiveTokenizerPolicy> &tokenizer)
 {
-	parser.skipWhiteSpace();
+	tokenizer.skipWhiteSpace();
 
 	// SAS begins with "begin_version"
-	if (parser.testAndSkip<std::string>("begin"))
+	if (tokenizer.testAndSkip<std::string>("begin"))
 	{
-		parser.seek(0);
+		tokenizer.seek(0);
 		return Language::Type::SAS;
 	}
 
 	// Skip potential PDDL comments
-	while (parser.currentCharacter() == ';')
+	while (tokenizer.currentCharacter() == ';')
 	{
-		parser.skipLine();
+		tokenizer.skipLine();
-		parser.skipWhiteSpace();
+		tokenizer.skipWhiteSpace();
 	}
 
 	// PDDL contains sections starting with "(define"
-	if (parser.testAndSkip<std::string>("(") && parser.testAndSkip<std::string>("define"))
+	if (tokenizer.testAndSkip<std::string>("(") && tokenizer.testAndSkip<std::string>("define"))
 	{
-		parser.seek(std::ios::beg);
+		tokenizer.seek(std::ios::beg);
 		return Language::Type::PDDL;
 	}
 
-	parser.seek(std::ios::beg);
+	tokenizer.seek(std::ios::beg);
 	return Language::Type::Unknown;
 }
 

include/plasp/output/Logger.h

@@ -6,7 +6,7 @@
 #include <plasp/output/ColorStream.h>
 #include <plasp/output/Priority.h>
 
-#include <parsebase/Location.h>
+#include <tokenize/Location.h>
 
 namespace plasp
 {
@@ -37,8 +37,8 @@ class Logger
 
 		void log(Priority priority, const char *message);
 		void log(Priority priority, const std::string &message);
-		void log(Priority priority, const parsebase::Location &location, const char *message);
+		void log(Priority priority, const tokenize::Location &location, const char *message);
-		void log(Priority priority, const parsebase::Location &location, const std::string &message);
+		void log(Priority priority, const tokenize::Location &location, const std::string &message);
 
 	private:
 		ColorStream m_outputStream;

include/plasp/pddl/Action.h

@@ -6,7 +6,7 @@
 #include <plasp/pddl/Expression.h>
 #include <plasp/pddl/expressions/Variable.h>
 
-#include <parsebase/Parser.h>
+#include <tokenize/Tokenizer.h>
 
 namespace plasp
 {

include/plasp/pddl/Context.h

@@ -6,7 +6,7 @@
 #include <vector>
 
 #include <plasp/output/Logger.h>
-#include <plasp/pddl/Parser.h>
+#include <plasp/pddl/Tokenizer.h>
 
 namespace plasp
 {
@@ -25,8 +25,8 @@ class Context
 		Context() = default;
 		~Context() = default;
 
-		explicit Context(Parser &&otherParser, output::Logger &otherLogger)
+		explicit Context(Tokenizer &&otherTokenizer, output::Logger &otherLogger)
-		:	parser{std::move(otherParser)},
+		:	tokenizer{std::move(otherTokenizer)},
 			logger(otherLogger)
 		{
 		}
@@ -35,7 +35,7 @@ class Context
 		Context &operator=(const Context &other) = delete;
 
 		Context(Context &&other)
-		:	parser(std::move(other.parser)),
+		:	tokenizer(std::move(other.tokenizer)),
 			logger(other.logger)
 		{
 		}
@@ -47,7 +47,7 @@ class Context
 			return "__plasp_";
 		}
 
-		Parser parser;
+		Tokenizer tokenizer;
 		output::Logger &logger;
 };
 

include/plasp/pddl/Description.h

@@ -4,7 +4,7 @@
 #include <plasp/pddl/Domain.h>
 #include <plasp/pddl/Problem.h>
 
-#include <parsebase/Parser.h>
+#include <tokenize/Tokenizer.h>
 
 namespace plasp
 {
@@ -46,9 +46,9 @@ class Description
 
 		Context &m_context;
 
-		parsebase::Stream::Position m_domainPosition;
+		tokenize::Stream::Position m_domainPosition;
 		std::unique_ptr<Domain> m_domain;
-		parsebase::Stream::Position m_problemPosition;
+		tokenize::Stream::Position m_problemPosition;
 		std::unique_ptr<Problem> m_problem;
 };
 

include/plasp/pddl/Domain.h

@@ -4,8 +4,8 @@
 #include <plasp/pddl/Action.h>
 #include <plasp/pddl/Context.h>
 #include <plasp/pddl/Expression.h>
-#include <plasp/pddl/Parser.h>
 #include <plasp/pddl/Requirement.h>
+#include <plasp/pddl/Tokenizer.h>
 #include <plasp/pddl/expressions/Constant.h>
 #include <plasp/pddl/expressions/DerivedPredicate.h>
 #include <plasp/pddl/expressions/PredicateDeclaration.h>
@@ -75,19 +75,19 @@ class Domain
 
 		std::string m_name;
 
-		parsebase::Stream::Position m_requirementsPosition;
+		tokenize::Stream::Position m_requirementsPosition;
 		Requirements m_requirements;
 
-		parsebase::Stream::Position m_typesPosition;
+		tokenize::Stream::Position m_typesPosition;
 		expressions::PrimitiveTypes m_types;
 
-		parsebase::Stream::Position m_constantsPosition;
+		tokenize::Stream::Position m_constantsPosition;
 		expressions::Constants m_constants;
 
-		parsebase::Stream::Position m_predicatesPosition;
+		tokenize::Stream::Position m_predicatesPosition;
 		expressions::PredicateDeclarations m_predicates;
 
-		std::vector<parsebase::Stream::Position> m_actionPositions;
+		std::vector<tokenize::Stream::Position> m_actionPositions;
 		std::vector<std::unique_ptr<Action>> m_actions;
 
 		expressions::DerivedPredicates m_derivedPredicates;

include/plasp/pddl/Expression.h

@@ -6,7 +6,7 @@
 
 #include <boost/intrusive_ptr.hpp>
 
-#include <parsebase/Parser.h>
+#include <tokenize/Tokenizer.h>
 
 namespace plasp
 {

include/plasp/pddl/IO.h

@@ -3,7 +3,7 @@
 
 #include <iostream>
 
-#include <plasp/pddl/Parser.h>
+#include <plasp/pddl/Tokenizer.h>
 
 namespace plasp
 {
@@ -16,14 +16,14 @@ namespace pddl
 //
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-inline void skipSection(Parser &parser)
+inline void skipSection(Tokenizer &tokenizer)
 {
 	size_t openParentheses = 1;
 
 	while (true)
 	{
-		const auto character = parser.currentCharacter();
+		const auto character = tokenizer.currentCharacter();
-		parser.advance();
+		tokenizer.advance();
 
 		if (character == '(')
 			openParentheses++;

include/plasp/pddl/Problem.h

@@ -4,8 +4,8 @@
 #include <plasp/pddl/Context.h>
 #include <plasp/pddl/Expression.h>
 #include <plasp/pddl/InitialState.h>
-#include <plasp/pddl/Parser.h>
 #include <plasp/pddl/Requirement.h>
+#include <plasp/pddl/Tokenizer.h>
 
 namespace plasp
 {
@@ -64,18 +64,18 @@ class Problem
 
 		std::string m_name;
 
-		parsebase::Stream::Position m_domainPosition;
+		tokenize::Stream::Position m_domainPosition;
 
-		parsebase::Stream::Position m_requirementsPosition;
+		tokenize::Stream::Position m_requirementsPosition;
 		Requirements m_requirements;
 
-		parsebase::Stream::Position m_objectsPosition;
+		tokenize::Stream::Position m_objectsPosition;
 		expressions::Constants m_objects;
 
-		parsebase::Stream::Position m_initialStatePosition;
+		tokenize::Stream::Position m_initialStatePosition;
 		std::unique_ptr<InitialState> m_initialState;
 
-		parsebase::Stream::Position m_goalPosition;
+		tokenize::Stream::Position m_goalPosition;
 		ExpressionPointer m_goal;
 };
 

include/plasp/pddl/Tokenizer.h

@@ -1,7 +1,7 @@
-#ifndef __PLASP__PDDL__PARSER_H
+#ifndef __PLASP__PDDL__TOKENIZER_H
-#define __PLASP__PDDL__PARSER_H
+#define __PLASP__PDDL__TOKENIZER_H
 
-#include <parsebase/Parser.h>
+#include <tokenize/Tokenizer.h>
 
 namespace plasp
 {
@@ -10,11 +10,11 @@ namespace pddl
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 //
-// Parser
+// Tokenizer
 //
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-class PDDLParserPolicy
+class PDDLTokenizerPolicy
 {
 	public:
 		static char transformCharacter(char c) noexcept
@@ -44,7 +44,7 @@ class PDDLParserPolicy
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-using Parser = parsebase::Parser<PDDLParserPolicy>;
+using Tokenizer = tokenize::Tokenizer<PDDLTokenizerPolicy>;
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 

include/plasp/pddl/expressions/At.h

@@ -59,33 +59,33 @@ template<typename ExpressionParser>
 AtPointer At::parse(Context &context, ExpressionContext &expressionContext,
 	ExpressionParser parseExpression)
 {
-	auto &parser = context.parser;
+	auto &tokenizer = context.tokenizer;
 
-	const auto position = parser.position();
+	const auto position = tokenizer.position();
 
-	if (!parser.testAndSkip<std::string>("(")
+	if (!tokenizer.testAndSkip<std::string>("(")
-		|| !parser.testIdentifierAndSkip("at"))
+		|| !tokenizer.testIdentifierAndSkip("at"))
 	{
-		parser.seek(position);
+		tokenizer.seek(position);
 		return nullptr;
 	}
 
 	size_t timePoint;
 
-	const auto timePointPosition = parser.position();
+	const auto timePointPosition = tokenizer.position();
 
-	if (parser.testIdentifierAndSkip("start"))
+	if (tokenizer.testIdentifierAndSkip("start"))
 		timePoint = TimePointStart;
-	else if (parser.testIdentifierAndSkip("end"))
+	else if (tokenizer.testIdentifierAndSkip("end"))
 		timePoint = TimePointEnd;
-	else if (parser.probeNumber())
+	else if (tokenizer.probeNumber())
 	{
-		parser.seek(timePointPosition);
+		tokenizer.seek(timePointPosition);
-		timePoint = parser.parse<size_t>();
+		timePoint = tokenizer.get<size_t>();
 	}
 	else
 	{
-		parser.seek(position);
+		tokenizer.seek(position);
 		return nullptr;
 	}
 
@@ -93,12 +93,12 @@ AtPointer At::parse(Context &context, ExpressionContext &expressionContext,
 
 	expression->m_timePoint = timePoint;
 
-	context.parser.skipWhiteSpace();
+	tokenizer.skipWhiteSpace();
 
 	// Parse argument
 	expression->setArgument(parseExpression(context, expressionContext));
 
-	parser.expect<std::string>(")");
+	tokenizer.expect<std::string>(")");
 
 	return expression;
 }

include/plasp/pddl/expressions/Binary.h

@@ -54,14 +54,14 @@ template<typename ExpressionParser>
 boost::intrusive_ptr<Derived> Binary<Derived>::parse(Context &context,
 	ExpressionContext &expressionContext, ExpressionParser parseExpression)
 {
-	auto &parser = context.parser;
+	auto &tokenizer = context.tokenizer;
 
-	const auto position = parser.position();
+	const auto position = tokenizer.position();
 
-	if (!parser.testAndSkip<std::string>("(")
+	if (!tokenizer.testAndSkip<std::string>("(")
-		|| !parser.testIdentifierAndSkip(Derived::Identifier))
+		|| !tokenizer.testIdentifierAndSkip(Derived::Identifier))
 	{
-		parser.seek(position);
+		tokenizer.seek(position);
 		return nullptr;
 	}
 
@@ -71,7 +71,7 @@ boost::intrusive_ptr<Derived> Binary<Derived>::parse(Context &context,
 	expression->Binary<Derived>::setArgument(0, parseExpression(context, expressionContext));
 	expression->Binary<Derived>::setArgument(1, parseExpression(context, expressionContext));
 
-	parser.expect<std::string>(")");
+	tokenizer.expect<std::string>(")");
 
 	return expression;
 }

include/plasp/pddl/expressions/NAry.h

@@ -55,33 +55,33 @@ template<typename ExpressionParser>
 boost::intrusive_ptr<Derived> NAry<Derived>::parse(Context &context,
 	ExpressionContext &expressionContext, ExpressionParser parseExpression)
 {
-	auto &parser = context.parser;
+	auto &tokenizer = context.tokenizer;
 
-	const auto position = parser.position();
+	const auto position = tokenizer.position();
 
-	if (!parser.testAndSkip<std::string>("(")
+	if (!tokenizer.testAndSkip<std::string>("(")
-		|| !parser.testIdentifierAndSkip(Derived::Identifier))
+		|| !tokenizer.testIdentifierAndSkip(Derived::Identifier))
 	{
-		parser.seek(position);
+		tokenizer.seek(position);
 		return nullptr;
 	}
 
 	auto expression = boost::intrusive_ptr<Derived>(new Derived);
 
-	parser.skipWhiteSpace();
+	tokenizer.skipWhiteSpace();
 
 	// Parse arguments of the expression
-	while (parser.currentCharacter() != ')')
+	while (tokenizer.currentCharacter() != ')')
 	{
 		expression->addArgument(parseExpression(context, expressionContext));
 
-		parser.skipWhiteSpace();
+		tokenizer.skipWhiteSpace();
 	}
 
 	if (expression->m_arguments.empty())
-		context.logger.log(output::Priority::Warning, parser.location(), "“" + Derived::Identifier + "” expressions should not be empty");
+		context.logger.log(output::Priority::Warning, tokenizer.location(), "“" + Derived::Identifier + "” expressions should not be empty");
 
-	parser.expect<std::string>(")");
+	tokenizer.expect<std::string>(")");
 
 	return expression;
 }

include/plasp/pddl/expressions/Not.h

@@ -53,25 +53,25 @@ template<typename ExpressionParser>
 NotPointer Not::parse(Context &context, ExpressionContext &expressionContext,
 	ExpressionParser parseExpression)
 {
-	auto &parser = context.parser;
+	auto &tokenizer = context.tokenizer;
 
-	const auto position = parser.position();
+	const auto position = tokenizer.position();
 
-	if (!parser.testAndSkip<std::string>("(")
+	if (!tokenizer.testAndSkip<std::string>("(")
-		|| !parser.testIdentifierAndSkip("not"))
+		|| !tokenizer.testIdentifierAndSkip("not"))
 	{
-		parser.seek(position);
+		tokenizer.seek(position);
 		return nullptr;
 	}
 
 	auto expression = NotPointer(new Not);
 
-	context.parser.skipWhiteSpace();
+	tokenizer.skipWhiteSpace();
 
 	// Parse argument
 	expression->setArgument(parseExpression(context, expressionContext));
 
-	parser.expect<std::string>(")");
+	tokenizer.expect<std::string>(")");
 
 	return expression;
 }

include/plasp/pddl/expressions/Quantified.h

@@ -69,23 +69,23 @@ template<typename ExpressionParser>
 boost::intrusive_ptr<Derived> QuantifiedCRTP<Derived>::parse(Context &context,
 	ExpressionContext &expressionContext, ExpressionParser parseExpression)
 {
-	auto &parser = context.parser;
+	auto &tokenizer = context.tokenizer;
 
-	const auto position = parser.position();
+	const auto position = tokenizer.position();
 
-	if (!parser.testAndSkip<std::string>("(")
+	if (!tokenizer.testAndSkip<std::string>("(")
-		|| !parser.testIdentifierAndSkip(Derived::Identifier))
+		|| !tokenizer.testIdentifierAndSkip(Derived::Identifier))
 	{
-		parser.seek(position);
+		tokenizer.seek(position);
 		return nullptr;
 	}
 
 	auto expression = boost::intrusive_ptr<Derived>(new Derived);
 
 	// Parse variable list
-	parser.expect<std::string>("(");
+	tokenizer.expect<std::string>("(");
 	Variable::parseTypedDeclarations(context, expressionContext, expression->m_variables);
-	parser.expect<std::string>(")");
+	tokenizer.expect<std::string>(")");
 
 	// Push newly parsed variables to the stack
 	expressionContext.variables.push(&expression->m_variables);
@@ -96,7 +96,7 @@ boost::intrusive_ptr<Derived> QuantifiedCRTP<Derived>::parse(Context &context,
 	// Clean up variable stack
 	expressionContext.variables.pop();
 
-	parser.expect<std::string>(")");
+	tokenizer.expect<std::string>(")");
 
 	return expression;
 }

include/plasp/pddl/expressions/When.h

@@ -36,14 +36,14 @@ WhenPointer When::parse(Context &context, ExpressionContext &expressionContext,
 	ConditionExpressionParser parseConditionExpression,
 	ImplicationExpressionParser parseImplicationExpression)
 {
-	auto &parser = context.parser;
+	auto &tokenizer = context.tokenizer;
 
-	const auto position = parser.position();
+	const auto position = tokenizer.position();
 
-	if (!parser.testAndSkip<std::string>("(")
+	if (!tokenizer.testAndSkip<std::string>("(")
-		|| !parser.testIdentifierAndSkip(Identifier))
+		|| !tokenizer.testIdentifierAndSkip(Identifier))
 	{
-		parser.seek(position);
+		tokenizer.seek(position);
 		return nullptr;
 	}
 
@@ -53,7 +53,7 @@ WhenPointer When::parse(Context &context, ExpressionContext &expressionContext,
 	expression->setArgument(0, parseConditionExpression(context, expressionContext));
 	expression->setArgument(1, parseImplicationExpression(context, expressionContext));
 
-	parser.expect<std::string>(")");
+	tokenizer.expect<std::string>(")");
 
 	return expression;
 }

include/plasp/sas/AssignedVariable.h

@@ -7,7 +7,7 @@
 #include <plasp/sas/Value.h>
 #include <plasp/sas/Variable.h>
 
-#include <parsebase/Parser.h>
+#include <tokenize/Tokenizer.h>
 
 namespace plasp
 {
@@ -28,8 +28,8 @@ using AssignedVariables = std::vector<AssignedVariable>;
 class AssignedVariable
 {
 	public:
-		static AssignedVariable fromSAS(parsebase::Parser<> &parser, const Variables &variables);
+		static AssignedVariable fromSAS(tokenize::Tokenizer<> &tokenizer, const Variables &variables);
-		static AssignedVariable fromSAS(parsebase::Parser<> &parser, const Variable &variable);
+		static AssignedVariable fromSAS(tokenize::Tokenizer<> &tokenizer, const Variable &variable);
 
 	public:
 		explicit AssignedVariable(const Variable &variable, const Value &value);

include/plasp/sas/AxiomRule.h

@@ -6,7 +6,7 @@
 #include <plasp/sas/AssignedVariable.h>
 #include <plasp/sas/Variable.h>
 
-#include <parsebase/Parser.h>
+#include <tokenize/Tokenizer.h>
 
 namespace plasp
 {
@@ -30,7 +30,7 @@ class AxiomRule
 		using Condition = AssignedVariable;
 		using Conditions = AssignedVariables;
 
-		static AxiomRule fromSAS(parsebase::Parser<> &parser, const Variables &variables);
+		static AxiomRule fromSAS(tokenize::Tokenizer<> &tokenizer, const Variables &variables);
 
 	public:
 		const Conditions &conditions() const;

include/plasp/sas/Description.h

@@ -14,7 +14,7 @@
 #include <plasp/sas/Operator.h>
 #include <plasp/sas/Variable.h>
 
-#include <parsebase/Parser.h>
+#include <tokenize/Tokenizer.h>
 
 namespace plasp
 {
@@ -30,7 +30,7 @@ namespace sas
 class Description
 {
 	public:
-		static Description fromParser(parsebase::Parser<> &&parser);
+		static Description fromTokenizer(tokenize::Tokenizer<> &&tokenizer);
 		static Description fromStream(std::istream &istream);
 		static Description fromFile(const std::experimental::filesystem::path &path);
 
@@ -51,16 +51,16 @@ class Description
 	private:
 		Description();
 
-		void parseContent(parsebase::Parser<> &parser);
+		void parseContent(tokenize::Tokenizer<> &tokenizer);
 
-		void parseVersionSection(parsebase::Parser<> &parser) const;
+		void parseVersionSection(tokenize::Tokenizer<> &tokenizer) const;
-		void parseMetricSection(parsebase::Parser<> &parser);
+		void parseMetricSection(tokenize::Tokenizer<> &tokenizer);
-		void parseVariablesSection(parsebase::Parser<> &parser);
+		void parseVariablesSection(tokenize::Tokenizer<> &tokenizer);
-		void parseMutexSection(parsebase::Parser<> &parser);
+		void parseMutexSection(tokenize::Tokenizer<> &tokenizer);
-		void parseInitialStateSection(parsebase::Parser<> &parser);
+		void parseInitialStateSection(tokenize::Tokenizer<> &tokenizer);
-		void parseGoalSection(parsebase::Parser<> &parser);
+		void parseGoalSection(tokenize::Tokenizer<> &tokenizer);
-		void parseOperatorSection(parsebase::Parser<> &parser);
+		void parseOperatorSection(tokenize::Tokenizer<> &tokenizer);
-		void parseAxiomSection(parsebase::Parser<> &parser);
+		void parseAxiomSection(tokenize::Tokenizer<> &tokenizer);
 
 		bool m_usesActionCosts;
 

include/plasp/sas/Effect.h

@@ -6,7 +6,7 @@
 #include <plasp/sas/AssignedVariable.h>
 #include <plasp/sas/Variable.h>
 
-#include <parsebase/Parser.h>
+#include <tokenize/Tokenizer.h>
 
 namespace plasp
 {
@@ -30,7 +30,7 @@ class Effect
 		using Condition = AssignedVariable;
 		using Conditions = AssignedVariables;
 
-		static Effect fromSAS(parsebase::Parser<> &parser, const Variables &variables, Conditions &preconditions);
+		static Effect fromSAS(tokenize::Tokenizer<> &tokenizer, const Variables &variables, Conditions &preconditions);
 
 	public:
 		const Conditions &conditions() const;

include/plasp/sas/Goal.h

@@ -3,7 +3,7 @@
 
 #include <plasp/sas/AssignedVariable.h>
 
-#include <parsebase/Parser.h>
+#include <tokenize/Tokenizer.h>
 
 namespace plasp
 {
@@ -22,7 +22,7 @@ class Goal
 		using Fact = AssignedVariable;
 		using Facts = AssignedVariables;
 
-		static Goal fromSAS(parsebase::Parser<> &parser, const Variables &variables);
+		static Goal fromSAS(tokenize::Tokenizer<> &tokenizer, const Variables &variables);
 
 	public:
 		const Facts &facts() const;

include/plasp/sas/InitialState.h

@@ -3,7 +3,7 @@
 
 #include <plasp/sas/AssignedVariable.h>
 
-#include <parsebase/Parser.h>
+#include <tokenize/Tokenizer.h>
 
 namespace plasp
 {
@@ -22,7 +22,7 @@ class InitialState
 		using Fact = AssignedVariable;
 		using Facts = AssignedVariables;
 
-		static InitialState fromSAS(parsebase::Parser<> &parser, const Variables &variables);
+		static InitialState fromSAS(tokenize::Tokenizer<> &tokenizer, const Variables &variables);
 
 	public:
 		const Facts &facts() const;

include/plasp/sas/MutexGroup.h

@@ -5,7 +5,7 @@
 
 #include <plasp/sas/AssignedVariable.h>
 
-#include <parsebase/Parser.h>
+#include <tokenize/Tokenizer.h>
 
 namespace plasp
 {
@@ -29,7 +29,7 @@ class MutexGroup
 		using Fact = AssignedVariable;
 		using Facts = AssignedVariables;
 
-		static MutexGroup fromSAS(parsebase::Parser<> &parser, const Variables &variables);
+		static MutexGroup fromSAS(tokenize::Tokenizer<> &tokenizer, const Variables &variables);
 
 	public:
 		const Facts &facts() const;

include/plasp/sas/Operator.h

@@ -10,7 +10,7 @@
 #include <plasp/sas/Predicate.h>
 #include <plasp/sas/Variable.h>
 
-#include <parsebase/Parser.h>
+#include <tokenize/Tokenizer.h>
 
 namespace plasp
 {
@@ -34,7 +34,7 @@ class Operator
 		using Condition = AssignedVariable;
 		using Conditions = AssignedVariables;
 
-		static Operator fromSAS(parsebase::Parser<> &parser, const Variables &variables);
+		static Operator fromSAS(tokenize::Tokenizer<> &tokenizer, const Variables &variables);
 
 	public:
 		void printPredicateAsASP(output::ColorStream &stream) const;

include/plasp/sas/Predicate.h

@@ -7,7 +7,7 @@
 
 #include <plasp/output/ColorStream.h>
 
-#include <parsebase/Parser.h>
+#include <tokenize/Tokenizer.h>
 
 namespace plasp
 {
@@ -23,7 +23,7 @@ namespace sas
 class Predicate
 {
 	public:
-		static Predicate fromSAS(parsebase::Parser<> &parser);
+		static Predicate fromSAS(tokenize::Tokenizer<> &tokenizer);
 
 		using Arguments = std::vector<std::string>;
 

include/plasp/sas/Value.h

@@ -7,7 +7,7 @@
 
 #include <plasp/output/ColorStream.h>
 
-#include <parsebase/Parser.h>
+#include <tokenize/Tokenizer.h>
 
 namespace plasp
 {
@@ -40,8 +40,8 @@ struct Value
 		static const Value Any;
 		static const Value None;
 
-		static Value fromSAS(parsebase::Parser<> &parser);
+		static Value fromSAS(tokenize::Tokenizer<> &tokenizer);
-		static const Value &referenceFromSAS(parsebase::Parser<> &parser, const Variable &variable);
+		static const Value &referenceFromSAS(tokenize::Tokenizer<> &tokenizer, const Variable &variable);
 
 	public:
 		Value negated() const;

include/plasp/sas/Variable.h

@@ -8,7 +8,7 @@
 #include <plasp/output/ColorStream.h>
 #include <plasp/sas/Value.h>
 
-#include <parsebase/Parser.h>
+#include <tokenize/Tokenizer.h>
 
 namespace plasp
 {
@@ -29,8 +29,8 @@ using Variables = std::vector<Variable>;
 class Variable
 {
 	public:
-		static Variable fromSAS(parsebase::Parser<> &parser);
+		static Variable fromSAS(tokenize::Tokenizer<> &tokenizer);
-		static const Variable &referenceFromSAS(parsebase::Parser<> &parser, const Variables &variables);
+		static const Variable &referenceFromSAS(tokenize::Tokenizer<> &tokenizer, const Variables &variables);
 
 	public:
 		void printNameAsASPPredicate(output::ColorStream &outputStream) const;

include/plasp/sas/VariableTransition.h

@@ -6,7 +6,7 @@
 #include <plasp/sas/Value.h>
 #include <plasp/sas/Variable.h>
 
-#include <parsebase/Parser.h>
+#include <tokenize/Tokenizer.h>
 
 namespace plasp
 {
@@ -27,7 +27,7 @@ using VariableTransitions = std::vector<VariableTransition>;
 class VariableTransition
 {
 	public:
-		static VariableTransition fromSAS(parsebase::Parser<> &parser, const Variables &variables);
+		static VariableTransition fromSAS(tokenize::Tokenizer<> &tokenizer, const Variables &variables);
 
 	public:
 		const Variable &variable() const;

lib/pddlparse/include/pddlparse/AST.h

@@ -7,8 +7,6 @@
 #include <type_traits>
 #include <vector>
 
-#include <parsebase/Stream.h>
-
 #include <pddlparse/ASTForward.h>
 
 namespace pddl

lib/pddlparse/include/pddlparse/Tokenizer.h

@@ -1,22 +1,20 @@
-#ifndef __PDDL_PARSE__DETAIL__PARSER_H
+#ifndef __PDDL_PARSE__TOKENIZER_H
-#define __PDDL_PARSE__DETAIL__PARSER_H
+#define __PDDL_PARSE__TOKENIZER_H
 
 #include <iostream>
 
-#include <parsebase/Parser.h>
+#include <tokenize/Tokenizer.h>
 
 namespace pddl
 {
-namespace detail
-{
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 //
-// Parser
+// Tokenizer
 //
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-struct PDDLParserPolicy
+struct PDDLTokenizerPolicy
 {
 	static char transformCharacter(char c) noexcept
 	{
@@ -45,11 +43,10 @@ struct PDDLParserPolicy
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-using Parser = parsebase::Parser<PDDLParserPolicy>;
+using Tokenizer = tokenize::Tokenizer<PDDLTokenizerPolicy>;
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-}
 }
 
 #endif

lib/pddlparse/src/CMakeLists.txt

@@ -8,7 +8,7 @@ file(GLOB detail_headers "../include/pddlparse/detail/*.h")
 
 set(includes
 	${PROJECT_SOURCE_DIR}/include
-	${PROJECT_SOURCE_DIR}/../../lib/parsebase/include
+	${PROJECT_SOURCE_DIR}/../../lib/tokenize/include
 )
 
 set(sources
@@ -20,7 +20,7 @@ set(sources
 )
 
 set(libraries
-	parsebase
+	tokenize
 )
 
 add_library(${target} ${sources})

lib/tokenize/CMakeLists.txt

@@ -1,7 +1,7 @@
 cmake_minimum_required(VERSION 2.6)
-project(parsebase)
+project(tokenize)
 
-option(PARSEBASE_BUILD_TESTS "Build unit tests" OFF)
+option(TOKENIZE_BUILD_TESTS "Build unit tests" OFF)
 
 set(CMAKE_CXX_FLAGS "-Wall -Wextra -Wpedantic -Werror")
 set(CMAKE_CXX_FLAGS_DEBUG "-g")
@@ -23,6 +23,6 @@ if (CMAKE_GENERATOR STREQUAL "Ninja" AND
 endif()
 
 add_subdirectory(src)
-if(PARSEBASE_BUILD_TESTS)
+if(TOKENIZE_BUILD_TESTS)
 	add_subdirectory(tests)
-endif(PARSEBASE_BUILD_TESTS)
+endif(TOKENIZE_BUILD_TESTS)

lib/tokenize/include/tokenize/Location.h

@@ -1,9 +1,9 @@
-#ifndef __PARSE_BASE__LOCATION_H
+#ifndef __TOKENIZE__LOCATION_H
-#define __PARSE_BASE__LOCATION_H
+#define __TOKENIZE__LOCATION_H
 
 #include <cstdlib>
 
-namespace parsebase
+namespace tokenize
 {
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////

lib/tokenize/include/tokenize/Stream.h

@@ -1,5 +1,5 @@
-#ifndef __PARSE_BASE__STREAM_H
+#ifndef __TOKENIZE__STREAM_H
-#define __PARSE_BASE__STREAM_H
+#define __TOKENIZE__STREAM_H
 
 #include <experimental/filesystem>
 #include <iostream>
@@ -7,9 +7,9 @@
 #include <sstream>
 #include <vector>
 
-#include <parsebase/Location.h>
+#include <tokenize/Location.h>
 
-namespace parsebase
+namespace tokenize
 {
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////

lib/tokenize/include/tokenize/Tokenizer.h

@@ -1,5 +1,5 @@
-#ifndef __PARSE_BASE__PARSER_H
+#ifndef __TOKENIZE__TOKENIZER_H
-#define __PARSE_BASE__PARSER_H
+#define __TOKENIZE__TOKENIZER_H
 
 #include <algorithm>
 #include <cassert>
@@ -8,11 +8,11 @@
 #include <sstream>
 #include <vector>
 
-#include <parsebase/ParserException.h>
+#include <tokenize/TokenizerException.h>
-#include <parsebase/ParserPolicy.h>
+#include <tokenize/TokenizerPolicy.h>
-#include <parsebase/Stream.h>
+#include <tokenize/Stream.h>
 
-namespace parsebase
+namespace tokenize
 {
 
 template<typename Type>
@@ -22,25 +22,25 @@ struct Tag
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 //
-// Parser
+// Tokenizer
 //
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-template<class ParserPolicy = CaseSensitiveParserPolicy>
+template<class TokenizerPolicy = CaseSensitiveTokenizerPolicy>
-class Parser: public Stream, public ParserPolicy
+class Tokenizer: public Stream, public TokenizerPolicy
 {
-	template<class OtherParserPolicy>
+	template<class OtherTokenizerPolicy>
-	friend class Parser;
+	friend class Tokenizer;
 
 	public:
-		explicit Parser();
+		explicit Tokenizer();
-		explicit Parser(std::string streamName, std::istream &istream);
+		explicit Tokenizer(std::string streamName, std::istream &istream);
 
-		template<class OtherParser>
+		template<class OtherTokenizer>
-		Parser(OtherParser &&otherParser)
+		Tokenizer(OtherTokenizer &&otherTokenizer)
 		{
-			m_stream = std::move(otherParser.m_stream);
+			m_stream = std::move(otherTokenizer.m_stream);
-			m_delimiters = std::move(otherParser.m_delimiters);
+			m_delimiters = std::move(otherTokenizer.m_delimiters);
 		}
 
 		void removeComments(const std::string &startSequence, const std::string &endSequence, bool removeEnd);
@@ -48,7 +48,7 @@ class Parser: public Stream, public ParserPolicy
 		char currentCharacter() const;
 
 		template<typename Type>
-		Type parse();
+		Type get();
 
 		template<typename Type>
 		bool testAndReturn(const Type &expectedValue);
@@ -59,27 +59,28 @@ class Parser: public Stream, public ParserPolicy
 		template<typename Type>
 		void expect(const Type &expectedValue);
 
-		std::string parseIdentifier();
+		// TODO: refactor
+		std::string getIdentifier();
 		bool testIdentifierAndReturn(const std::string &identifier);
 		bool testIdentifierAndSkip(const std::string &identifier);
 
 		// TODO: remove
 		bool probeNumber();
 
-		std::string parseLine();
+		std::string getLine();
 
 		void skipWhiteSpace();
 		void skipBlankSpace();
 		void skipLine();
 
 	private:
-		std::string parseImpl(Tag<std::string>);
+		std::string getImpl(Tag<std::string>);
-		char parseImpl(Tag<char>);
+		char getImpl(Tag<char>);
-		uint64_t parseImpl(Tag<uint64_t>);
+		uint64_t getImpl(Tag<uint64_t>);
-		int64_t parseImpl(Tag<int64_t>);
+		int64_t getImpl(Tag<int64_t>);
-		uint32_t parseImpl(Tag<uint32_t>);
+		uint32_t getImpl(Tag<uint32_t>);
-		int32_t parseImpl(Tag<int32_t>);
+		int32_t getImpl(Tag<int32_t>);
-		bool parseImpl(Tag<bool>);
+		bool getImpl(Tag<bool>);
 
 		bool testImpl(const std::string &expectedValue);
 		bool testImpl(char expectedValue);
@@ -89,13 +90,13 @@ class Parser: public Stream, public ParserPolicy
 		bool testImpl(int32_t expectedValue);
 		bool testImpl(bool expectedValue);
 
-		uint64_t parseIntegerBody();
+		uint64_t getIntegerBody();
 };
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-template<class ParserPolicy>
+template<class TokenizerPolicy>
-Parser<ParserPolicy>::Parser()
+Tokenizer<TokenizerPolicy>::Tokenizer()
 :	Stream()
 {
 }
@@ -103,38 +104,38 @@ Parser<ParserPolicy>::Parser()
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-template<class ParserPolicy>
+template<class TokenizerPolicy>
-Parser<ParserPolicy>::Parser(std::string streamName, std::istream &istream)
+Tokenizer<TokenizerPolicy>::Tokenizer(std::string streamName, std::istream &istream)
 :	Stream(streamName, istream)
 {
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-template<class ParserPolicy>
+template<class TokenizerPolicy>
-void Parser<ParserPolicy>::skipWhiteSpace()
+void Tokenizer<TokenizerPolicy>::skipWhiteSpace()
 {
 	check();
 
-	while (!atEnd() && ParserPolicy::isWhiteSpaceCharacter(currentCharacter()))
+	while (!atEnd() && TokenizerPolicy::isWhiteSpaceCharacter(currentCharacter()))
 		advance();
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-template<class ParserPolicy>
+template<class TokenizerPolicy>
-void Parser<ParserPolicy>::skipBlankSpace()
+void Tokenizer<TokenizerPolicy>::skipBlankSpace()
 {
 	check();
 
-	while (!atEnd() && ParserPolicy::isBlankCharacter(currentCharacter()))
+	while (!atEnd() && TokenizerPolicy::isBlankCharacter(currentCharacter()))
 		advance();
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-template<class ParserPolicy>
+template<class TokenizerPolicy>
-void Parser<ParserPolicy>::skipLine()
+void Tokenizer<TokenizerPolicy>::skipLine()
 {
 	check();
 
@@ -146,18 +147,18 @@ void Parser<ParserPolicy>::skipLine()
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-template<class ParserPolicy>
+template<class TokenizerPolicy>
 template<typename Type>
-Type Parser<ParserPolicy>::parse()
+Type Tokenizer<TokenizerPolicy>::get()
 {
-	return parseImpl(Tag<Type>());
+	return getImpl(Tag<Type>());
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-template<class ParserPolicy>
+template<class TokenizerPolicy>
 template<typename Type>
-bool Parser<ParserPolicy>::testAndReturn(const Type &expectedValue)
+bool Tokenizer<TokenizerPolicy>::testAndReturn(const Type &expectedValue)
 {
 	const auto previousPosition = position();
 
@@ -170,9 +171,9 @@ bool Parser<ParserPolicy>::testAndReturn(const Type &expectedValue)
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-template<class ParserPolicy>
+template<class TokenizerPolicy>
 template<typename Type>
-bool Parser<ParserPolicy>::testAndSkip(const Type &expectedValue)
+bool Tokenizer<TokenizerPolicy>::testAndSkip(const Type &expectedValue)
 {
 	const auto previousPosition = position();
 
@@ -186,9 +187,9 @@ bool Parser<ParserPolicy>::testAndSkip(const Type &expectedValue)
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-template<class ParserPolicy>
+template<class TokenizerPolicy>
 template<typename Type>
-void Parser<ParserPolicy>::expect(const Type &expectedValue)
+void Tokenizer<TokenizerPolicy>::expect(const Type &expectedValue)
 {
 	if (testAndSkip(expectedValue))
 		return;
@@ -196,13 +197,13 @@ void Parser<ParserPolicy>::expect(const Type &expectedValue)
 	std::stringstream message;
 	message << "unexpected value, expected “" << expectedValue << "”";
 
-	throw ParserException(location(), message.str());
+	throw TokenizerException(location(), message.str());
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-template<class ParserPolicy>
+template<class TokenizerPolicy>
-std::string Parser<ParserPolicy>::parseIdentifier()
+std::string Tokenizer<TokenizerPolicy>::getIdentifier()
 {
 	skipWhiteSpace();
 
@@ -212,7 +213,7 @@ std::string Parser<ParserPolicy>::parseIdentifier()
 	{
 		const auto character = currentCharacter();
 
-		if (!ParserPolicy::isIdentifierCharacter(character))
+		if (!TokenizerPolicy::isIdentifierCharacter(character))
 			return value;
 
 		value.push_back(character);
@@ -222,22 +223,22 @@ std::string Parser<ParserPolicy>::parseIdentifier()
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-template<class ParserPolicy>
+template<class TokenizerPolicy>
-bool Parser<ParserPolicy>::testIdentifierAndSkip(const std::string &expectedValue)
+bool Tokenizer<TokenizerPolicy>::testIdentifierAndSkip(const std::string &expectedValue)
 {
-	return testAndSkip(expectedValue) && !ParserPolicy::isIdentifierCharacter(currentCharacter());
+	return testAndSkip(expectedValue) && !TokenizerPolicy::isIdentifierCharacter(currentCharacter());
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-template<class ParserPolicy>
+template<class TokenizerPolicy>
-bool Parser<ParserPolicy>::probeNumber()
+bool Tokenizer<TokenizerPolicy>::probeNumber()
 {
 	const auto previousPosition = position();
 
 	skipWhiteSpace();
 
-	while (!ParserPolicy::isWhiteSpaceCharacter(currentCharacter()))
+	while (!TokenizerPolicy::isWhiteSpaceCharacter(currentCharacter()))
 		if (!std::isdigit(currentCharacter()))
 		{
 			seek(previousPosition);
@@ -250,8 +251,8 @@ bool Parser<ParserPolicy>::probeNumber()
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-template<class ParserPolicy>
+template<class TokenizerPolicy>
-std::string Parser<ParserPolicy>::parseLine()
+std::string Tokenizer<TokenizerPolicy>::getLine()
 {
 	std::string value;
 
@@ -273,8 +274,8 @@ std::string Parser<ParserPolicy>::parseLine()
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-template<class ParserPolicy>
+template<class TokenizerPolicy>
-void Parser<ParserPolicy>::removeComments(const std::string &startSequence, const std::string &endSequence, bool removeEnd)
+void Tokenizer<TokenizerPolicy>::removeComments(const std::string &startSequence, const std::string &endSequence, bool removeEnd)
 {
 	const auto inPosition = m_stream.tellg();
 	const auto outPosition = m_stream.tellp();
@@ -344,22 +345,22 @@ void Parser<ParserPolicy>::removeComments(const std::string &startSequence, cons
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-template<class ParserPolicy>
+template<class TokenizerPolicy>
-char Parser<ParserPolicy>::currentCharacter() const
+char Tokenizer<TokenizerPolicy>::currentCharacter() const
 {
-	return ParserPolicy::transformCharacter(Stream::currentCharacter());
+	return TokenizerPolicy::transformCharacter(Stream::currentCharacter());
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-template<class ParserPolicy>
+template<class TokenizerPolicy>
-std::string Parser<ParserPolicy>::parseImpl(Tag<std::string>)
+std::string Tokenizer<TokenizerPolicy>::getImpl(Tag<std::string>)
 {
 	skipWhiteSpace();
 
 	const auto startPosition = position();
 
-	while (!ParserPolicy::isWhiteSpaceCharacter(currentCharacter()))
+	while (!TokenizerPolicy::isWhiteSpaceCharacter(currentCharacter()))
 		advance();
 
 	const auto endPosition = position();
@@ -381,8 +382,8 @@ std::string Parser<ParserPolicy>::parseImpl(Tag<std::string>)
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-template<class ParserPolicy>
+template<class TokenizerPolicy>
-char Parser<ParserPolicy>::parseImpl(Tag<char>)
+char Tokenizer<TokenizerPolicy>::getImpl(Tag<char>)
 {
 	const auto value = currentCharacter();
 
@@ -393,13 +394,13 @@ char Parser<ParserPolicy>::parseImpl(Tag<char>)
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-template<class ParserPolicy>
+template<class TokenizerPolicy>
-uint64_t Parser<ParserPolicy>::parseIntegerBody()
+uint64_t Tokenizer<TokenizerPolicy>::getIntegerBody()
 {
 	check();
 
 	if (!std::isdigit(currentCharacter()))
-		throw ParserException(location(), "could not parse integer value");
+		throw TokenizerException(location(), "could not read integer value");
 
 	uint64_t value = 0;
 
@@ -421,51 +422,51 @@ uint64_t Parser<ParserPolicy>::parseIntegerBody()
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-template<class ParserPolicy>
+template<class TokenizerPolicy>
-int64_t Parser<ParserPolicy>::parseImpl(Tag<int64_t>)
+int64_t Tokenizer<TokenizerPolicy>::getImpl(Tag<int64_t>)
 {
 	skipWhiteSpace();
 
 	bool positive = testAndSkip<char>('+') || !testAndSkip<char>('-');
 
-	const auto value = parseIntegerBody();
+	const auto value = getIntegerBody();
 
 	return (positive ? value : -value);
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-template<class ParserPolicy>
+template<class TokenizerPolicy>
-uint64_t Parser<ParserPolicy>::parseImpl(Tag<uint64_t>)
+uint64_t Tokenizer<TokenizerPolicy>::getImpl(Tag<uint64_t>)
 {
 	skipWhiteSpace();
 
 	if (currentCharacter() == '-')
-		throw ParserException(location(), "expected unsigned integer, got signed one");
+		throw TokenizerException(location(), "expected unsigned integer, got signed one");
 
-	return parseIntegerBody();
+	return getIntegerBody();
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-template<class ParserPolicy>
+template<class TokenizerPolicy>
-int32_t Parser<ParserPolicy>::parseImpl(Tag<int32_t>)
+int32_t Tokenizer<TokenizerPolicy>::getImpl(Tag<int32_t>)
 {
-	return static_cast<int32_t>(parseImpl(Tag<int64_t>()));
+	return static_cast<int32_t>(getImpl(Tag<int64_t>()));
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-template<class ParserPolicy>
+template<class TokenizerPolicy>
-uint32_t Parser<ParserPolicy>::parseImpl(Tag<uint32_t>)
+uint32_t Tokenizer<TokenizerPolicy>::getImpl(Tag<uint32_t>)
 {
-	return static_cast<uint32_t>(parseImpl(Tag<uint64_t>()));
+	return static_cast<uint32_t>(getImpl(Tag<uint64_t>()));
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-template<class ParserPolicy>
+template<class TokenizerPolicy>
-bool Parser<ParserPolicy>::parseImpl(Tag<bool>)
+bool Tokenizer<TokenizerPolicy>::getImpl(Tag<bool>)
 {
 	skipWhiteSpace();
 
@@ -475,15 +476,15 @@ bool Parser<ParserPolicy>::parseImpl(Tag<bool>)
 	if (testAndSkip<char>('1'))
 		return true;
 
-	throw ParserException(location(), "could not parse Boolean value");
+	throw TokenizerException(location(), "could not read Boolean value");
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-template<class ParserPolicy>
+template<class TokenizerPolicy>
-bool Parser<ParserPolicy>::testImpl(const std::string &expectedValue)
+bool Tokenizer<TokenizerPolicy>::testImpl(const std::string &expectedValue)
 {
-	if (!ParserPolicy::isWhiteSpaceCharacter(expectedValue.front()))
+	if (!TokenizerPolicy::isWhiteSpaceCharacter(expectedValue.front()))
 		skipWhiteSpace();
 
 	const auto match = std::find_if(expectedValue.cbegin(), expectedValue.cend(),
@@ -504,8 +505,8 @@ bool Parser<ParserPolicy>::testImpl(const std::string &expectedValue)
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-template<class ParserPolicy>
+template<class TokenizerPolicy>
-bool Parser<ParserPolicy>::testImpl(char expectedValue)
+bool Tokenizer<TokenizerPolicy>::testImpl(char expectedValue)
 {
 	const auto result = (currentCharacter() == expectedValue);
 
@@ -516,46 +517,46 @@ bool Parser<ParserPolicy>::testImpl(char expectedValue)
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-template<class ParserPolicy>
+template<class TokenizerPolicy>
-bool Parser<ParserPolicy>::testImpl(int64_t expectedValue)
+bool Tokenizer<TokenizerPolicy>::testImpl(int64_t expectedValue)
 {
-	const auto value = parseImpl(Tag<int64_t>());
+	const auto value = getImpl(Tag<int64_t>());
 
 	return (value == expectedValue);
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-template<class ParserPolicy>
+template<class TokenizerPolicy>
-bool Parser<ParserPolicy>::testImpl(uint64_t expectedValue)
+bool Tokenizer<TokenizerPolicy>::testImpl(uint64_t expectedValue)
 {
-	const auto value = parseImpl(Tag<uint64_t>());
+	const auto value = getImpl(Tag<uint64_t>());
 
 	return (value == expectedValue);
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-template<class ParserPolicy>
+template<class TokenizerPolicy>
-bool Parser<ParserPolicy>::testImpl(int32_t expectedValue)
+bool Tokenizer<TokenizerPolicy>::testImpl(int32_t expectedValue)
 {
 	return testImpl(static_cast<int64_t>(expectedValue));
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-template<class ParserPolicy>
+template<class TokenizerPolicy>
-bool Parser<ParserPolicy>::testImpl(uint32_t expectedValue)
+bool Tokenizer<TokenizerPolicy>::testImpl(uint32_t expectedValue)
 {
 	return testImpl(static_cast<uint64_t>(expectedValue));
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-template<class ParserPolicy>
+template<class TokenizerPolicy>
-bool Parser<ParserPolicy>::testImpl(bool expectedValue)
+bool Tokenizer<TokenizerPolicy>::testImpl(bool expectedValue)
 {
-	const auto value = parseImpl(Tag<bool>());
+	const auto value = getImpl(Tag<bool>());
 
 	return (value == expectedValue);
 }

lib/tokenize/include/tokenize/TokenizerException.h

@@ -1,34 +1,34 @@
-#ifndef __PARSE_BASE__PARSER_EXCEPTION_H
+#ifndef __TOKENIZE__TOKENIZER_EXCEPTION_H
-#define __PARSE_BASE__PARSER_EXCEPTION_H
+#define __TOKENIZE__TOKENIZER_EXCEPTION_H
 
 #include <exception>
 #include <string>
 
-#include <parsebase/Location.h>
+#include <tokenize/Location.h>
 
-namespace parsebase
+namespace tokenize
 {
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 //
-// ParserException
+// TokenizerException
 //
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-class ParserException: public std::exception
+class TokenizerException: public std::exception
 {
 	public:
-		explicit ParserException(const Location &location)
+		explicit TokenizerException(const Location &location)
-		:	ParserException(location, "unspecified parser error")
+		:	TokenizerException(location, "unspecified tokenizer error")
 		{
 		}
 
-		explicit ParserException(const Location &location, const char *message)
+		explicit TokenizerException(const Location &location, const char *message)
-		:	ParserException(location, static_cast<std::string>(message))
+		:	TokenizerException(location, static_cast<std::string>(message))
 		{
 		}
 
-		explicit ParserException(const Location &location, const std::string &message)
+		explicit TokenizerException(const Location &location, const std::string &message)
 		:	m_location{location},
 			m_message{message},
 			// TODO: refactor
@@ -37,11 +37,9 @@ class ParserException: public std::exception
 		{
 		}
 
-		~ParserException() throw()
+		~TokenizerException() noexcept = default;
-		{
-		}
 
-		const char *what() const throw()
+		const char *what() const noexcept
 		{
 			return m_plainMessage.c_str();
 		}

lib/tokenize/include/tokenize/TokenizerPolicy.h

@@ -1,18 +1,18 @@
-#ifndef __PARSE_BASE__PARSER_POLICY_H
+#ifndef __TOKENIZE__TOKENIZER_POLICY_H
-#define __PARSE_BASE__PARSER_POLICY_H
+#define __TOKENIZE__TOKENIZER_POLICY_H
 
 #include <iostream>
 
-namespace parsebase
+namespace tokenize
 {
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 //
-// ParserPolicy
+// TokenizerPolicy
 //
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-struct CaseSensitiveParserPolicy
+struct CaseSensitiveTokenizerPolicy
 {
 	static constexpr char transformCharacter(char c) noexcept
 	{
@@ -37,7 +37,7 @@ struct CaseSensitiveParserPolicy
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-struct CaseInsensitiveParserPolicy
+struct CaseInsensitiveTokenizerPolicy
 {
 	static char transformCharacter(char c) noexcept
 	{

lib/tokenize/src/CMakeLists.txt

@@ -1,7 +1,7 @@
-set(target parsebase)
+set(target tokenize)
 
-file(GLOB core_sources "parsebase/*.cpp")
+file(GLOB core_sources "tokenize/*.cpp")
-file(GLOB core_headers "../include/parsebase/*.h")
+file(GLOB core_headers "../include/tokenize/*.h")
 
 set(includes
 	${PROJECT_SOURCE_DIR}/include

lib/tokenize/src/tokenize/Stream.cpp

@@ -1,11 +1,11 @@
-#include <parsebase/Stream.h>
+#include <tokenize/Stream.h>
 
 #include <algorithm>
 #include <fstream>
 
-#include <parsebase/ParserException.h>
+#include <tokenize/TokenizerException.h>
 
-namespace parsebase
+namespace tokenize
 {
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -89,7 +89,7 @@ Location Stream::location() const
 			return currentPosition >= fileDelimiter.position;
 		});
 
-	// If the parser is at the end of the stream, still count from the beginning of the last section
+	// If the tokenizer is at the end of the stream, still count from the beginning of the last section
 	if (currentFile == m_delimiters.crend())
 		currentFile = m_delimiters.crbegin();
 
@@ -143,10 +143,10 @@ bool Stream::atEnd() const
 void Stream::check() const
 {
 	if (atEnd())
-		throw ParserException(location(), "reading past end of file");
+		throw TokenizerException(location(), "reading past end of file");
 
 	if (m_stream.fail())
-		throw ParserException(location());
+		throw TokenizerException(location());
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////

lib/tokenize/tests/CMakeLists.txt

@@ -1,22 +1,21 @@
-set(target parsebase-tests)
+set(target tokenize-tests)
 
 file(GLOB core_sources "*.cpp")
 
 set(includes
-	${Boost_INCLUDE_DIRS}
 	${PROJECT_SOURCE_DIR}/include
 	${PROJECT_SOURCE_DIR}/../../lib/catch/single_include
 )
 
 set(libraries
-	parsebase
+	tokenize
 )
 
 add_executable(${target} ${core_sources})
 target_include_directories(${target} PRIVATE ${includes})
 target_link_libraries(${target} ${libraries})
 
-add_custom_target(run-parsebase-tests
+add_custom_target(run-tokenize-tests
-	COMMAND ${CMAKE_BINARY_DIR}/bin/parsebase-tests
+	COMMAND ${CMAKE_BINARY_DIR}/bin/tokenize-tests
 	DEPENDS ${target}
 	WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/tests)

lib/tokenize/tests/TestTokenizer.cpp

@@ -1,33 +1,33 @@
 #include <catch.hpp>
 
-#include <parsebase/Parser.h>
+#include <tokenize/Tokenizer.h>
-#include <parsebase/ParserException.h>
+#include <tokenize/TokenizerException.h>
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-TEST_CASE("[parser] Simple strings are parsed correctly", "[parser]")
+TEST_CASE("[tokenizer] Simple strings are tokenized correctly", "[tokenizer]")
 {
 	std::stringstream s("  identifier  5   \n-51\t 0 1 100 200 -300 -400");
-	parsebase::Parser<> p("input", s);
+	tokenize::Tokenizer<> p("input", s);
 
-	REQUIRE(p.parse<std::string>() == "identifier");
+	REQUIRE(p.get<std::string>() == "identifier");
-	REQUIRE(p.parse<size_t>() == 5u);
+	REQUIRE(p.get<size_t>() == 5u);
-	REQUIRE(p.parse<int>() == -51);
+	REQUIRE(p.get<int>() == -51);
-	REQUIRE(p.parse<bool>() == false);
+	REQUIRE(p.get<bool>() == false);
-	REQUIRE(p.parse<bool>() == true);
+	REQUIRE(p.get<bool>() == true);
 
-	REQUIRE(p.parse<int>() == 100);
+	REQUIRE(p.get<int>() == 100);
-	REQUIRE(p.parse<size_t>() == 200u);
+	REQUIRE(p.get<size_t>() == 200u);
-	REQUIRE(p.parse<int>() == -300);
+	REQUIRE(p.get<int>() == -300);
-	REQUIRE_THROWS_AS(p.parse<size_t>(), parsebase::ParserException);
+	REQUIRE_THROWS_AS(p.get<size_t>(), tokenize::TokenizerException);
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-TEST_CASE("[parser] Parsing exceptions are correctly reported", "[parser]")
+TEST_CASE("[tokenizer] Tokenizing exceptions are correctly reported", "[tokenizer]")
 {
 	std::stringstream s("  identifier  5   \n-51\t 0 1 100 200 -300 -400");
-	parsebase::Parser<> p("input", s);
+	tokenize::Tokenizer<> p("input", s);
 
 	REQUIRE_NOTHROW(p.expect<std::string>("identifier"));
 	REQUIRE_NOTHROW(p.expect<size_t>(5u));
@@ -38,41 +38,41 @@ TEST_CASE("[parser] Parsing exceptions are correctly reported", "[parser]")
 	REQUIRE_NOTHROW(p.expect<int>(100));
 	REQUIRE_NOTHROW(p.expect<size_t>(200u));
 	REQUIRE_NOTHROW(p.expect<int>(-300));
-	REQUIRE_THROWS_AS(p.expect<size_t>(-400), parsebase::ParserException);
+	REQUIRE_THROWS_AS(p.expect<size_t>(-400), tokenize::TokenizerException);
 
 	p.seek(0);
-	REQUIRE_THROWS_AS(p.expect<std::string>("error"), parsebase::ParserException);
+	REQUIRE_THROWS_AS(p.expect<std::string>("error"), tokenize::TokenizerException);
 
 	p.seek(14);
-	REQUIRE_THROWS_AS(p.expect<size_t>(6u), parsebase::ParserException);
+	REQUIRE_THROWS_AS(p.expect<size_t>(6u), tokenize::TokenizerException);
 
 	p.seek(17);
-	REQUIRE_THROWS_AS(p.expect<int>(-50), parsebase::ParserException);
+	REQUIRE_THROWS_AS(p.expect<int>(-50), tokenize::TokenizerException);
 
 	p.seek(24);
-	REQUIRE_THROWS_AS(p.expect<bool>(true), parsebase::ParserException);
+	REQUIRE_THROWS_AS(p.expect<bool>(true), tokenize::TokenizerException);
 
 	p.seek(26);
-	REQUIRE_THROWS_AS(p.expect<bool>(false), parsebase::ParserException);
+	REQUIRE_THROWS_AS(p.expect<bool>(false), tokenize::TokenizerException);
 
 	p.seek(28);
-	REQUIRE_THROWS_AS(p.expect<int>(101), parsebase::ParserException);
+	REQUIRE_THROWS_AS(p.expect<int>(101), tokenize::TokenizerException);
 
 	p.seek(31);
-	REQUIRE_THROWS_AS(p.expect<size_t>(201), parsebase::ParserException);
+	REQUIRE_THROWS_AS(p.expect<size_t>(201), tokenize::TokenizerException);
 
 	p.seek(34);
-	REQUIRE_THROWS_AS(p.expect<int>(-299), parsebase::ParserException);
+	REQUIRE_THROWS_AS(p.expect<int>(-299), tokenize::TokenizerException);
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-TEST_CASE("[parser] While parsing, the cursor position is as expected", "[parser]")
+TEST_CASE("[tokenizer] While tokenizing, the cursor position is as expected", "[tokenizer]")
 {
 	std::stringstream s("  identifier  5   \n-51\t 0 1");
-	parsebase::Parser<> p("input", s);
+	tokenize::Tokenizer<> p("input", s);
 
-	parsebase::Parser<>::Position pos;
+	tokenize::Tokenizer<>::Position pos;
 
 	pos = p.position();
 	REQUIRE(p.testAndReturn<std::string>("error") == false);
@@ -127,61 +127,61 @@ TEST_CASE("[parser] While parsing, the cursor position is as expected", "[parser
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-TEST_CASE("[parser] The end of the input stream is correctly handled", "[parser]")
+TEST_CASE("[tokenizer] The end of the input stream is correctly handled", "[tokenizer]")
 {
 	std::stringstream s1("test");
-	parsebase::Parser<> p1("input", s1);
+	tokenize::Tokenizer<> p1("input", s1);
 
 	REQUIRE_NOTHROW(p1.expect<std::string>("test"));
-	REQUIRE_THROWS_AS(p1.parse<std::string>(), parsebase::ParserException);
+	REQUIRE_THROWS_AS(p1.get<std::string>(), tokenize::TokenizerException);
 
 	std::stringstream s2("test1 test2 test3");
-	parsebase::Parser<> p2("input", s2);
+	tokenize::Tokenizer<> p2("input", s2);
 
 	REQUIRE_NOTHROW(p2.expect<std::string>("test1"));
 	REQUIRE_NOTHROW(p2.expect<std::string>("test2"));
 	REQUIRE_NOTHROW(p2.expect<std::string>("test3"));
-	REQUIRE_THROWS_AS(p2.parse<std::string>(), parsebase::ParserException);
+	REQUIRE_THROWS_AS(p2.get<std::string>(), tokenize::TokenizerException);
 
 	std::stringstream s3("-127");
-	parsebase::Parser<> p3("input", s3);
+	tokenize::Tokenizer<> p3("input", s3);
 
 	p3.expect<int>(-127);
-	REQUIRE_THROWS_AS(p3.parse<int>(), parsebase::ParserException);
+	REQUIRE_THROWS_AS(p3.get<int>(), tokenize::TokenizerException);
 
 	std::stringstream s4("128 -1023 -4095");
-	parsebase::Parser<> p4("input", s4);
+	tokenize::Tokenizer<> p4("input", s4);
 
 	REQUIRE_NOTHROW(p4.expect<size_t>(128));
 	REQUIRE_NOTHROW(p4.expect<int>(-1023));
 	REQUIRE_NOTHROW(p4.expect<int>(-4095));
-	REQUIRE_THROWS_AS(p4.parse<int>(), parsebase::ParserException);
+	REQUIRE_THROWS_AS(p4.get<int>(), tokenize::TokenizerException);
 
 	std::stringstream s5("0");
-	parsebase::Parser<> p5("input", s5);
+	tokenize::Tokenizer<> p5("input", s5);
 
 	p5.expect<bool>(false);
-	REQUIRE_THROWS_AS(p5.parse<bool>(), parsebase::ParserException);
+	REQUIRE_THROWS_AS(p5.get<bool>(), tokenize::TokenizerException);
 
 	std::stringstream s6("0 1 0");
-	parsebase::Parser<> p6("input", s6);
+	tokenize::Tokenizer<> p6("input", s6);
 
 	REQUIRE_NOTHROW(p6.expect<bool>(false));
 	REQUIRE_NOTHROW(p6.expect<bool>(true));
 	REQUIRE_NOTHROW(p6.expect<bool>(false));
-	REQUIRE_THROWS_AS(p6.parse<bool>(), parsebase::ParserException);
+	REQUIRE_THROWS_AS(p6.get<bool>(), tokenize::TokenizerException);
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-TEST_CASE("[parser] While parsing, the cursor location is as expcected", "[parser]")
+TEST_CASE("[tokenizer] While tokenizing, the cursor location is as expcected", "[tokenizer]")
 {
 	std::stringstream s("123 \n4\ntest1\n test2\ntest3 \ntest4\n\n\n\n");
-	parsebase::Parser<> p("input", s);
+	tokenize::Tokenizer<> p("input", s);
 
 	const auto startPosition = p.position();
 
-	parsebase::Location l;
+	tokenize::Location l;
 
 	l = p.location();
 	REQUIRE(l.rowStart == 1u);
@@ -277,19 +277,19 @@ TEST_CASE("[parser] While parsing, the cursor location is as expcected", "[parse
 
 	REQUIRE_NOTHROW(p.expect<std::string>("test1"));
 
-	// TODO: test parser with multiple sections
+	// TODO: test tokenizer with multiple sections
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-TEST_CASE("[parser] Comments are correctly removed", "[parser]")
+TEST_CASE("[tokenizer] Comments are correctly removed", "[tokenizer]")
 {
 	std::stringstream s1("; comment at beginning\ntest1; comment in between\ntest2; comment at end");
-	parsebase::Parser<> p1("input", s1);
+	tokenize::Tokenizer<> p1("input", s1);
 
 	p1.removeComments(";", "\n", false);
 
-	parsebase::Location l;
+	tokenize::Location l;
 
 	REQUIRE_NOTHROW(p1.expect<std::string>("test1"));
 
@@ -308,7 +308,7 @@ TEST_CASE("[parser] Comments are correctly removed", "[parser]")
 	REQUIRE(p1.atEnd());
 
 	std::stringstream s2("test;");
-	parsebase::Parser<> p2("input", s2);
+	tokenize::Tokenizer<> p2("input", s2);
 
 	p2.removeComments(";", "\n", false);
 
@@ -319,7 +319,7 @@ TEST_CASE("[parser] Comments are correctly removed", "[parser]")
 	REQUIRE(p2.atEnd());
 
 	std::stringstream s3("/* comment at start */ test1 /* comment in between */ test2 /*");
-	parsebase::Parser<> p3("input", s3);
+	tokenize::Tokenizer<> p3("input", s3);
 
 	p3.removeComments("/*", "*/", true);
 

src/CMakeLists.txt

@@ -27,7 +27,7 @@ file(GLOB utils_headers "../include/plasp/utils/*.h")
 set(includes
 	${Boost_INCLUDE_DIRS}
 	${PROJECT_SOURCE_DIR}/include
-	${PROJECT_SOURCE_DIR}/lib/parsebase/include
+	${PROJECT_SOURCE_DIR}/lib/tokenize/include
 )
 
 set(sources
@@ -58,7 +58,7 @@ set(sources
 
 set(libraries
 	${Boost_LIBRARIES}
-	parsebase
+	tokenize
 	pthread
 )
 

src/plasp/output/Logger.cpp

@@ -123,7 +123,7 @@ void Logger::log(Priority priority, const std::string &message)
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-void Logger::log(Priority priority, const parsebase::Location &location, const char *message)
+void Logger::log(Priority priority, const tokenize::Location &location, const char *message)
 {
 	const auto priorityID = static_cast<int>(priority);
 
@@ -150,7 +150,7 @@ void Logger::log(Priority priority, const parsebase::Location &location, const c
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-void Logger::log(Priority priority, const parsebase::Location &location, const std::string &message)
+void Logger::log(Priority priority, const tokenize::Location &location, const std::string &message)
 {
 	log(priority, location, message.c_str());
 }

src/plasp/pddl/Action.cpp

@@ -20,14 +20,14 @@ namespace pddl
 
 void Action::parseDeclaration(Context &context, Domain &domain)
 {
-	auto &parser = context.parser;
+	auto &tokenizer = context.tokenizer;
 
 	auto action = std::make_unique<Action>(Action());
 
-	action->m_name = parser.parseIdentifier();
+	action->m_name = tokenizer.getIdentifier();
 
-	parser.expect<std::string>(":parameters");
+	tokenizer.expect<std::string>(":parameters");
-	parser.expect<std::string>("(");
+	tokenizer.expect<std::string>("(");
 
 	ExpressionContext expressionContext(domain);
 	expressionContext.variables.push(&action->m_parameters);
@@ -35,19 +35,19 @@ void Action::parseDeclaration(Context &context, Domain &domain)
 	// Read parameters
 	expressions::Variable::parseTypedDeclarations(context, expressionContext, action->m_parameters);
 
-	parser.expect<std::string>(")");
+	tokenizer.expect<std::string>(")");
 
 	// Parse preconditions and effects
-	while (!parser.testAndReturn(')'))
+	while (!tokenizer.testAndReturn(')'))
 	{
-		parser.expect<std::string>(":");
+		tokenizer.expect<std::string>(":");
 
-		if (parser.testIdentifierAndSkip("precondition"))
+		if (tokenizer.testIdentifierAndSkip("precondition"))
 			action->m_precondition = parsePreconditionExpression(context, expressionContext);
-		else if (parser.testIdentifierAndSkip("effect"))
+		else if (tokenizer.testIdentifierAndSkip("effect"))
 			action->m_effect = parseEffectExpression(context, expressionContext);
 
-		parser.skipWhiteSpace();
+		tokenizer.skipWhiteSpace();
 	}
 
 	// Store new action

src/plasp/pddl/Description.cpp

@@ -6,7 +6,7 @@
 #include <plasp/pddl/ConsistencyException.h>
 #include <plasp/pddl/IO.h>
 
-#include <parsebase/ParserException.h>
+#include <tokenize/TokenizerException.h>
 
 namespace plasp
 {
@@ -44,7 +44,7 @@ Description Description::fromStream(std::istream &istream, Context &context)
 {
 	Description description(context);
 
-	description.m_context.parser.read("std::cin", istream);
+	description.m_context.tokenizer.read("std::cin", istream);
 	description.parse();
 
 	return description;
@@ -56,7 +56,7 @@ Description Description::fromFile(const std::string &path, Context &context)
 {
 	Description description(context);
 
-	description.m_context.parser.read(path);
+	description.m_context.tokenizer.read(path);
 	description.parse();
 
 	return description;
@@ -75,7 +75,7 @@ Description Description::fromFiles(const std::vector<std::string> &paths, Contex
 	std::for_each(paths.cbegin(), paths.cend(),
 		[&](const auto &path)
 		{
-			description.m_context.parser.read(path);
+			description.m_context.tokenizer.read(path);
 		});
 
 	description.parse();
@@ -126,9 +126,9 @@ const Problem &Description::problem() const
 
 void Description::parse()
 {
-	auto &parser = m_context.parser;
+	auto &tokenizer = m_context.tokenizer;
 
-	parser.removeComments(";", "\n", false);
+	tokenizer.removeComments(";", "\n", false);
 
 	// First, determine the locations of domain and problem
 	findSections();
@@ -136,12 +136,12 @@ void Description::parse()
 	if (m_domainPosition == -1)
 		throw ConsistencyException("no PDDL domain specified");
 
-	parser.seek(m_domainPosition);
+	tokenizer.seek(m_domainPosition);
 	m_domain->parse();
 
 	if (m_problemPosition != -1)
 	{
-		parser.seek(m_problemPosition);
+		tokenizer.seek(m_problemPosition);
 		m_problem->parse();
 	}
 
@@ -152,47 +152,47 @@ void Description::parse()
 
 void Description::findSections()
 {
-	auto &parser = m_context.parser;
+	auto &tokenizer = m_context.tokenizer;
 
-	parser.skipWhiteSpace();
+	tokenizer.skipWhiteSpace();
 
-	while (!parser.atEnd())
+	while (!tokenizer.atEnd())
 	{
-		const auto position = parser.position();
+		const auto position = tokenizer.position();
 
-		parser.expect<std::string>("(");
+		tokenizer.expect<std::string>("(");
-		parser.expect<std::string>("define");
+		tokenizer.expect<std::string>("define");
-		parser.expect<std::string>("(");
+		tokenizer.expect<std::string>("(");
 
-		if (parser.testAndSkip<std::string>("domain"))
+		if (tokenizer.testAndSkip<std::string>("domain"))
 		{
 			if (m_domainPosition != -1)
-				throw parsebase::ParserException(parser.location(), "PDDL description may not contain two domains");
+				throw tokenize::TokenizerException(tokenizer.location(), "PDDL description may not contain two domains");
 
 			m_domainPosition = position;
 
-			parser.seek(position);
+			tokenizer.seek(position);
 			m_domain->findSections();
 		}
-		else if (m_context.parser.testAndSkip<std::string>("problem"))
+		else if (tokenizer.testAndSkip<std::string>("problem"))
 		{
 			if (m_problemPosition != -1)
-				throw parsebase::ParserException(parser.location(), "PDDL description may currently not contain two problems");
+				throw tokenize::TokenizerException(tokenizer.location(), "PDDL description may currently not contain two problems");
 
 			m_problem = std::make_unique<Problem>(Problem(m_context, *m_domain));
 
 			m_problemPosition = position;
 
-			parser.seek(position);
+			tokenizer.seek(position);
 			m_problem->findSections();
 		}
 		else
 		{
-			const auto sectionIdentifier = parser.parse<std::string>();
+			const auto sectionIdentifier = tokenizer.get<std::string>();
-			throw parsebase::ParserException(parser.location(), "unknown PDDL section “" + sectionIdentifier + "”");
+			throw tokenize::TokenizerException(tokenizer.location(), "unknown PDDL section “" + sectionIdentifier + "”");
 		}
 
-		m_context.parser.skipWhiteSpace();
+		tokenizer.skipWhiteSpace();
 	}
 }
 

src/plasp/pddl/Domain.cpp

@@ -9,7 +9,7 @@
 #include <plasp/pddl/expressions/PrimitiveType.h>
 #include <plasp/pddl/expressions/Variable.h>
 
-#include <parsebase/ParserException.h>
+#include <tokenize/TokenizerException.h>
 
 namespace plasp
 {
@@ -35,119 +35,119 @@ Domain::Domain(Context &context)
 
 void Domain::findSections()
 {
-	auto &parser = m_context.parser;
+	auto &tokenizer = m_context.tokenizer;
 
-	parser.expect<std::string>("(");
+	tokenizer.expect<std::string>("(");
-	parser.expect<std::string>("define");
+	tokenizer.expect<std::string>("define");
-	parser.expect<std::string>("(");
+	tokenizer.expect<std::string>("(");
-	parser.expect<std::string>("domain");
+	tokenizer.expect<std::string>("domain");
 
-	m_name = m_context.parser.parseIdentifier();
+	m_name = tokenizer.getIdentifier();
 
-	parser.expect<std::string>(")");
+	tokenizer.expect<std::string>(")");
 
 	const auto setSectionPosition =
 		[&](const std::string &sectionName, auto &sectionPosition, const auto value, bool unique = false)
 		{
 			if (unique && sectionPosition != -1)
 			{
-				parser.seek(value);
+				tokenizer.seek(value);
-				throw parsebase::ParserException(parser.location(), "only one “:" + sectionName + "” section allowed");
+				throw tokenize::TokenizerException(tokenizer.location(), "only one “:" + sectionName + "” section allowed");
 			}
 
 			sectionPosition = value;
 		};
 
-	parser.skipWhiteSpace();
+	tokenizer.skipWhiteSpace();
 
 	// Find sections
-	while (parser.currentCharacter() != ')')
+	while (tokenizer.currentCharacter() != ')')
 	{
-		const auto position = parser.position();
+		const auto position = tokenizer.position();
 
-		parser.expect<std::string>("(");
+		tokenizer.expect<std::string>("(");
-		parser.expect<std::string>(":");
+		tokenizer.expect<std::string>(":");
 
-		const auto sectionIdentifierPosition = parser.position();
+		const auto sectionIdentifierPosition = tokenizer.position();
 
-		// Save the parser position of the individual sections for later parsing
+		// Save the tokenizer position of the individual sections for later parsing
-		if (parser.testIdentifierAndSkip("requirements"))
+		if (tokenizer.testIdentifierAndSkip("requirements"))
 			setSectionPosition("requirements", m_requirementsPosition, position, true);
-		else if (parser.testIdentifierAndSkip("types"))
+		else if (tokenizer.testIdentifierAndSkip("types"))
 			setSectionPosition("types", m_typesPosition, position, true);
-		else if (parser.testIdentifierAndSkip("constants"))
+		else if (tokenizer.testIdentifierAndSkip("constants"))
 			setSectionPosition("constants", m_constantsPosition, position, true);
-		else if (parser.testIdentifierAndSkip("predicates"))
+		else if (tokenizer.testIdentifierAndSkip("predicates"))
 			setSectionPosition("predicates", m_predicatesPosition, position, true);
-		else if (parser.testIdentifierAndSkip("action"))
+		else if (tokenizer.testIdentifierAndSkip("action"))
 		{
 			m_actionPositions.emplace_back(-1);
 			setSectionPosition("action", m_actionPositions.back(), position);
 		}
-		else if (parser.testIdentifierAndSkip("functions")
+		else if (tokenizer.testIdentifierAndSkip("functions")
-			|| parser.testIdentifierAndSkip("constraints")
+			|| tokenizer.testIdentifierAndSkip("constraints")
-			|| parser.testIdentifierAndSkip("durative-action")
+			|| tokenizer.testIdentifierAndSkip("durative-action")
-			|| parser.testIdentifierAndSkip("derived"))
+			|| tokenizer.testIdentifierAndSkip("derived"))
 		{
-			parser.seek(sectionIdentifierPosition);
+			tokenizer.seek(sectionIdentifierPosition);
 
-			const auto sectionIdentifier = parser.parseIdentifier();
+			const auto sectionIdentifier = tokenizer.getIdentifier();
 
-			m_context.logger.log(output::Priority::Warning, parser.location(), "section type “" + sectionIdentifier + "” currently unsupported");
+			m_context.logger.log(output::Priority::Warning, tokenizer.location(), "section type “" + sectionIdentifier + "” currently unsupported");
 
-			parser.seek(sectionIdentifierPosition);
+			tokenizer.seek(sectionIdentifierPosition);
 		}
 		else
 		{
-			const auto sectionIdentifier = parser.parseIdentifier();
+			const auto sectionIdentifier = tokenizer.getIdentifier();
 
-			parser.seek(position);
+			tokenizer.seek(position);
-			throw parsebase::ParserException(parser.location(), "unknown domain section “" + sectionIdentifier + "”");
+			throw tokenize::TokenizerException(tokenizer.location(), "unknown domain section “" + sectionIdentifier + "”");
 		}
 
 		// Skip section for now and parse it later
-		skipSection(parser);
+		skipSection(tokenizer);
 
-		parser.skipWhiteSpace();
+		tokenizer.skipWhiteSpace();
 	}
 
-	parser.expect<std::string>(")");
+	tokenizer.expect<std::string>(")");
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
 void Domain::parse()
 {
-	auto &parser = m_context.parser;
+	auto &tokenizer = m_context.tokenizer;
 
 	if (m_requirementsPosition != -1)
 	{
-		parser.seek(m_requirementsPosition);
+		tokenizer.seek(m_requirementsPosition);
 		parseRequirementSection();
 	}
 
 	if (m_typesPosition != -1)
 	{
-		parser.seek(m_typesPosition);
+		tokenizer.seek(m_typesPosition);
 		parseTypeSection();
 	}
 
 	if (m_constantsPosition != -1)
 	{
-		parser.seek(m_constantsPosition);
+		tokenizer.seek(m_constantsPosition);
 		parseConstantSection();
 	}
 
 	if (m_predicatesPosition != -1)
 	{
-		parser.seek(m_predicatesPosition);
+		tokenizer.seek(m_predicatesPosition);
 		parsePredicateSection();
 	}
 
 	for (size_t i = 0; i < m_actionPositions.size(); i++)
 		if (m_actionPositions[i] != -1)
 		{
-			parser.seek(m_actionPositions[i]);
+			tokenizer.seek(m_actionPositions[i]);
 			parseActionSection();
 		}
 
@@ -249,19 +249,19 @@ const expressions::DerivedPredicates &Domain::derivedPredicates() const
 
 void Domain::parseRequirementSection()
 {
-	auto &parser = m_context.parser;
+	auto &tokenizer = m_context.tokenizer;
 
-	parser.expect<std::string>("(");
+	tokenizer.expect<std::string>("(");
-	parser.expect<std::string>(":");
+	tokenizer.expect<std::string>(":");
-	parser.expect<std::string>("requirements");
+	tokenizer.expect<std::string>("requirements");
 
-	while (parser.currentCharacter() != ')')
+	while (tokenizer.currentCharacter() != ')')
 	{
-		parser.expect<std::string>(":");
+		tokenizer.expect<std::string>(":");
 
 		m_requirements.emplace_back(Requirement::parse(m_context));
 
-		parser.skipWhiteSpace();
+		tokenizer.skipWhiteSpace();
 	}
 
 	// TODO: do this check only once the problem is parsed
@@ -269,7 +269,7 @@ void Domain::parseRequirementSection()
 	if (m_requirements.empty())
 		m_requirements.emplace_back(Requirement::Type::STRIPS);
 
-	parser.expect<std::string>(")");
+	tokenizer.expect<std::string>(")");
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -292,7 +292,7 @@ void Domain::checkRequirement(Requirement::Type requirementType)
 	if (hasRequirement(requirementType))
 		return;
 
-	m_context.logger.log(output::Priority::Warning, m_context.parser.location(), "requirement “" + Requirement(requirementType).toPDDL() + "” used but never declared");
+	m_context.logger.log(output::Priority::Warning, m_context.tokenizer.location(), "requirement “" + Requirement(requirementType).toPDDL() + "” used but never declared");
 
 	m_requirements.push_back(requirementType);
 }
@@ -341,82 +341,82 @@ void Domain::computeDerivedRequirements()
 
 void Domain::parseTypeSection()
 {
-	auto &parser = m_context.parser;
+	auto &tokenizer = m_context.tokenizer;
 
-	parser.expect<std::string>("(");
+	tokenizer.expect<std::string>("(");
-	parser.expect<std::string>(":");
+	tokenizer.expect<std::string>(":");
-	parser.expect<std::string>("types");
+	tokenizer.expect<std::string>("types");
 
 	checkRequirement(Requirement::Type::Typing);
 
-	parser.skipWhiteSpace();
+	tokenizer.skipWhiteSpace();
 
 	// Store types and their parent types
-	while (parser.currentCharacter() != ')')
+	while (tokenizer.currentCharacter() != ')')
 	{
-		if (parser.currentCharacter() == '(')
+		if (tokenizer.currentCharacter() == '(')
-			throw parsebase::ParserException(parser.location(), "only primitive types are allowed in type section");
+			throw tokenize::TokenizerException(tokenizer.location(), "only primitive types are allowed in type section");
 
 		expressions::PrimitiveType::parseTypedDeclaration(m_context, *this);
 
-		parser.skipWhiteSpace();
+		tokenizer.skipWhiteSpace();
 	}
 
-	parser.expect<std::string>(")");
+	tokenizer.expect<std::string>(")");
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
 void Domain::parseConstantSection()
 {
-	auto &parser = m_context.parser;
+	auto &tokenizer = m_context.tokenizer;
 
-	parser.expect<std::string>("(");
+	tokenizer.expect<std::string>("(");
-	parser.expect<std::string>(":");
+	tokenizer.expect<std::string>(":");
-	parser.expect<std::string>("constants");
+	tokenizer.expect<std::string>("constants");
 
 	// Store constants
 	expressions::Constant::parseTypedDeclarations(m_context, *this);
 
-	parser.expect<std::string>(")");
+	tokenizer.expect<std::string>(")");
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
 void Domain::parsePredicateSection()
 {
-	auto &parser = m_context.parser;
+	auto &tokenizer = m_context.tokenizer;
 
-	parser.expect<std::string>("(");
+	tokenizer.expect<std::string>("(");
-	parser.expect<std::string>(":");
+	tokenizer.expect<std::string>(":");
-	parser.expect<std::string>("predicates");
+	tokenizer.expect<std::string>("predicates");
 
-	parser.skipWhiteSpace();
+	tokenizer.skipWhiteSpace();
 
 	// Store predicates and their arguments
-	while (parser.currentCharacter() != ')')
+	while (tokenizer.currentCharacter() != ')')
 	{
 		expressions::PredicateDeclaration::parse(m_context, *this);
 
-		parser.skipWhiteSpace();
+		tokenizer.skipWhiteSpace();
 	}
 
-	parser.expect<std::string>(")");
+	tokenizer.expect<std::string>(")");
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
 void Domain::parseActionSection()
 {
-	auto &parser = m_context.parser;
+	auto &tokenizer = m_context.tokenizer;
 
-	parser.expect<std::string>("(");
+	tokenizer.expect<std::string>("(");
-	parser.expect<std::string>(":");
+	tokenizer.expect<std::string>(":");
-	parser.expect<std::string>("action");
+	tokenizer.expect<std::string>("action");
 
 	Action::parseDeclaration(m_context, *this);
 
-	parser.expect<std::string>(")");
+	tokenizer.expect<std::string>(")");
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////

src/plasp/pddl/Expression.cpp

@@ -16,7 +16,7 @@
 #include <plasp/pddl/expressions/Unsupported.h>
 #include <plasp/pddl/expressions/When.h>
 
-#include <parsebase/ParserException.h>
+#include <tokenize/TokenizerException.h>
 
 namespace plasp
 {
@@ -100,9 +100,9 @@ ExpressionPointer parsePredicate(Context &context, ExpressionContext &expression
 ExpressionPointer parsePreconditionExpression(Context &context,
 	ExpressionContext &expressionContext)
 {
-	auto &parser = context.parser;
+	auto &tokenizer = context.tokenizer;
 
-	parser.skipWhiteSpace();
+	tokenizer.skipWhiteSpace();
 
 	ExpressionPointer expression;
 
@@ -112,23 +112,23 @@ ExpressionPointer parsePreconditionExpression(Context &context,
 		return expression;
 	}
 
-	const auto position = parser.position();
+	const auto position = tokenizer.position();
 
-	parser.expect<std::string>("(");
+	tokenizer.expect<std::string>("(");
 
-	const auto expressionIdentifierPosition = parser.position();
+	const auto expressionIdentifierPosition = tokenizer.position();
 
-	if (parser.testIdentifierAndSkip("preference"))
+	if (tokenizer.testIdentifierAndSkip("preference"))
 	{
 		// TODO: refactor
-		parser.seek(expressionIdentifierPosition);
+		tokenizer.seek(expressionIdentifierPosition);
-		const auto expressionIdentifier = parser.parseIdentifier();
+		const auto expressionIdentifier = tokenizer.getIdentifier();
 
-		parser.seek(position);
+		tokenizer.seek(position);
 		return expressions::Unsupported::parse(context);
 	}
 
-	parser.seek(position);
+	tokenizer.seek(position);
 	return parseExpression(context, expressionContext);
 }
 
@@ -136,9 +136,9 @@ ExpressionPointer parsePreconditionExpression(Context &context,
 
 ExpressionPointer parseExpression(Context &context, ExpressionContext &expressionContext)
 {
-	auto &parser = context.parser;
+	auto &tokenizer = context.tokenizer;
 
-	parser.skipWhiteSpace();
+	tokenizer.skipWhiteSpace();
 
 	ExpressionPointer expression;
 
@@ -153,43 +153,43 @@ ExpressionPointer parseExpression(Context &context, ExpressionContext &expressio
 		return expression;
 	}
 
-	const auto position = parser.position();
+	const auto position = tokenizer.position();
 
-	parser.expect<std::string>("(");
+	tokenizer.expect<std::string>("(");
 
-	const auto expressionIdentifierPosition = parser.position();
+	const auto expressionIdentifierPosition = tokenizer.position();
 
-	if (parser.testIdentifierAndSkip("-")
+	if (tokenizer.testIdentifierAndSkip("-")
-		|| parser.testIdentifierAndSkip("=")
+		|| tokenizer.testIdentifierAndSkip("=")
-		|| parser.testIdentifierAndSkip("*")
+		|| tokenizer.testIdentifierAndSkip("*")
-		|| parser.testIdentifierAndSkip("+")
+		|| tokenizer.testIdentifierAndSkip("+")
-		|| parser.testIdentifierAndSkip("-")
+		|| tokenizer.testIdentifierAndSkip("-")
-		|| parser.testIdentifierAndSkip("/")
+		|| tokenizer.testIdentifierAndSkip("/")
-		|| parser.testIdentifierAndSkip(">")
+		|| tokenizer.testIdentifierAndSkip(">")
-		|| parser.testIdentifierAndSkip("<")
+		|| tokenizer.testIdentifierAndSkip("<")
-		|| parser.testIdentifierAndSkip("=")
+		|| tokenizer.testIdentifierAndSkip("=")
-		|| parser.testIdentifierAndSkip(">=")
+		|| tokenizer.testIdentifierAndSkip(">=")
-		|| parser.testIdentifierAndSkip("<="))
+		|| tokenizer.testIdentifierAndSkip("<="))
 	{
-		parser.seek(expressionIdentifierPosition);
+		tokenizer.seek(expressionIdentifierPosition);
-		const auto expressionIdentifier = parser.parseIdentifier();
+		const auto expressionIdentifier = tokenizer.getIdentifier();
 
-		parser.seek(position);
+		tokenizer.seek(position);
 		return expressions::Unsupported::parse(context);
 	}
 
-	parser.seek(expressionIdentifierPosition);
+	tokenizer.seek(expressionIdentifierPosition);
-	const auto expressionIdentifier = parser.parseIdentifier();
+	const auto expressionIdentifier = tokenizer.getIdentifier();
 
-	parser.seek(position);
+	tokenizer.seek(position);
-	throw parsebase::ParserException(parser.location(), "expression type “" + expressionIdentifier + "” unknown or not allowed in this context");
+	throw tokenize::TokenizerException(tokenizer.location(), "expression type “" + expressionIdentifier + "” unknown or not allowed in this context");
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
 ExpressionPointer parseEffectExpression(Context &context, ExpressionContext &expressionContext)
 {
-	auto &parser = context.parser;
+	auto &tokenizer = context.tokenizer;
 
 	ExpressionPointer expression;
 
@@ -200,22 +200,22 @@ ExpressionPointer parseEffectExpression(Context &context, ExpressionContext &exp
 		return expression;
 	}
 
-	const auto position = parser.position();
+	const auto position = tokenizer.position();
 
-	parser.expect<std::string>("(");
+	tokenizer.expect<std::string>("(");
 
-	const auto expressionIdentifierPosition = parser.position();
+	const auto expressionIdentifierPosition = tokenizer.position();
 
-	if (parser.testIdentifierAndSkip("when"))
+	if (tokenizer.testIdentifierAndSkip("when"))
 	{
-		parser.seek(expressionIdentifierPosition);
+		tokenizer.seek(expressionIdentifierPosition);
-		const auto expressionIdentifier = parser.parseIdentifier();
+		const auto expressionIdentifier = tokenizer.getIdentifier();
 
-		parser.seek(position);
+		tokenizer.seek(position);
 		return expressions::Unsupported::parse(context);
 	}
 
-	parser.seek(position);
+	tokenizer.seek(position);
 	return parseEffectBodyExpression(context, expressionContext);
 }
 
@@ -223,7 +223,7 @@ ExpressionPointer parseEffectExpression(Context &context, ExpressionContext &exp
 
 ExpressionPointer parseEffectBodyExpression(Context &context, ExpressionContext &expressionContext)
 {
-	auto &parser = context.parser;
+	auto &tokenizer = context.tokenizer;
 
 	ExpressionPointer expression;
 
@@ -233,31 +233,31 @@ ExpressionPointer parseEffectBodyExpression(Context &context, ExpressionContext
 		return expression;
 	}
 
-	const auto position = parser.position();
+	const auto position = tokenizer.position();
 
-	parser.expect<std::string>("(");
+	tokenizer.expect<std::string>("(");
 
-	const auto expressionIdentifierPosition = parser.position();
+	const auto expressionIdentifierPosition = tokenizer.position();
 
-	if (parser.testIdentifierAndSkip("=")
+	if (tokenizer.testIdentifierAndSkip("=")
-		|| parser.testIdentifierAndSkip("assign")
+		|| tokenizer.testIdentifierAndSkip("assign")
-		|| parser.testIdentifierAndSkip("scale-up")
+		|| tokenizer.testIdentifierAndSkip("scale-up")
-		|| parser.testIdentifierAndSkip("scale-down")
+		|| tokenizer.testIdentifierAndSkip("scale-down")
-		|| parser.testIdentifierAndSkip("increase")
+		|| tokenizer.testIdentifierAndSkip("increase")
-		|| parser.testIdentifierAndSkip("decrease"))
+		|| tokenizer.testIdentifierAndSkip("decrease"))
 	{
-		parser.seek(expressionIdentifierPosition);
+		tokenizer.seek(expressionIdentifierPosition);
-		const auto expressionIdentifier = parser.parseIdentifier();
+		const auto expressionIdentifier = tokenizer.getIdentifier();
 
-		parser.seek(position);
+		tokenizer.seek(position);
 		return expressions::Unsupported::parse(context);
 	}
 
-	parser.seek(expressionIdentifierPosition);
+	tokenizer.seek(expressionIdentifierPosition);
-	const auto expressionIdentifier = parser.parseIdentifier();
+	const auto expressionIdentifier = tokenizer.getIdentifier();
 
-	parser.seek(position);
+	tokenizer.seek(position);
-	throw parsebase::ParserException(parser.location(), "expression type “" + expressionIdentifier + "” unknown or not allowed in this context");
+	throw tokenize::TokenizerException(tokenizer.location(), "expression type “" + expressionIdentifier + "” unknown or not allowed in this context");
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -276,14 +276,12 @@ ExpressionPointer parseConditionalEffectExpression(Context &context, ExpressionC
 
 ExpressionPointer parsePredicate(Context &context, ExpressionContext &expressionContext)
 {
-	auto &parser = context.parser;
-
 	ExpressionPointer expression;
 
 	if ((expression = expressions::Predicate::parse(context, expressionContext)))
 		return expression;
 
-	throw parsebase::ParserException(parser.location(), "expected predicate");
+	throw tokenize::TokenizerException(context.tokenizer.location(), "expected predicate");
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -305,30 +303,30 @@ ExpressionPointer parseLiteral(Context &context, ExpressionContext &expressionCo
 
 ExpressionPointer parseAtomicFormula(Context &context, ExpressionContext &expressionContext)
 {
-	auto &parser = context.parser;
+	auto &tokenizer = context.tokenizer;
 
 	ExpressionPointer expression;
 
 	if ((expression = expressions::Predicate::parse(context, expressionContext)))
 		return expression;
 
-	const auto position = parser.position();
+	const auto position = tokenizer.position();
 
-	if (!parser.testAndSkip<std::string>("("))
+	if (!tokenizer.testAndSkip<std::string>("("))
 		return nullptr;
 
-	const auto expressionIdentifierPosition = parser.position();
+	const auto expressionIdentifierPosition = tokenizer.position();
 
-	if (parser.testIdentifierAndSkip("="))
+	if (tokenizer.testIdentifierAndSkip("="))
 	{
-		parser.seek(expressionIdentifierPosition);
+		tokenizer.seek(expressionIdentifierPosition);
-		const auto expressionIdentifier = parser.parseIdentifier();
+		const auto expressionIdentifier = tokenizer.getIdentifier();
 
-		parser.seek(position);
+		tokenizer.seek(position);
 		return expressions::Unsupported::parse(context);
 	}
 
-	parser.seek(position);
+	tokenizer.seek(position);
 	return nullptr;
 }
 

src/plasp/pddl/InitialState.cpp

@@ -8,7 +8,7 @@
 #include <plasp/pddl/expressions/Predicate.h>
 #include <plasp/pddl/expressions/Unsupported.h>
 
-#include <parsebase/ParserException.h>
+#include <tokenize/TokenizerException.h>
 
 namespace plasp
 {
@@ -24,7 +24,7 @@ namespace pddl
 std::unique_ptr<InitialState> InitialState::parseDeclaration(Context &context,
 	ExpressionContext &expressionContext)
 {
-	auto &parser = context.parser;
+	auto &tokenizer = context.tokenizer;
 
 	auto initialState = std::make_unique<InitialState>(InitialState());
 
@@ -40,38 +40,38 @@ std::unique_ptr<InitialState> InitialState::parseDeclaration(Context &context,
 				return expression;
 			}
 
-			const auto position = parser.position();
+			const auto position = tokenizer.position();
 
-			parser.expect<std::string>("(");
+			tokenizer.expect<std::string>("(");
 
-			const auto expressionIdentifierPosition = parser.position();
+			const auto expressionIdentifierPosition = tokenizer.position();
 
-			if (parser.testIdentifierAndSkip("="))
+			if (tokenizer.testIdentifierAndSkip("="))
 			{
-				parser.seek(expressionIdentifierPosition);
+				tokenizer.seek(expressionIdentifierPosition);
-				const auto expressionIdentifier = parser.parseIdentifier();
+				const auto expressionIdentifier = tokenizer.getIdentifier();
 
-				parser.seek(position);
+				tokenizer.seek(position);
 				return expressions::Unsupported::parse(context);
 			}
 
-			parser.seek(expressionIdentifierPosition);
+			tokenizer.seek(expressionIdentifierPosition);
-			const auto expressionIdentifier = parser.parseIdentifier();
+			const auto expressionIdentifier = tokenizer.getIdentifier();
 
-			parser.seek(position);
+			tokenizer.seek(position);
-			throw parsebase::ParserException(parser.location(), "expression type “" + expressionIdentifier + "” unknown or not allowed in this context");
+			throw tokenize::TokenizerException(tokenizer.location(), "expression type “" + expressionIdentifier + "” unknown or not allowed in this context");
 		};
 
-	parser.skipWhiteSpace();
+	tokenizer.skipWhiteSpace();
 
-	while (parser.currentCharacter() != ')')
+	while (tokenizer.currentCharacter() != ')')
 	{
 		ExpressionPointer expression;
 
 		if ((expression = parseInitialStateElement()))
 			initialState->m_facts.emplace_back(std::move(expression));
 
-		parser.skipWhiteSpace();
+		tokenizer.skipWhiteSpace();
 	}
 
 	return initialState;

src/plasp/pddl/Problem.cpp

@@ -8,7 +8,7 @@
 #include <plasp/pddl/IO.h>
 #include <plasp/pddl/expressions/Constant.h>
 
-#include <parsebase/ParserException.h>
+#include <tokenize/TokenizerException.h>
 
 namespace plasp
 {
@@ -36,114 +36,114 @@ Problem::Problem(Context &context, Domain &domain)
 
 void Problem::findSections()
 {
-	auto &parser = m_context.parser;
+	auto &tokenizer = m_context.tokenizer;
 
-	parser.expect<std::string>("(");
+	tokenizer.expect<std::string>("(");
-	parser.expect<std::string>("define");
+	tokenizer.expect<std::string>("define");
-	parser.expect<std::string>("(");
+	tokenizer.expect<std::string>("(");
-	parser.expect<std::string>("problem");
+	tokenizer.expect<std::string>("problem");
 
-	m_name = parser.parseIdentifier();
+	m_name = tokenizer.getIdentifier();
 
-	parser.expect<std::string>(")");
+	tokenizer.expect<std::string>(")");
 
 	const auto setSectionPosition =
 	[&](const std::string &sectionName, auto &sectionPosition, const auto value, bool unique = false)
 	{
 		if (unique && sectionPosition != -1)
 		{
-			parser.seek(value);
+			tokenizer.seek(value);
-			throw parsebase::ParserException(parser.location(), "only one “:" + sectionName + "” section allowed");
+			throw tokenize::TokenizerException(tokenizer.location(), "only one “:" + sectionName + "” section allowed");
 		}
 
 		sectionPosition = value;
 	};
 
-	parser.skipWhiteSpace();
+	tokenizer.skipWhiteSpace();
 
-	while (parser.currentCharacter() != ')')
+	while (tokenizer.currentCharacter() != ')')
 	{
-		const auto position = parser.position();
+		const auto position = tokenizer.position();
 
-		parser.expect<std::string>("(");
+		tokenizer.expect<std::string>("(");
-		parser.expect<std::string>(":");
+		tokenizer.expect<std::string>(":");
 
-		const auto sectionIdentifierPosition = parser.position();
+		const auto sectionIdentifierPosition = tokenizer.position();
 
 		// TODO: check order of the sections
-		if (parser.testIdentifierAndSkip("domain"))
+		if (tokenizer.testIdentifierAndSkip("domain"))
 			setSectionPosition("domain", m_domainPosition, position, true);
-		else if (parser.testIdentifierAndSkip("requirements"))
+		else if (tokenizer.testIdentifierAndSkip("requirements"))
 			setSectionPosition("requirements", m_requirementsPosition, position, true);
-		else if (parser.testIdentifierAndSkip("objects"))
+		else if (tokenizer.testIdentifierAndSkip("objects"))
 			setSectionPosition("objects", m_objectsPosition, position, true);
-		else if (parser.testIdentifierAndSkip("init"))
+		else if (tokenizer.testIdentifierAndSkip("init"))
 			setSectionPosition("init", m_initialStatePosition, position, true);
-		else if (parser.testIdentifierAndSkip("goal"))
+		else if (tokenizer.testIdentifierAndSkip("goal"))
 			setSectionPosition("goal", m_goalPosition, position, true);
-		else if (parser.testIdentifierAndSkip("constraints")
+		else if (tokenizer.testIdentifierAndSkip("constraints")
-			|| parser.testIdentifierAndSkip("metric")
+			|| tokenizer.testIdentifierAndSkip("metric")
-			|| parser.testIdentifierAndSkip("length"))
+			|| tokenizer.testIdentifierAndSkip("length"))
 		{
-			parser.seek(sectionIdentifierPosition);
+			tokenizer.seek(sectionIdentifierPosition);
 
-			const auto sectionIdentifier = parser.parseIdentifier();
+			const auto sectionIdentifier = tokenizer.getIdentifier();
 
-			m_context.logger.log(output::Priority::Warning, parser.location(), "section type “" + sectionIdentifier + "” currently unsupported");
+			m_context.logger.log(output::Priority::Warning, tokenizer.location(), "section type “" + sectionIdentifier + "” currently unsupported");
 
-			parser.seek(sectionIdentifierPosition);
+			tokenizer.seek(sectionIdentifierPosition);
 		}
 		else
 		{
-			const auto sectionIdentifier = parser.parseIdentifier();
+			const auto sectionIdentifier = tokenizer.getIdentifier();
 
-			parser.seek(position);
+			tokenizer.seek(position);
-			throw parsebase::ParserException(parser.location(), "unknown problem section “" + sectionIdentifier + "”");
+			throw tokenize::TokenizerException(tokenizer.location(), "unknown problem section “" + sectionIdentifier + "”");
 		}
 
 		// Skip section for now and parse it later
-		skipSection(parser);
+		skipSection(tokenizer);
 
-		parser.skipWhiteSpace();
+		tokenizer.skipWhiteSpace();
 	}
 
-	parser.expect<std::string>(")");
+	tokenizer.expect<std::string>(")");
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
 void Problem::parse()
 {
-	auto &parser = m_context.parser;
+	auto &tokenizer = m_context.tokenizer;
 
 	if (m_domainPosition == -1)
 		throw ConsistencyException("problem description does not specify the corresponding domain");
 
-	parser.seek(m_domainPosition);
+	tokenizer.seek(m_domainPosition);
 	parseDomainSection();
 
 	if (m_requirementsPosition != -1)
 	{
-		parser.seek(m_requirementsPosition);
+		tokenizer.seek(m_requirementsPosition);
 		parseRequirementSection();
 	}
 
 	if (m_objectsPosition != -1)
 	{
-		parser.seek(m_objectsPosition);
+		tokenizer.seek(m_objectsPosition);
 		parseObjectSection();
 	}
 
 	if (m_initialStatePosition == -1)
 		throw ConsistencyException("problem description does not specify an initial state");
 
-	parser.seek(m_initialStatePosition);
+	tokenizer.seek(m_initialStatePosition);
 	parseInitialStateSection();
 
 	if (m_goalPosition == -1)
 		throw ConsistencyException("problem description does not specify a goal");
 
-	parser.seek(m_goalPosition);
+	tokenizer.seek(m_goalPosition);
 	parseGoalSection();
 }
 
@@ -193,41 +193,41 @@ const expressions::Constants &Problem::objects() const
 
 void Problem::parseDomainSection()
 {
-	auto &parser = m_context.parser;
+	auto &tokenizer = m_context.tokenizer;
 
-	parser.expect<std::string>("(");
+	tokenizer.expect<std::string>("(");
-	parser.expect<std::string>(":");
+	tokenizer.expect<std::string>(":");
-	parser.expect<std::string>("domain");
+	tokenizer.expect<std::string>("domain");
 
-	parser.skipWhiteSpace();
+	tokenizer.skipWhiteSpace();
 
-	const auto domainName = parser.parseIdentifier();
+	const auto domainName = tokenizer.getIdentifier();
 
 	if (m_domain.name() != domainName)
-		throw parsebase::ParserException(parser.location(), "domains do not match (“" + m_domain.name() + "” and “" + domainName + "”)");
+		throw tokenize::TokenizerException(tokenizer.location(), "domains do not match (“" + m_domain.name() + "” and “" + domainName + "”)");
 
-	parser.expect<std::string>(")");
+	tokenizer.expect<std::string>(")");
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
 void Problem::parseRequirementSection()
 {
-	auto &parser = m_context.parser;
+	auto &tokenizer = m_context.tokenizer;
 
-	parser.expect<std::string>("(");
+	tokenizer.expect<std::string>("(");
-	parser.expect<std::string>(":");
+	tokenizer.expect<std::string>(":");
-	parser.expect<std::string>("requirements");
+	tokenizer.expect<std::string>("requirements");
 
-	parser.skipWhiteSpace();
+	tokenizer.skipWhiteSpace();
 
-	while (parser.currentCharacter() != ')')
+	while (tokenizer.currentCharacter() != ')')
 	{
-		parser.expect<std::string>(":");
+		tokenizer.expect<std::string>(":");
 
 		m_requirements.emplace_back(Requirement::parse(m_context));
 
-		parser.skipWhiteSpace();
+		tokenizer.skipWhiteSpace();
 	}
 
 	// TODO: do this check only once the domain is parsed
@@ -235,7 +235,7 @@ void Problem::parseRequirementSection()
 	if (m_requirements.empty())
 		m_requirements.emplace_back(Requirement::Type::STRIPS);
 
-	parser.expect<std::string>(")");
+	tokenizer.expect<std::string>(")");
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -261,7 +261,7 @@ void Problem::checkRequirement(Requirement::Type requirementType)
 	if (hasRequirement(requirementType))
 		return;
 
-	m_context.logger.log(output::Priority::Warning, m_context.parser.location(), "requirement “" + Requirement(requirementType).toPDDL() + "” used but never declared");
+	m_context.logger.log(output::Priority::Warning, m_context.tokenizer.location(), "requirement “" + Requirement(requirementType).toPDDL() + "” used but never declared");
 
 	m_requirements.push_back(requirementType);
 }
@@ -310,52 +310,52 @@ void Problem::computeDerivedRequirements()
 
 void Problem::parseObjectSection()
 {
-	auto &parser = m_context.parser;
+	auto &tokenizer = m_context.tokenizer;
 
-	parser.expect<std::string>("(");
+	tokenizer.expect<std::string>("(");
-	parser.expect<std::string>(":");
+	tokenizer.expect<std::string>(":");
-	parser.expect<std::string>("objects");
+	tokenizer.expect<std::string>("objects");
 
-	parser.skipWhiteSpace();
+	tokenizer.skipWhiteSpace();
 
 	// Store constants
 	expressions::Constant::parseTypedDeclarations(m_context, *this);
 
-	parser.expect<std::string>(")");
+	tokenizer.expect<std::string>(")");
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
 void Problem::parseInitialStateSection()
 {
-	auto &parser = m_context.parser;
+	auto &tokenizer = m_context.tokenizer;
 
-	parser.expect<std::string>("(");
+	tokenizer.expect<std::string>("(");
-	parser.expect<std::string>(":");
+	tokenizer.expect<std::string>(":");
-	parser.expect<std::string>("init");
+	tokenizer.expect<std::string>("init");
 
 	ExpressionContext expressionContext(m_domain, this);
 
 	m_initialState = InitialState::parseDeclaration(m_context, expressionContext);
 
-	parser.expect<std::string>(")");
+	tokenizer.expect<std::string>(")");
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
 void Problem::parseGoalSection()
 {
-	auto &parser = m_context.parser;
+	auto &tokenizer = m_context.tokenizer;
 
-	parser.expect<std::string>("(");
+	tokenizer.expect<std::string>("(");
-	parser.expect<std::string>(":");
+	tokenizer.expect<std::string>(":");
-	parser.expect<std::string>("goal");
+	tokenizer.expect<std::string>("goal");
 
 	ExpressionContext expressionContext(m_domain, this);
 
 	m_goal = parsePreconditionExpression(m_context, expressionContext);
 
-	parser.expect<std::string>(")");
+	tokenizer.expect<std::string>(")");
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////

src/plasp/pddl/Requirement.cpp

@@ -4,7 +4,7 @@
 #include <boost/assign.hpp>
 #include <boost/bimap.hpp>
 
-#include <parsebase/ParserException.h>
+#include <tokenize/TokenizerException.h>
 
 namespace plasp
 {
@@ -82,19 +82,19 @@ Requirement::Requirement(Requirement::Type type)
 
 Requirement Requirement::parse(Context &context)
 {
-	auto &parser = context.parser;
+	auto &tokenizer = context.tokenizer;
 
-	const auto requirementName = parser.parseIdentifier();
+	const auto requirementName = tokenizer.getIdentifier();
 
 	const auto match = requirementTypesToPDDL.right.find(requirementName);
 
 	if (match == requirementTypesToPDDL.right.end())
-		throw parsebase::ParserException(parser.location(), "unknown PDDL requirement “" + requirementName + "”");
+		throw tokenize::TokenizerException(tokenizer.location(), "unknown PDDL requirement “" + requirementName + "”");
 
 	const auto requirementType = match->second;
 
 	if (requirementType == Requirement::Type::GoalUtilities)
-		context.logger.log(output::Priority::Warning, parser.location(), "requirement “goal-utilities” is not part of the PDDL 3.1 specification");
+		context.logger.log(output::Priority::Warning, tokenizer.location(), "requirement “goal-utilities” is not part of the PDDL 3.1 specification");
 
 	return Requirement(match->second);
 }

src/plasp/pddl/VariableStack.cpp

@@ -29,13 +29,13 @@ void VariableStack::pop()
 
 expressions::VariablePointer VariableStack::parseAndFind(plasp::pddl::Context &context)
 {
-	auto &parser = context.parser;
+	auto &tokenizer = context.tokenizer;
 
-	parser.skipWhiteSpace();
+	tokenizer.skipWhiteSpace();
 
-	parser.expect<std::string>("?");
+	tokenizer.expect<std::string>("?");
 
-	const auto variableName = parser.parseIdentifier();
+	const auto variableName = tokenizer.getIdentifier();
 
 	for (auto i = m_variableStack.crbegin(); i != m_variableStack.crend(); i++)
 	{
@@ -53,7 +53,7 @@ expressions::VariablePointer VariableStack::parseAndFind(plasp::pddl::Context &c
 			return match->get();
 	}
 
-	throw parsebase::ParserException(parser.location(), "variable “" + variableName + "” used but never declared");
+	throw tokenize::TokenizerException(tokenizer.location(), "variable “" + variableName + "” used but never declared");
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////

src/plasp/pddl/expressions/Constant.cpp

@@ -10,7 +10,7 @@
 #include <plasp/pddl/Problem.h>
 #include <plasp/pddl/expressions/PrimitiveType.h>
 
-#include <parsebase/ParserException.h>
+#include <tokenize/TokenizerException.h>
 
 namespace plasp
 {
@@ -35,11 +35,11 @@ Constant::Constant()
 
 ConstantPointer Constant::parseDeclaration(Context &context)
 {
-	context.parser.skipWhiteSpace();
+	context.tokenizer.skipWhiteSpace();
 
 	auto constant = ConstantPointer(new Constant);
 
-	constant->m_name = context.parser.parseIdentifier();
+	constant->m_name = context.tokenizer.getIdentifier();
 
 	BOOST_ASSERT(constant->m_name != "-");
 
@@ -70,10 +70,10 @@ void Constant::parseTypedDeclaration(Context &context, Domain &domain, Constants
 	// Parse and store constant
 	constants.emplace_back(parseDeclaration(context));
 
-	context.parser.skipWhiteSpace();
+	context.tokenizer.skipWhiteSpace();
 
 	// Check for typing information
-	if (!context.parser.testAndSkip<char>('-'))
+	if (!context.tokenizer.testAndSkip<char>('-'))
 		return;
 
 	// If existing, parse and store parent type
@@ -95,13 +95,13 @@ void Constant::parseTypedDeclaration(Context &context, Domain &domain, Constants
 
 void Constant::parseTypedDeclarations(Context &context, Domain &domain)
 {
-	auto &parser = context.parser;
+	auto &tokenizer = context.tokenizer;
 
-	while (parser.currentCharacter() != ')')
+	while (tokenizer.currentCharacter() != ')')
 	{
 		parseTypedDeclaration(context, domain);
 
-		parser.skipWhiteSpace();
+		tokenizer.skipWhiteSpace();
 	}
 
 	if (domain.constants().empty())
@@ -115,20 +115,20 @@ void Constant::parseTypedDeclarations(Context &context, Domain &domain)
 		domain.checkRequirement(Requirement::Type::Typing);
 	// If no types are given, check that typing is not a requirement
 	else if (domain.hasRequirement(Requirement::Type::Typing))
-		throw parsebase::ParserException(parser.location(), "constant has undeclared type");
+		throw tokenize::TokenizerException(tokenizer.location(), "constant has undeclared type");
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
 void Constant::parseTypedDeclarations(Context &context, Problem &problem)
 {
-	auto &parser = context.parser;
+	auto &tokenizer = context.tokenizer;
 
-	while (context.parser.currentCharacter() != ')')
+	while (tokenizer.currentCharacter() != ')')
 	{
 		parseTypedDeclaration(context, problem);
 
-		parser.skipWhiteSpace();
+		tokenizer.skipWhiteSpace();
 	}
 
 	if (problem.objects().empty())
@@ -142,36 +142,36 @@ void Constant::parseTypedDeclarations(Context &context, Problem &problem)
 		problem.checkRequirement(Requirement::Type::Typing);
 	// If no types are given, check that typing is not a requirement
 	else if (problem.hasRequirement(Requirement::Type::Typing))
-		throw parsebase::ParserException(parser.location(), "constant has undeclared type");
+		throw tokenize::TokenizerException(tokenizer.location(), "constant has undeclared type");
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
 ConstantPointer Constant::parseAndFind(Context &context, const Domain &domain)
 {
-	auto &parser = context.parser;
+	auto &tokenizer = context.tokenizer;
 
-	parser.skipWhiteSpace();
+	tokenizer.skipWhiteSpace();
 
-	const auto constantName = parser.parseIdentifier();
+	const auto constantName = tokenizer.getIdentifier();
 
 	auto constant = parseAndFind(constantName, domain.constants());
 
 	if (constant != nullptr)
 		return constant;
 
-	throw parsebase::ParserException(parser.location(), "constant “" + constantName + "” used but never declared");
+	throw tokenize::TokenizerException(tokenizer.location(), "constant “" + constantName + "” used but never declared");
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
 ConstantPointer Constant::parseAndFind(Context &context, const Problem &problem)
 {
-	auto &parser = context.parser;
+	auto &tokenizer = context.tokenizer;
 
-	parser.skipWhiteSpace();
+	tokenizer.skipWhiteSpace();
 
-	const auto constantName = parser.parseIdentifier();
+	const auto constantName = tokenizer.getIdentifier();
 
 	auto constant = parseAndFind(constantName, problem.domain().constants());
 
@@ -183,7 +183,7 @@ ConstantPointer Constant::parseAndFind(Context &context, const Problem &problem)
 	if (constant)
 		return constant;
 
-	throw parsebase::ParserException(parser.location(), "constant “" + constantName + "” used but never declared");
+	throw tokenize::TokenizerException(tokenizer.location(), "constant “" + constantName + "” used but never declared");
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////

src/plasp/pddl/expressions/Predicate.cpp

@@ -29,17 +29,17 @@ Predicate::Predicate()
 
 PredicatePointer Predicate::parse(Context &context, ExpressionContext &expressionContext)
 {
-	auto &parser = context.parser;
+	auto &tokenizer = context.tokenizer;
 
-	const auto position = parser.position();
+	const auto position = tokenizer.position();
 
-	if (!parser.testAndSkip<std::string>("("))
+	if (!tokenizer.testAndSkip<std::string>("("))
 	{
-		parser.seek(position);
+		tokenizer.seek(position);
 		return nullptr;
 	}
 
-	const auto predicateName = parser.parseIdentifier();
+	const auto predicateName = tokenizer.getIdentifier();
 	const auto &predicates = expressionContext.domain.predicates();
 
 	const auto matchingPredicate = std::find_if(predicates.cbegin(), predicates.cend(),
@@ -50,7 +50,7 @@ PredicatePointer Predicate::parse(Context &context, ExpressionContext &expressio
 
 	if (matchingPredicate == predicates.cend())
 	{
-		parser.seek(position);
+		tokenizer.seek(position);
 		return nullptr;
 	}
 
@@ -58,13 +58,13 @@ PredicatePointer Predicate::parse(Context &context, ExpressionContext &expressio
 
 	predicate->m_name = predicateName;
 
-	context.parser.skipWhiteSpace();
+	tokenizer.skipWhiteSpace();
 
 	// Parse arguments
-	while (context.parser.currentCharacter() != ')')
+	while (tokenizer.currentCharacter() != ')')
 	{
 		// Parse variables
-		if (context.parser.currentCharacter() == '?')
+		if (tokenizer.currentCharacter() == '?')
 		{
 			const auto variable = expressionContext.variables.parseAndFind(context);
 			predicate->m_arguments.emplace_back(variable);
@@ -78,12 +78,12 @@ PredicatePointer Predicate::parse(Context &context, ExpressionContext &expressio
 			predicate->m_arguments.emplace_back(constant);
 		}
 
-		context.parser.skipWhiteSpace();
+		tokenizer.skipWhiteSpace();
 	}
 
 	// TODO: check that signature matches one of the declared ones
 
-	parser.expect<std::string>(")");
+	tokenizer.expect<std::string>(")");
 
 	return predicate;
 }
@@ -92,17 +92,17 @@ PredicatePointer Predicate::parse(Context &context, ExpressionContext &expressio
 
 PredicatePointer Predicate::parse(Context &context, const Problem &problem)
 {
-	auto &parser = context.parser;
+	auto &tokenizer = context.tokenizer;
 
-	const auto position = parser.position();
+	const auto position = tokenizer.position();
 
-	if (!parser.testAndSkip<std::string>("("))
+	if (!tokenizer.testAndSkip<std::string>("("))
 	{
-		parser.seek(position);
+		tokenizer.seek(position);
 		return nullptr;
 	}
 
-	const auto predicateName = parser.parseIdentifier();
+	const auto predicateName = tokenizer.getIdentifier();
 	const auto &predicates = problem.domain().predicates();
 
 	const auto matchingPredicate = std::find_if(predicates.cbegin(), predicates.cend(),
@@ -113,7 +113,7 @@ PredicatePointer Predicate::parse(Context &context, const Problem &problem)
 
 	if (matchingPredicate == predicates.cend())
 	{
-		parser.seek(position);
+		tokenizer.seek(position);
 		return nullptr;
 	}
 
@@ -121,12 +121,12 @@ PredicatePointer Predicate::parse(Context &context, const Problem &problem)
 
 	predicate->m_name = predicateName;
 
-	parser.skipWhiteSpace();
+	tokenizer.skipWhiteSpace();
 
-	while (parser.currentCharacter() != ')')
+	while (tokenizer.currentCharacter() != ')')
 	{
-		if (parser.currentCharacter() == '?')
+		if (tokenizer.currentCharacter() == '?')
-			throw parsebase::ParserException(parser.location(), "variables not allowed in this context");
+			throw tokenize::TokenizerException(tokenizer.location(), "variables not allowed in this context");
 
 		// Parse objects and constants
 		const auto constant = Constant::parseAndFind(context, problem);
@@ -135,7 +135,7 @@ PredicatePointer Predicate::parse(Context &context, const Problem &problem)
 
 	// TODO: check that signature matches one of the declared ones
 
-	parser.expect<std::string>(")");
+	tokenizer.expect<std::string>(")");
 
 	return predicate;
 }

src/plasp/pddl/expressions/PredicateDeclaration.cpp

@@ -28,16 +28,18 @@ PredicateDeclaration::PredicateDeclaration()
 
 void PredicateDeclaration::parse(Context &context, Domain &domain)
 {
-	context.parser.expect<std::string>("(");
+	auto &tokenizer = context.tokenizer;
+
+	tokenizer.expect<std::string>("(");
 
 	auto predicate = PredicateDeclarationPointer(new PredicateDeclaration);
 
-	predicate->m_name = context.parser.parseIdentifier();
+	predicate->m_name = tokenizer.getIdentifier();
 
 	// Flag predicate as correctly declared in the types section
 	predicate->setDeclared();
 
-	context.parser.skipWhiteSpace();
+	tokenizer.skipWhiteSpace();
 
 	ExpressionContext expressionContext(domain);
 	expressionContext.variables.push(&predicate->m_parameters);
@@ -45,7 +47,7 @@ void PredicateDeclaration::parse(Context &context, Domain &domain)
 	// Parse parameters
 	Variable::parseTypedDeclarations(context, expressionContext, predicate->m_parameters);
 
-	context.parser.expect<std::string>(")");
+	tokenizer.expect<std::string>(")");
 
 	domain.predicates().emplace_back(std::move(predicate));
 }

src/plasp/pddl/expressions/PrimitiveType.cpp

@@ -8,7 +8,7 @@
 #include <plasp/pddl/Domain.h>
 #include <plasp/pddl/ExpressionContext.h>
 
-#include <parsebase/ParserException.h>
+#include <tokenize/TokenizerException.h>
 
 namespace plasp
 {
@@ -43,9 +43,9 @@ void PrimitiveType::parseDeclaration(Context &context, Domain &domain)
 {
 	auto &types = domain.types();
 
-	context.parser.skipWhiteSpace();
+	context.tokenizer.skipWhiteSpace();
 
-	const auto typeName = context.parser.parseIdentifier();
+	const auto typeName = context.tokenizer.getIdentifier();
 
 	const auto match = std::find_if(types.cbegin(), types.cend(),
 		[&](const auto &primitiveType)
@@ -75,10 +75,10 @@ void PrimitiveType::parseTypedDeclaration(Context &context, Domain &domain)
 	// Parse and store type
 	parseDeclaration(context, domain);
 
-	context.parser.skipWhiteSpace();
+	context.tokenizer.skipWhiteSpace();
 
 	// Check for type inheritance
-	if (!context.parser.testAndSkip<char>('-'))
+	if (!context.tokenizer.testAndSkip<char>('-'))
 		return;
 
 	domain.checkRequirement(Requirement::Type::Typing);
@@ -104,16 +104,16 @@ void PrimitiveType::parseTypedDeclaration(Context &context, Domain &domain)
 
 PrimitiveTypePointer PrimitiveType::parseAndFind(Context &context, Domain &domain)
 {
-	auto &parser = context.parser;
+	auto &tokenizer = context.tokenizer;
 
 	auto &types = domain.types();
 
-	parser.skipWhiteSpace();
+	tokenizer.skipWhiteSpace();
 
-	const auto typeName = parser.parseIdentifier();
+	const auto typeName = tokenizer.getIdentifier();
 
 	if (typeName.empty())
-		throw parsebase::ParserException(parser.location(), "no type supplied");
+		throw tokenize::TokenizerException(tokenizer.location(), "no type supplied");
 
 	const auto match = std::find_if(types.cbegin(), types.cend(),
 		[&](const auto &primitiveType)
@@ -126,11 +126,11 @@ PrimitiveTypePointer PrimitiveType::parseAndFind(Context &context, Domain &domai
 		// Only "object" is allowed as an implicit type
 		if (typeName == "object" || typeName == "objects")
 		{
-			context.logger.log(output::Priority::Warning, parser.location(), "primitive type “" + typeName + "” should be declared");
+			context.logger.log(output::Priority::Warning, tokenizer.location(), "primitive type “" + typeName + "” should be declared");
 			types.emplace_back(PrimitiveTypePointer(new PrimitiveType(typeName)));
 		}
 		else
-			throw parsebase::ParserException(parser.location(), "type “" + typeName + "” used but never declared");
+			throw tokenize::TokenizerException(tokenizer.location(), "type “" + typeName + "” used but never declared");
 
 		return types.back().get();
 	}

src/plasp/pddl/expressions/Unsupported.cpp

@@ -17,17 +17,17 @@ namespace expressions
 
 UnsupportedPointer Unsupported::parse(Context &context)
 {
-	auto &parser = context.parser;
+	auto &tokenizer = context.tokenizer;
 
 	auto expression = UnsupportedPointer(new Unsupported);
 
-	parser.expect<std::string>("(");
+	tokenizer.expect<std::string>("(");
 
-	expression->m_type = parser.parseIdentifier();
+	expression->m_type = tokenizer.getIdentifier();
 
-	context.logger.log(output::Priority::Warning, parser.location(), "expression type “" + expression->m_type + "” currently unsupported in this context");
+	context.logger.log(output::Priority::Warning, tokenizer.location(), "expression type “" + expression->m_type + "” currently unsupported in this context");
 
-	skipSection(parser);
+	skipSection(tokenizer);
 
 	return expression;
 }

src/plasp/pddl/expressions/Variable.cpp

@@ -11,7 +11,7 @@
 #include <plasp/pddl/expressions/PrimitiveType.h>
 #include <plasp/pddl/expressions/Type.h>
 
-#include <parsebase/ParserException.h>
+#include <tokenize/TokenizerException.h>
 
 namespace plasp
 {
@@ -43,15 +43,15 @@ Variable::Variable(std::string name)
 
 void Variable::parseDeclaration(Context &context, Variables &parameters)
 {
-	auto &parser = context.parser;
+	auto &tokenizer = context.tokenizer;
 
-	parser.skipWhiteSpace();
+	tokenizer.skipWhiteSpace();
 
-	parser.expect<std::string>("?");
+	tokenizer.expect<std::string>("?");
 
 	auto variable = VariablePointer(new Variable);
 
-	variable->m_name = parser.parseIdentifier();
+	variable->m_name = tokenizer.getIdentifier();
 
 	// Check if variable of that name already exists in the current scope
 	const auto match = std::find_if(parameters.cbegin(), parameters.cend(),
@@ -61,7 +61,7 @@ void Variable::parseDeclaration(Context &context, Variables &parameters)
 		});
 
 	if (match != parameters.cend())
-		throw parsebase::ParserException(parser.location(), "variable “" + variable->m_name + "” already declared in this scope");
+		throw tokenize::TokenizerException(tokenizer.location(), "variable “" + variable->m_name + "” already declared in this scope");
 
 	// Flag variable for potentially upcoming type declaration
 	variable->setDirty();
@@ -74,17 +74,17 @@ void Variable::parseDeclaration(Context &context, Variables &parameters)
 void Variable::parseTypedDeclaration(Context &context, ExpressionContext &expressionContext,
 		Variables &variables)
 {
-	auto &parser = context.parser;
+	auto &tokenizer = context.tokenizer;
 
 	// Parse and store variable itself
 	parseDeclaration(context, variables);
 
 	auto variable = variables.back();
 
-	parser.skipWhiteSpace();
+	tokenizer.skipWhiteSpace();
 
 	// Check if the variable has a type declaration
-	if (!parser.testAndSkip<char>('-'))
+	if (!tokenizer.testAndSkip<char>('-'))
 		return;
 
 	const auto setType =
@@ -101,7 +101,7 @@ void Variable::parseTypedDeclaration(Context &context, ExpressionContext &expres
 			}
 		};
 
-	parser.skipWhiteSpace();
+	tokenizer.skipWhiteSpace();
 
 	// Parse argument if it has "either" type (always begins with opening parenthesis)
 	variable->m_type = Either::parse(context, expressionContext, parseExistingPrimitiveType);
@@ -118,13 +118,13 @@ void Variable::parseTypedDeclaration(Context &context, ExpressionContext &expres
 void Variable::parseTypedDeclarations(Context &context, ExpressionContext &expressionContext,
 	Variables &variables)
 {
-	auto &parser = context.parser;
+	auto &tokenizer = context.tokenizer;
 
-	while (parser.currentCharacter() != ')')
+	while (tokenizer.currentCharacter() != ')')
 	{
 		parseTypedDeclaration(context, expressionContext, variables);
 
-		parser.skipWhiteSpace();
+		tokenizer.skipWhiteSpace();
 	}
 
 	if (variables.empty())
@@ -138,7 +138,7 @@ void Variable::parseTypedDeclarations(Context &context, ExpressionContext &expre
 		expressionContext.checkRequirement(Requirement::Type::Typing);
 	// If no types are given, check that typing is not a requirement
 	else if (expressionContext.hasRequirement(Requirement::Type::Typing))
-		throw parsebase::ParserException(parser.location(), "variable has undeclared type");
+		throw tokenize::TokenizerException(tokenizer.location(), "variable has undeclared type");
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////

src/plasp/sas/AssignedVariable.cpp

@@ -31,24 +31,24 @@ AssignedVariable::AssignedVariable(const Variable &variable, const Value &value)
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-AssignedVariable AssignedVariable::fromSAS(parsebase::Parser<> &parser, const Variables &variables)
+AssignedVariable AssignedVariable::fromSAS(tokenize::Tokenizer<> &tokenizer, const Variables &variables)
 {
 	AssignedVariable assignedVariable;
 
-	assignedVariable.m_variable = &Variable::referenceFromSAS(parser, variables);
+	assignedVariable.m_variable = &Variable::referenceFromSAS(tokenizer, variables);
-	assignedVariable.m_value = &Value::referenceFromSAS(parser, *assignedVariable.m_variable);
+	assignedVariable.m_value = &Value::referenceFromSAS(tokenizer, *assignedVariable.m_variable);
 
 	return assignedVariable;
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-AssignedVariable AssignedVariable::fromSAS(parsebase::Parser<> &parser, const Variable &variable)
+AssignedVariable AssignedVariable::fromSAS(tokenize::Tokenizer<> &tokenizer, const Variable &variable)
 {
 	AssignedVariable assignedVariable;
 
 	assignedVariable.m_variable = &variable;
-	assignedVariable.m_value = &Value::referenceFromSAS(parser, *assignedVariable.m_variable);
+	assignedVariable.m_value = &Value::referenceFromSAS(tokenizer, *assignedVariable.m_variable);
 
 	return assignedVariable;
 }

src/plasp/sas/AxiomRule.cpp

@@ -23,24 +23,24 @@ AxiomRule::AxiomRule(AxiomRule::Conditions conditions, AxiomRule::Condition post
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-AxiomRule AxiomRule::fromSAS(parsebase::Parser<> &parser, const Variables &variables)
+AxiomRule AxiomRule::fromSAS(tokenize::Tokenizer<> &tokenizer, const Variables &variables)
 {
-	parser.expect<std::string>("begin_rule");
+	tokenizer.expect<std::string>("begin_rule");
 
-	const auto numberOfConditions = parser.parse<size_t>();
+	const auto numberOfConditions = tokenizer.get<size_t>();
 
 	Conditions conditions;
 	conditions.reserve(numberOfConditions);
 
 	for (size_t j = 0; j < numberOfConditions; j++)
-		conditions.emplace_back(Condition::fromSAS(parser, variables));
+		conditions.emplace_back(Condition::fromSAS(tokenizer, variables));
 
-	const auto variableTransition = VariableTransition::fromSAS(parser, variables);
+	const auto variableTransition = VariableTransition::fromSAS(tokenizer, variables);
 
 	if (&variableTransition.valueBefore() != &Value::Any)
 		conditions.emplace_back(Condition(variableTransition.variable(), variableTransition.valueBefore()));
 
-	parser.expect<std::string>("end_rule");
+	tokenizer.expect<std::string>("end_rule");
 
 	const Condition postcondition(variableTransition.variable(), variableTransition.valueAfter());
 	const AxiomRule axiomRule(std::move(conditions), std::move(postcondition));

src/plasp/sas/Description.cpp

@@ -7,7 +7,7 @@
 
 #include <plasp/sas/VariableTransition.h>
 
-#include <parsebase/ParserException.h>
+#include <tokenize/TokenizerException.h>
 
 namespace plasp
 {
@@ -27,10 +27,10 @@ Description::Description()
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-Description Description::fromParser(parsebase::Parser<> &&parser)
+Description Description::fromTokenizer(tokenize::Tokenizer<> &&tokenizer)
 {
 	Description description;
-	description.parseContent(parser);
+	description.parseContent(tokenizer);
 
 	return description;
 }
@@ -39,11 +39,11 @@ Description Description::fromParser(parsebase::Parser<> &&parser)
 
 Description Description::fromStream(std::istream &istream)
 {
-	parsebase::Parser<> parser;
+	tokenize::Tokenizer<> tokenizer;
-	parser.read("std::cin", istream);
+	tokenizer.read("std::cin", istream);
 
 	Description description;
-	description.parseContent(parser);
+	description.parseContent(tokenizer);
 
 	return description;
 }
@@ -55,11 +55,11 @@ Description Description::fromFile(const std::experimental::filesystem::path &pat
 	if (!std::experimental::filesystem::is_regular_file(path))
 		throw std::runtime_error("File does not exist: “" + path.string() + "”");
 
-	parsebase::Parser<> parser;
+	tokenize::Tokenizer<> tokenizer;
-	parser.read(path);
+	tokenizer.read(path);
 
 	Description description;
-	description.parseContent(parser);
+	description.parseContent(tokenizer);
 
 	return description;
 }
@@ -159,104 +159,104 @@ bool Description::hasRequirements() const
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-void Description::parseContent(parsebase::Parser<> &parser)
+void Description::parseContent(tokenize::Tokenizer<> &tokenizer)
 {
-	parseVersionSection(parser);
+	parseVersionSection(tokenizer);
-	parseMetricSection(parser);
+	parseMetricSection(tokenizer);
-	parseVariablesSection(parser);
+	parseVariablesSection(tokenizer);
-	parseMutexSection(parser);
+	parseMutexSection(tokenizer);
-	parseInitialStateSection(parser);
+	parseInitialStateSection(tokenizer);
-	parseGoalSection(parser);
+	parseGoalSection(tokenizer);
-	parseOperatorSection(parser);
+	parseOperatorSection(tokenizer);
-	parseAxiomSection(parser);
+	parseAxiomSection(tokenizer);
 
-	parser.skipWhiteSpace();
+	tokenizer.skipWhiteSpace();
 
-	if (!parser.atEnd())
+	if (!tokenizer.atEnd())
-		throw parsebase::ParserException(parser.location(), "expected end of SAS description (perhaps, input contains two SAS descriptions?)");
+		throw tokenize::TokenizerException(tokenizer.location(), "expected end of SAS description (perhaps, input contains two SAS descriptions?)");
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-void Description::parseVersionSection(parsebase::Parser<> &parser) const
+void Description::parseVersionSection(tokenize::Tokenizer<> &tokenizer) const
 {
-	parser.expect<std::string>("begin_version");
+	tokenizer.expect<std::string>("begin_version");
 
-	const auto formatVersion = parser.parse<size_t>();
+	const auto formatVersion = tokenizer.get<size_t>();
 
 	if (formatVersion != 3)
-		throw parsebase::ParserException(parser.location(), "unsupported SAS format version (" + std::to_string(formatVersion) + ")");
+		throw tokenize::TokenizerException(tokenizer.location(), "unsupported SAS format version (" + std::to_string(formatVersion) + ")");
 
-	parser.expect<std::string>("end_version");
+	tokenizer.expect<std::string>("end_version");
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-void Description::parseMetricSection(parsebase::Parser<> &parser)
+void Description::parseMetricSection(tokenize::Tokenizer<> &tokenizer)
 {
-	parser.expect<std::string>("begin_metric");
+	tokenizer.expect<std::string>("begin_metric");
 
-	m_usesActionCosts = parser.parse<bool>();
+	m_usesActionCosts = tokenizer.get<bool>();
 
-	parser.expect<std::string>("end_metric");
+	tokenizer.expect<std::string>("end_metric");
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-void Description::parseVariablesSection(parsebase::Parser<> &parser)
+void Description::parseVariablesSection(tokenize::Tokenizer<> &tokenizer)
 {
-	const auto numberOfVariables = parser.parse<size_t>();
+	const auto numberOfVariables = tokenizer.get<size_t>();
 	m_variables.reserve(numberOfVariables);
 
 	for (size_t i = 0; i < numberOfVariables; i++)
-		m_variables.emplace_back(Variable::fromSAS(parser));
+		m_variables.emplace_back(Variable::fromSAS(tokenizer));
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-void Description::parseMutexSection(parsebase::Parser<> &parser)
+void Description::parseMutexSection(tokenize::Tokenizer<> &tokenizer)
 {
-	const auto numberOfMutexGroups = parser.parse<size_t>();
+	const auto numberOfMutexGroups = tokenizer.get<size_t>();
 	m_mutexGroups.reserve(numberOfMutexGroups);
 
 	for (size_t i = 0; i < numberOfMutexGroups; i++)
-		m_mutexGroups.emplace_back(MutexGroup::fromSAS(parser, m_variables));
+		m_mutexGroups.emplace_back(MutexGroup::fromSAS(tokenizer, m_variables));
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-void Description::parseInitialStateSection(parsebase::Parser<> &parser)
+void Description::parseInitialStateSection(tokenize::Tokenizer<> &tokenizer)
 {
-	m_initialState = std::make_unique<InitialState>(InitialState::fromSAS(parser, m_variables));
+	m_initialState = std::make_unique<InitialState>(InitialState::fromSAS(tokenizer, m_variables));
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-void Description::parseGoalSection(parsebase::Parser<> &parser)
+void Description::parseGoalSection(tokenize::Tokenizer<> &tokenizer)
 {
-	m_goal = std::make_unique<Goal>(Goal::fromSAS(parser, m_variables));
+	m_goal = std::make_unique<Goal>(Goal::fromSAS(tokenizer, m_variables));
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-void Description::parseOperatorSection(parsebase::Parser<> &parser)
+void Description::parseOperatorSection(tokenize::Tokenizer<> &tokenizer)
 {
-	const auto numberOfOperators = parser.parse<size_t>();
+	const auto numberOfOperators = tokenizer.get<size_t>();
 	m_operators.reserve(numberOfOperators);
 
 	for (size_t i = 0; i < numberOfOperators; i++)
-		m_operators.emplace_back(Operator::fromSAS(parser, m_variables));
+		m_operators.emplace_back(Operator::fromSAS(tokenizer, m_variables));
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-void Description::parseAxiomSection(parsebase::Parser<> &parser)
+void Description::parseAxiomSection(tokenize::Tokenizer<> &tokenizer)
 {
-	const auto numberOfAxiomRules = parser.parse<size_t>();
+	const auto numberOfAxiomRules = tokenizer.get<size_t>();
 	m_axiomRules.reserve(numberOfAxiomRules);
 
 	for (size_t i = 0; i < numberOfAxiomRules; i++)
-		m_axiomRules.emplace_back(AxiomRule::fromSAS(parser, m_variables));
+		m_axiomRules.emplace_back(AxiomRule::fromSAS(tokenizer, m_variables));
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////

src/plasp/sas/Effect.cpp

@@ -23,17 +23,17 @@ Effect::Effect(Conditions conditions, Condition postcondition)
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-Effect Effect::fromSAS(parsebase::Parser<> &parser, const Variables &variables, Conditions &preconditions)
+Effect Effect::fromSAS(tokenize::Tokenizer<> &tokenizer, const Variables &variables, Conditions &preconditions)
 {
 	Effect::Conditions conditions;
 
-	const auto numberOfEffectConditions = parser.parse<size_t>();
+	const auto numberOfEffectConditions = tokenizer.get<size_t>();
 	conditions.reserve(numberOfEffectConditions);
 
 	for (size_t k = 0; k < numberOfEffectConditions; k++)
-		conditions.emplace_back(Condition::fromSAS(parser, variables));
+		conditions.emplace_back(Condition::fromSAS(tokenizer, variables));
 
-	const auto variableTransition = VariableTransition::fromSAS(parser, variables);
+	const auto variableTransition = VariableTransition::fromSAS(tokenizer, variables);
 
 	if (&variableTransition.valueBefore() != &Value::Any)
 		preconditions.emplace_back(Condition(variableTransition.variable(), variableTransition.valueBefore()));

src/plasp/sas/Goal.cpp

@@ -13,19 +13,19 @@ namespace sas
 //
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-Goal Goal::fromSAS(parsebase::Parser<> &parser, const Variables &variables)
+Goal Goal::fromSAS(tokenize::Tokenizer<> &tokenizer, const Variables &variables)
 {
 	Goal goal;
 
-	parser.expect<std::string>("begin_goal");
+	tokenizer.expect<std::string>("begin_goal");
 
-	const auto numberOfGoalFacts = parser.parse<size_t>();
+	const auto numberOfGoalFacts = tokenizer.get<size_t>();
 	goal.m_facts.reserve(numberOfGoalFacts);
 
 	for (size_t i = 0; i < numberOfGoalFacts; i++)
-		goal.m_facts.emplace_back(Fact::fromSAS(parser, variables));
+		goal.m_facts.emplace_back(Fact::fromSAS(tokenizer, variables));
 
-	parser.expect<std::string>("end_goal");
+	tokenizer.expect<std::string>("end_goal");
 
 	return goal;
 }

src/plasp/sas/InitialState.cpp

@@ -11,18 +11,18 @@ namespace sas
 //
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-InitialState InitialState::fromSAS(parsebase::Parser<> &parser, const Variables &variables)
+InitialState InitialState::fromSAS(tokenize::Tokenizer<> &tokenizer, const Variables &variables)
 {
 	InitialState initialState;
 
-	parser.expect<std::string>("begin_state");
+	tokenizer.expect<std::string>("begin_state");
 
 	initialState.m_facts.reserve(variables.size());
 
 	for (size_t i = 0; i < variables.size(); i++)
-		initialState.m_facts.emplace_back(Fact::fromSAS(parser, variables[i]));
+		initialState.m_facts.emplace_back(Fact::fromSAS(tokenizer, variables[i]));
 
-	parser.expect<std::string>("end_state");
+	tokenizer.expect<std::string>("end_state");
 
 	return initialState;
 }

src/plasp/sas/MutexGroup.cpp

@@ -2,7 +2,7 @@
 
 #include <iostream>
 
-#include <parsebase/ParserException.h>
+#include <tokenize/TokenizerException.h>
 
 namespace plasp
 {
@@ -15,24 +15,24 @@ namespace sas
 //
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-MutexGroup MutexGroup::fromSAS(parsebase::Parser<> &parser, const Variables &variables)
+MutexGroup MutexGroup::fromSAS(tokenize::Tokenizer<> &tokenizer, const Variables &variables)
 {
 	MutexGroup mutexGroup;
 
-	parser.expect<std::string>("begin_mutex_group");
+	tokenizer.expect<std::string>("begin_mutex_group");
 
-	const auto numberOfFacts = parser.parse<size_t>();
+	const auto numberOfFacts = tokenizer.get<size_t>();
 	mutexGroup.m_facts.reserve(numberOfFacts);
 
 	for (size_t j = 0; j < numberOfFacts; j++)
 	{
-		mutexGroup.m_facts.emplace_back(Fact::fromSAS(parser, variables));
+		mutexGroup.m_facts.emplace_back(Fact::fromSAS(tokenizer, variables));
 
 		if (mutexGroup.m_facts[j].value() == Value::None)
-			throw parsebase::ParserException(parser.location(), "mutex groups must not contain <none of those> values");
+			throw tokenize::TokenizerException(tokenizer.location(), "mutex groups must not contain <none of those> values");
 	}
 
-	parser.expect<std::string>("end_mutex_group");
+	tokenizer.expect<std::string>("end_mutex_group");
 
 	return mutexGroup;
 }

src/plasp/sas/Operator.cpp

@@ -17,29 +17,29 @@ namespace sas
 //
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-Operator Operator::fromSAS(parsebase::Parser<> &parser, const Variables &variables)
+Operator Operator::fromSAS(tokenize::Tokenizer<> &tokenizer, const Variables &variables)
 {
 	Operator operator_;
 
-	parser.expect<std::string>("begin_operator");
+	tokenizer.expect<std::string>("begin_operator");
 
-	operator_.m_predicate = Predicate::fromSAS(parser);
+	operator_.m_predicate = Predicate::fromSAS(tokenizer);
 
-	const auto numberOfPrevailConditions = parser.parse<size_t>();
+	const auto numberOfPrevailConditions = tokenizer.get<size_t>();
 	operator_.m_preconditions.reserve(numberOfPrevailConditions);
 
 	for (size_t j = 0; j < numberOfPrevailConditions; j++)
-		operator_.m_preconditions.emplace_back(Condition::fromSAS(parser, variables));
+		operator_.m_preconditions.emplace_back(Condition::fromSAS(tokenizer, variables));
 
-	const auto numberOfEffects = parser.parse<size_t>();
+	const auto numberOfEffects = tokenizer.get<size_t>();
 	operator_.m_effects.reserve(numberOfEffects);
 
 	for (size_t j = 0; j < numberOfEffects; j++)
-		operator_.m_effects.emplace_back(Effect::fromSAS(parser, variables, operator_.m_preconditions));
+		operator_.m_effects.emplace_back(Effect::fromSAS(tokenizer, variables, operator_.m_preconditions));
 
-	operator_.m_costs = parser.parse<size_t>();
+	operator_.m_costs = tokenizer.get<size_t>();
 
-	parser.expect<std::string>("end_operator");
+	tokenizer.expect<std::string>("end_operator");
 
 	return operator_;
 }

src/plasp/sas/Predicate.cpp

@@ -6,7 +6,7 @@
 
 #include <plasp/output/Formatting.h>
 
-#include <parsebase/ParserException.h>
+#include <tokenize/TokenizerException.h>
 
 namespace plasp
 {
@@ -19,32 +19,32 @@ namespace sas
 //
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-Predicate Predicate::fromSAS(parsebase::Parser<> &parser)
+Predicate Predicate::fromSAS(tokenize::Tokenizer<> &tokenizer)
 {
 	Predicate predicate;
 
 	try
 	{
-		parser.skipLine();
+		tokenizer.skipLine();
 
-		predicate.m_name = parser.parse<std::string>();
+		predicate.m_name = tokenizer.get<std::string>();
 
 		while (true)
 		{
 			// Skip whitespace but not newlines
-			parser.skipBlankSpace();
+			tokenizer.skipBlankSpace();
 
 			// TODO: check \r handling
-			if (parser.currentCharacter() == '\n')
+			if (tokenizer.currentCharacter() == '\n')
 				break;
 
-			const auto value = parser.parse<std::string>();
+			const auto value = tokenizer.get<std::string>();
 			predicate.m_arguments.emplace_back(std::move(value));
 		}
 	}
 	catch (const std::exception &e)
 	{
-		throw parsebase::ParserException(parser.location(), "could not parse operator predicate");
+		throw tokenize::TokenizerException(tokenizer.location(), "could not parse operator predicate");
 	}
 
 	return predicate;

src/plasp/sas/Value.cpp

@@ -5,7 +5,7 @@
 #include <plasp/output/Formatting.h>
 #include <plasp/sas/Variable.h>
 
-#include <parsebase/ParserException.h>
+#include <tokenize/TokenizerException.h>
 
 namespace plasp
 {
@@ -55,14 +55,14 @@ Value Value::negated() const
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-Value Value::fromSAS(parsebase::Parser<> &parser)
+Value Value::fromSAS(tokenize::Tokenizer<> &tokenizer)
 {
-	const auto sasSign = parser.parse<std::string>();
+	const auto sasSign = tokenizer.get<std::string>();
 
 	if (sasSign == "<none")
 	{
-		parser.expect<std::string>("of");
+		tokenizer.expect<std::string>("of");
-		parser.expect<std::string>("those>");
+		tokenizer.expect<std::string>("those>");
 
 		// TODO: do not return a copy of Value::None
 		return Value::None;
@@ -75,12 +75,12 @@ Value Value::fromSAS(parsebase::Parser<> &parser)
 	else if (sasSign == "NegatedAtom")
 		value.m_sign = Value::Sign::Negative;
 	else
-		throw parsebase::ParserException(parser.location(), "invalid value sign “" + sasSign + "”");
+		throw tokenize::TokenizerException(tokenizer.location(), "invalid value sign “" + sasSign + "”");
 
 	try
 	{
-		parser.skipWhiteSpace();
+		tokenizer.skipWhiteSpace();
-		value.m_name = parser.parseLine();
+		value.m_name = tokenizer.getLine();
 
 		// Remove trailing ()
 		if (value.m_name.find("()") != std::string::npos)
@@ -91,7 +91,7 @@ Value Value::fromSAS(parsebase::Parser<> &parser)
 	}
 	catch (const std::exception &e)
 	{
-		throw parsebase::ParserException(parser.location(), std::string("could not parse variable value (") + e.what() + ")");
+		throw tokenize::TokenizerException(tokenizer.location(), std::string("could not parse variable value (") + e.what() + ")");
 	}
 
 	return value;
@@ -99,15 +99,15 @@ Value Value::fromSAS(parsebase::Parser<> &parser)
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-const Value &Value::referenceFromSAS(parsebase::Parser<> &parser, const Variable &variable)
+const Value &Value::referenceFromSAS(tokenize::Tokenizer<> &tokenizer, const Variable &variable)
 {
-	const auto valueID = parser.parse<int>();
+	const auto valueID = tokenizer.get<int>();
 
 	if (valueID == -1)
 		return Value::Any;
 
 	if (valueID < 0 || static_cast<size_t>(valueID) >= variable.values().size())
-		throw parsebase::ParserException(parser.location(), "value index out of range (variable " + variable.name() + ", index " + std::to_string(valueID) + ")");
+		throw tokenize::TokenizerException(tokenizer.location(), "value index out of range (variable " + variable.name() + ", index " + std::to_string(valueID) + ")");
 
 	return variable.values()[valueID];
 }

src/plasp/sas/Variable.cpp

@@ -4,7 +4,7 @@
 
 #include <plasp/output/Formatting.h>
 
-#include <parsebase/ParserException.h>
+#include <tokenize/TokenizerException.h>
 
 namespace plasp
 {
@@ -24,29 +24,29 @@ Variable::Variable()
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-Variable Variable::fromSAS(parsebase::Parser<> &parser)
+Variable Variable::fromSAS(tokenize::Tokenizer<> &tokenizer)
 {
 	Variable variable;
 
-	parser.expect<std::string>("begin_variable");
+	tokenizer.expect<std::string>("begin_variable");
-	parser.expect<std::string>("var");
+	tokenizer.expect<std::string>("var");
 
-	variable.m_name = parser.parse<std::string>();
+	variable.m_name = tokenizer.get<std::string>();
-	variable.m_axiomLayer = parser.parse<int>();
+	variable.m_axiomLayer = tokenizer.get<int>();
 
-	const auto numberOfValues = parser.parse<size_t>();
+	const auto numberOfValues = tokenizer.get<size_t>();
 	variable.m_values.reserve(numberOfValues);
 
 	for (size_t j = 0; j < numberOfValues; j++)
 	{
-		variable.m_values.emplace_back(Value::fromSAS(parser));
+		variable.m_values.emplace_back(Value::fromSAS(tokenizer));
 
 		// <none of those> values are only allowed at the end
 		if (j < numberOfValues - 1 && variable.m_values[j] == Value::None)
-			throw parsebase::ParserException(parser.location(), "<none of those> value must be the last value of a variable");
+			throw tokenize::TokenizerException(tokenizer.location(), "<none of those> value must be the last value of a variable");
 	}
 
-	parser.expect<std::string>("end_variable");
+	tokenizer.expect<std::string>("end_variable");
 
 	return variable;
 }
@@ -61,12 +61,12 @@ void Variable::printNameAsASPPredicate(output::ColorStream &stream) const
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-const Variable &Variable::referenceFromSAS(parsebase::Parser<> &parser, const Variables &variables)
+const Variable &Variable::referenceFromSAS(tokenize::Tokenizer<> &tokenizer, const Variables &variables)
 {
-	const auto variableID = parser.parse<size_t>();
+	const auto variableID = tokenizer.get<size_t>();
 
 	if (variableID >= variables.size())
-		throw parsebase::ParserException(parser.location(), "variable index out of range (index " + std::to_string(variableID) + ")");
+		throw tokenize::TokenizerException(tokenizer.location(), "variable index out of range (index " + std::to_string(variableID) + ")");
 
 	return variables[variableID];
 }

src/plasp/sas/VariableTransition.cpp

@@ -24,13 +24,13 @@ VariableTransition::VariableTransition()
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-VariableTransition VariableTransition::fromSAS(parsebase::Parser<> &parser, const Variables &variables)
+VariableTransition VariableTransition::fromSAS(tokenize::Tokenizer<> &tokenizer, const Variables &variables)
 {
 	VariableTransition variableTransition;
 
-	variableTransition.m_variable = &Variable::referenceFromSAS(parser, variables);
+	variableTransition.m_variable = &Variable::referenceFromSAS(tokenizer, variables);
-	variableTransition.m_valueBefore = &Value::referenceFromSAS(parser, *variableTransition.m_variable);
+	variableTransition.m_valueBefore = &Value::referenceFromSAS(tokenizer, *variableTransition.m_variable);
-	variableTransition.m_valueAfter = &Value::referenceFromSAS(parser, *variableTransition.m_variable);
+	variableTransition.m_valueAfter = &Value::referenceFromSAS(tokenizer, *variableTransition.m_variable);
 
 	return variableTransition;
 }

tests/CMakeLists.txt

@@ -6,7 +6,7 @@ set(includes
 	${Boost_INCLUDE_DIRS}
 	${PROJECT_SOURCE_DIR}/include
 	${PROJECT_SOURCE_DIR}/lib/catch/single_include
-	${PROJECT_SOURCE_DIR}/lib/parsebase/include
+	${PROJECT_SOURCE_DIR}/lib/tokenize/include
 )
 
 set(libraries

tests/TestPDDLParser.cpp

@@ -20,7 +20,7 @@ using namespace plasp::pddl;
 TEST_CASE("[PDDL parser] The Blocks World domain is parsed correctly", "[PDDL parser]")
 {
 	plasp::output::Logger logger;
-	Context context(Parser(), logger);
+	Context context(Tokenizer(), logger);
 
 	const auto description = Description::fromFile("data/blocksworld-domain.pddl", context);
 
@@ -102,7 +102,7 @@ TEST_CASE("[PDDL parser] The Blocks World domain is parsed correctly", "[PDDL pa
 TEST_CASE("[PDDL parser] A Blocks World problem is parsed correctly", "[PDDL parser]")
 {
 	plasp::output::Logger logger;
-	Context context(Parser(), logger);
+	Context context(Tokenizer(), logger);
 
 	const auto description = Description::fromFiles({"data/blocksworld-domain.pddl", "data/blocksworld-problem.pddl"}, context);
 
@@ -167,7 +167,7 @@ TEST_CASE("[PDDL parser] A Blocks World problem is parsed correctly", "[PDDL par
 TEST_CASE("[PDDL parser] The Storage domain is parsed correctly", "[PDDL parser]")
 {
 	plasp::output::Logger logger;
-	Context context(Parser(), logger);
+	Context context(Tokenizer(), logger);
 
 	const auto description = plasp::pddl::Description::fromFile("data/storage-domain.pddl", context);
 
@@ -261,7 +261,7 @@ TEST_CASE("[PDDL parser] The Storage domain is parsed correctly", "[PDDL parser]
 TEST_CASE("[PDDL parser] A Storage problem is parsed correctly", "[PDDL parser]")
 {
 	plasp::output::Logger logger;
-	Context context(Parser(), logger);
+	Context context(Tokenizer(), logger);
 
 	const auto description = Description::fromFiles({"data/storage-domain.pddl", "data/storage-problem.pddl"}, context);
 
@@ -323,7 +323,7 @@ TEST_CASE("[PDDL parser] A Storage problem is parsed correctly", "[PDDL parser]"
 TEST_CASE("[PDDL parser] Constants are parsed correctly", "[PDDL parser]")
 {
 	plasp::output::Logger logger;
-	Context context(Parser(), logger);
+	Context context(Tokenizer(), logger);
 
 	const auto description = Description::fromFile("data/woodworking-domain.pddl", context);
 
@@ -355,7 +355,7 @@ TEST_CASE("[PDDL parser] Constants are parsed correctly", "[PDDL parser]")
 TEST_CASE("[PDDL parser] White spaces are ignored", "[PDDL parser]")
 {
 	plasp::output::Logger logger;
-	Context context(Parser(), logger);
+	Context context(Tokenizer(), logger);
 
 	CHECK_NOTHROW(Description::fromFile("data/white-space-test.pddl", context));
 }
@@ -365,7 +365,7 @@ TEST_CASE("[PDDL parser] White spaces are ignored", "[PDDL parser]")
 TEST_CASE("[PDDL parser] Missing or unmatching domain descriptions are detected", "[PDDL parser]")
 {
 	plasp::output::Logger logger;
-	Context context(Parser(), logger);
+	Context context(Tokenizer(), logger);
 
 	SECTION("")
 	{
@@ -373,7 +373,7 @@ TEST_CASE("[PDDL parser] Missing or unmatching domain descriptions are detected"
 	}
 	SECTION("")
 	{
-		CHECK_THROWS_AS(Description::fromFiles({"data/blocksworld-problem.pddl", "data/storage-domain.pddl"}, context), parsebase::ParserException);
+		CHECK_THROWS_AS(Description::fromFiles({"data/blocksworld-problem.pddl", "data/storage-domain.pddl"}, context), tokenize::TokenizerException);
 	}
 }
 
@@ -382,7 +382,7 @@ TEST_CASE("[PDDL parser] Missing or unmatching domain descriptions are detected"
 TEST_CASE("[PDDL parser] Common PDDL syntax errors are detected", "[PDDL parser]")
 {
 	plasp::output::Logger logger;
-	Context context(Parser(), logger);
+	Context context(Tokenizer(), logger);
 
 	SECTION("")
 	{
@@ -513,7 +513,7 @@ TEST_CASE("[PDDL parser] Common PDDL syntax errors are detected", "[PDDL parser]
 TEST_CASE("[PDDL parser] Former issues are fixed", "[PDDL parser]")
 {
 	plasp::output::Logger logger;
-	Context context(Parser(), logger);
+	Context context(Tokenizer(), logger);
 
 	SECTION("white space issues with constants and parsing unsupported sections")
 	{

tests/TestPDDLTranslation.cpp

@@ -15,7 +15,7 @@ boost::iostreams::stream<boost::iostreams::null_sink> nullStream((boost::iostrea
 TEST_CASE("[PDDL translation] Former issues are fixed", "[PDDL translation]")
 {
 	plasp::output::Logger logger;
-	Context context(Parser(), logger);
+	Context context(Tokenizer(), logger);
 
 	SECTION("translating domains without typing information works")
 	{