Renamed tokenizing module for clarity.

2017-05-12 14:17:57 +02:00
parent c10187f6ba
commit e312a91632
77 changed files with 854 additions and 861 deletions
--- a/lib/pddlparse/include/pddlparse/AST.h
+++ b/lib/pddlparse/include/pddlparse/AST.h
@@ -7,8 +7,6 @@
 #include <type_traits>
 #include <vector>

-#include <parsebase/Stream.h>
-
 #include <pddlparse/ASTForward.h>

 namespace pddl
--- a/lib/pddlparse/include/pddlparse/detail/Parser.h
+++ b/lib/pddlparse/include/pddlparse/detail/Parser.h
@@ -1,22 +1,20 @@
-#ifndef __PDDL_PARSE__DETAIL__PARSER_H
-#define __PDDL_PARSE__DETAIL__PARSER_H
+#ifndef __PDDL_PARSE__TOKENIZER_H
+#define __PDDL_PARSE__TOKENIZER_H

 #include <iostream>

-#include <parsebase/Parser.h>
+#include <tokenize/Tokenizer.h>

 namespace pddl
 {
-namespace detail
-{

 ////////////////////////////////////////////////////////////////////////////////////////////////////
 //
-// Parser
+// Tokenizer
 //
 ////////////////////////////////////////////////////////////////////////////////////////////////////

-struct PDDLParserPolicy
+struct PDDLTokenizerPolicy
 {
 	static char transformCharacter(char c) noexcept
 	{
@@ -45,11 +43,10 @@ struct PDDLParserPolicy

 ////////////////////////////////////////////////////////////////////////////////////////////////////

-using Parser = parsebase::Parser<PDDLParserPolicy>;
+using Tokenizer = tokenize::Tokenizer<PDDLTokenizerPolicy>;

 ////////////////////////////////////////////////////////////////////////////////////////////////////

-}
 }

 #endif
--- a/lib/pddlparse/src/CMakeLists.txt
+++ b/lib/pddlparse/src/CMakeLists.txt
@@ -8,7 +8,7 @@ file(GLOB detail_headers "../include/pddlparse/detail/*.h")

 set(includes
 	${PROJECT_SOURCE_DIR}/include
-	${PROJECT_SOURCE_DIR}/../../lib/parsebase/include
+	${PROJECT_SOURCE_DIR}/../../lib/tokenize/include
 )

 set(sources
@@ -20,7 +20,7 @@ set(sources
 )

 set(libraries
-	parsebase
+	tokenize
 )

 add_library(${target} ${sources})
--- a/lib/parsebase/CMakeLists.txt
+++ b/lib/parsebase/CMakeLists.txt
@@ -1,7 +1,7 @@
 cmake_minimum_required(VERSION 2.6)
-project(parsebase)
+project(tokenize)

-option(PARSEBASE_BUILD_TESTS "Build unit tests" OFF)
+option(TOKENIZE_BUILD_TESTS "Build unit tests" OFF)

 set(CMAKE_CXX_FLAGS "-Wall -Wextra -Wpedantic -Werror")
 set(CMAKE_CXX_FLAGS_DEBUG "-g")
@@ -23,6 +23,6 @@ if (CMAKE_GENERATOR STREQUAL "Ninja" AND
 endif()

 add_subdirectory(src)
-if(PARSEBASE_BUILD_TESTS)
+if(TOKENIZE_BUILD_TESTS)
 	add_subdirectory(tests)
-endif(PARSEBASE_BUILD_TESTS)
+endif(TOKENIZE_BUILD_TESTS)
--- a/lib/parsebase/include/parsebase/Location.h
+++ b/lib/parsebase/include/parsebase/Location.h
@@ -1,9 +1,9 @@
-#ifndef __PARSE_BASE__LOCATION_H
-#define __PARSE_BASE__LOCATION_H
+#ifndef __TOKENIZE__LOCATION_H
+#define __TOKENIZE__LOCATION_H

 #include <cstdlib>

-namespace parsebase
+namespace tokenize
 {

 ////////////////////////////////////////////////////////////////////////////////////////////////////
--- a/lib/parsebase/include/parsebase/Stream.h
+++ b/lib/parsebase/include/parsebase/Stream.h
@@ -1,5 +1,5 @@
-#ifndef __PARSE_BASE__STREAM_H
-#define __PARSE_BASE__STREAM_H
+#ifndef __TOKENIZE__STREAM_H
+#define __TOKENIZE__STREAM_H

 #include <experimental/filesystem>
 #include <iostream>
@@ -7,9 +7,9 @@
 #include <sstream>
 #include <vector>

-#include <parsebase/Location.h>
+#include <tokenize/Location.h>

-namespace parsebase
+namespace tokenize
 {

 ////////////////////////////////////////////////////////////////////////////////////////////////////
--- a/lib/tokenize/include/tokenize/Tokenizer.h
+++ b/lib/tokenize/include/tokenize/Tokenizer.h
@@ -1,5 +1,5 @@
-#ifndef __PARSE_BASE__PARSER_H
-#define __PARSE_BASE__PARSER_H
+#ifndef __TOKENIZE__TOKENIZER_H
+#define __TOKENIZE__TOKENIZER_H

 #include <algorithm>
 #include <cassert>
@@ -8,11 +8,11 @@
 #include <sstream>
 #include <vector>

-#include <parsebase/ParserException.h>
-#include <parsebase/ParserPolicy.h>
-#include <parsebase/Stream.h>
+#include <tokenize/TokenizerException.h>
+#include <tokenize/TokenizerPolicy.h>
+#include <tokenize/Stream.h>

-namespace parsebase
+namespace tokenize
 {

 template<typename Type>
@@ -22,25 +22,25 @@ struct Tag

 ////////////////////////////////////////////////////////////////////////////////////////////////////
 //
-// Parser
+// Tokenizer
 //
 ////////////////////////////////////////////////////////////////////////////////////////////////////

-template<class ParserPolicy = CaseSensitiveParserPolicy>
-class Parser: public Stream, public ParserPolicy
+template<class TokenizerPolicy = CaseSensitiveTokenizerPolicy>
+class Tokenizer: public Stream, public TokenizerPolicy
 {
-	template<class OtherParserPolicy>
-	friend class Parser;
+	template<class OtherTokenizerPolicy>
+	friend class Tokenizer;

 	public:
-		explicit Parser();
-		explicit Parser(std::string streamName, std::istream &istream);
+		explicit Tokenizer();
+		explicit Tokenizer(std::string streamName, std::istream &istream);

-		template<class OtherParser>
-		Parser(OtherParser &&otherParser)
+		template<class OtherTokenizer>
+		Tokenizer(OtherTokenizer &&otherTokenizer)
 		{
-			m_stream = std::move(otherParser.m_stream);
-			m_delimiters = std::move(otherParser.m_delimiters);
+			m_stream = std::move(otherTokenizer.m_stream);
+			m_delimiters = std::move(otherTokenizer.m_delimiters);
 		}

 		void removeComments(const std::string &startSequence, const std::string &endSequence, bool removeEnd);
@@ -48,7 +48,7 @@ class Parser: public Stream, public ParserPolicy
 		char currentCharacter() const;

 		template<typename Type>
-		Type parse();
+		Type get();

 		template<typename Type>
 		bool testAndReturn(const Type &expectedValue);
@@ -59,27 +59,28 @@ class Parser: public Stream, public ParserPolicy
 		template<typename Type>
 		void expect(const Type &expectedValue);

-		std::string parseIdentifier();
+		// TODO: refactor
+		std::string getIdentifier();
 		bool testIdentifierAndReturn(const std::string &identifier);
 		bool testIdentifierAndSkip(const std::string &identifier);

 		// TODO: remove
 		bool probeNumber();

-		std::string parseLine();
+		std::string getLine();

 		void skipWhiteSpace();
 		void skipBlankSpace();
 		void skipLine();

 	private:
-		std::string parseImpl(Tag<std::string>);
-		char parseImpl(Tag<char>);
-		uint64_t parseImpl(Tag<uint64_t>);
-		int64_t parseImpl(Tag<int64_t>);
-		uint32_t parseImpl(Tag<uint32_t>);
-		int32_t parseImpl(Tag<int32_t>);
-		bool parseImpl(Tag<bool>);
+		std::string getImpl(Tag<std::string>);
+		char getImpl(Tag<char>);
+		uint64_t getImpl(Tag<uint64_t>);
+		int64_t getImpl(Tag<int64_t>);
+		uint32_t getImpl(Tag<uint32_t>);
+		int32_t getImpl(Tag<int32_t>);
+		bool getImpl(Tag<bool>);

 		bool testImpl(const std::string &expectedValue);
 		bool testImpl(char expectedValue);
@@ -89,13 +90,13 @@ class Parser: public Stream, public ParserPolicy
 		bool testImpl(int32_t expectedValue);
 		bool testImpl(bool expectedValue);

-		uint64_t parseIntegerBody();
+		uint64_t getIntegerBody();
 };

 ////////////////////////////////////////////////////////////////////////////////////////////////////

-template<class ParserPolicy>
-Parser<ParserPolicy>::Parser()
+template<class TokenizerPolicy>
+Tokenizer<TokenizerPolicy>::Tokenizer()
 :	Stream()
 {
 }
@@ -103,38 +104,38 @@ Parser<ParserPolicy>::Parser()

 ////////////////////////////////////////////////////////////////////////////////////////////////////

-template<class ParserPolicy>
-Parser<ParserPolicy>::Parser(std::string streamName, std::istream &istream)
+template<class TokenizerPolicy>
+Tokenizer<TokenizerPolicy>::Tokenizer(std::string streamName, std::istream &istream)
 :	Stream(streamName, istream)
 {
 }

 ////////////////////////////////////////////////////////////////////////////////////////////////////

-template<class ParserPolicy>
-void Parser<ParserPolicy>::skipWhiteSpace()
+template<class TokenizerPolicy>
+void Tokenizer<TokenizerPolicy>::skipWhiteSpace()
 {
 	check();

-	while (!atEnd() && ParserPolicy::isWhiteSpaceCharacter(currentCharacter()))
+	while (!atEnd() && TokenizerPolicy::isWhiteSpaceCharacter(currentCharacter()))
 		advance();
 }

 ////////////////////////////////////////////////////////////////////////////////////////////////////

-template<class ParserPolicy>
-void Parser<ParserPolicy>::skipBlankSpace()
+template<class TokenizerPolicy>
+void Tokenizer<TokenizerPolicy>::skipBlankSpace()
 {
 	check();

-	while (!atEnd() && ParserPolicy::isBlankCharacter(currentCharacter()))
+	while (!atEnd() && TokenizerPolicy::isBlankCharacter(currentCharacter()))
 		advance();
 }

 ////////////////////////////////////////////////////////////////////////////////////////////////////

-template<class ParserPolicy>
-void Parser<ParserPolicy>::skipLine()
+template<class TokenizerPolicy>
+void Tokenizer<TokenizerPolicy>::skipLine()
 {
 	check();

@@ -146,18 +147,18 @@ void Parser<ParserPolicy>::skipLine()

 ////////////////////////////////////////////////////////////////////////////////////////////////////

-template<class ParserPolicy>
+template<class TokenizerPolicy>
 template<typename Type>
-Type Parser<ParserPolicy>::parse()
+Type Tokenizer<TokenizerPolicy>::get()
 {
-	return parseImpl(Tag<Type>());
+	return getImpl(Tag<Type>());
 }

 ////////////////////////////////////////////////////////////////////////////////////////////////////

-template<class ParserPolicy>
+template<class TokenizerPolicy>
 template<typename Type>
-bool Parser<ParserPolicy>::testAndReturn(const Type &expectedValue)
+bool Tokenizer<TokenizerPolicy>::testAndReturn(const Type &expectedValue)
 {
 	const auto previousPosition = position();

@@ -170,9 +171,9 @@ bool Parser<ParserPolicy>::testAndReturn(const Type &expectedValue)

 ////////////////////////////////////////////////////////////////////////////////////////////////////

-template<class ParserPolicy>
+template<class TokenizerPolicy>
 template<typename Type>
-bool Parser<ParserPolicy>::testAndSkip(const Type &expectedValue)
+bool Tokenizer<TokenizerPolicy>::testAndSkip(const Type &expectedValue)
 {
 	const auto previousPosition = position();

@@ -186,9 +187,9 @@ bool Parser<ParserPolicy>::testAndSkip(const Type &expectedValue)

 ////////////////////////////////////////////////////////////////////////////////////////////////////

-template<class ParserPolicy>
+template<class TokenizerPolicy>
 template<typename Type>
-void Parser<ParserPolicy>::expect(const Type &expectedValue)
+void Tokenizer<TokenizerPolicy>::expect(const Type &expectedValue)
 {
 	if (testAndSkip(expectedValue))
 		return;
@@ -196,13 +197,13 @@ void Parser<ParserPolicy>::expect(const Type &expectedValue)
 	std::stringstream message;
 	message << "unexpected value, expected “" << expectedValue << "”";

-	throw ParserException(location(), message.str());
+	throw TokenizerException(location(), message.str());
 }

 ////////////////////////////////////////////////////////////////////////////////////////////////////

-template<class ParserPolicy>
-std::string Parser<ParserPolicy>::parseIdentifier()
+template<class TokenizerPolicy>
+std::string Tokenizer<TokenizerPolicy>::getIdentifier()
 {
 	skipWhiteSpace();

@@ -212,7 +213,7 @@ std::string Parser<ParserPolicy>::parseIdentifier()
 	{
 		const auto character = currentCharacter();

-		if (!ParserPolicy::isIdentifierCharacter(character))
+		if (!TokenizerPolicy::isIdentifierCharacter(character))
 			return value;

 		value.push_back(character);
@@ -222,22 +223,22 @@ std::string Parser<ParserPolicy>::parseIdentifier()

 ////////////////////////////////////////////////////////////////////////////////////////////////////

-template<class ParserPolicy>
-bool Parser<ParserPolicy>::testIdentifierAndSkip(const std::string &expectedValue)
+template<class TokenizerPolicy>
+bool Tokenizer<TokenizerPolicy>::testIdentifierAndSkip(const std::string &expectedValue)
 {
-	return testAndSkip(expectedValue) && !ParserPolicy::isIdentifierCharacter(currentCharacter());
+	return testAndSkip(expectedValue) && !TokenizerPolicy::isIdentifierCharacter(currentCharacter());
 }

 ////////////////////////////////////////////////////////////////////////////////////////////////////

-template<class ParserPolicy>
-bool Parser<ParserPolicy>::probeNumber()
+template<class TokenizerPolicy>
+bool Tokenizer<TokenizerPolicy>::probeNumber()
 {
 	const auto previousPosition = position();

 	skipWhiteSpace();

-	while (!ParserPolicy::isWhiteSpaceCharacter(currentCharacter()))
+	while (!TokenizerPolicy::isWhiteSpaceCharacter(currentCharacter()))
 		if (!std::isdigit(currentCharacter()))
 		{
 			seek(previousPosition);
@@ -250,8 +251,8 @@ bool Parser<ParserPolicy>::probeNumber()

 ////////////////////////////////////////////////////////////////////////////////////////////////////

-template<class ParserPolicy>
-std::string Parser<ParserPolicy>::parseLine()
+template<class TokenizerPolicy>
+std::string Tokenizer<TokenizerPolicy>::getLine()
 {
 	std::string value;

@@ -273,8 +274,8 @@ std::string Parser<ParserPolicy>::parseLine()

 ////////////////////////////////////////////////////////////////////////////////////////////////////

-template<class ParserPolicy>
-void Parser<ParserPolicy>::removeComments(const std::string &startSequence, const std::string &endSequence, bool removeEnd)
+template<class TokenizerPolicy>
+void Tokenizer<TokenizerPolicy>::removeComments(const std::string &startSequence, const std::string &endSequence, bool removeEnd)
 {
 	const auto inPosition = m_stream.tellg();
 	const auto outPosition = m_stream.tellp();
@@ -344,22 +345,22 @@ void Parser<ParserPolicy>::removeComments(const std::string &startSequence, cons

 ////////////////////////////////////////////////////////////////////////////////////////////////////

-template<class ParserPolicy>
-char Parser<ParserPolicy>::currentCharacter() const
+template<class TokenizerPolicy>
+char Tokenizer<TokenizerPolicy>::currentCharacter() const
 {
-	return ParserPolicy::transformCharacter(Stream::currentCharacter());
+	return TokenizerPolicy::transformCharacter(Stream::currentCharacter());
 }

 ////////////////////////////////////////////////////////////////////////////////////////////////////

-template<class ParserPolicy>
-std::string Parser<ParserPolicy>::parseImpl(Tag<std::string>)
+template<class TokenizerPolicy>
+std::string Tokenizer<TokenizerPolicy>::getImpl(Tag<std::string>)
 {
 	skipWhiteSpace();

 	const auto startPosition = position();

-	while (!ParserPolicy::isWhiteSpaceCharacter(currentCharacter()))
+	while (!TokenizerPolicy::isWhiteSpaceCharacter(currentCharacter()))
 		advance();

 	const auto endPosition = position();
@@ -381,8 +382,8 @@ std::string Parser<ParserPolicy>::parseImpl(Tag<std::string>)

 ////////////////////////////////////////////////////////////////////////////////////////////////////

-template<class ParserPolicy>
-char Parser<ParserPolicy>::parseImpl(Tag<char>)
+template<class TokenizerPolicy>
+char Tokenizer<TokenizerPolicy>::getImpl(Tag<char>)
 {
 	const auto value = currentCharacter();

@@ -393,13 +394,13 @@ char Parser<ParserPolicy>::parseImpl(Tag<char>)

 ////////////////////////////////////////////////////////////////////////////////////////////////////

-template<class ParserPolicy>
-uint64_t Parser<ParserPolicy>::parseIntegerBody()
+template<class TokenizerPolicy>
+uint64_t Tokenizer<TokenizerPolicy>::getIntegerBody()
 {
 	check();

 	if (!std::isdigit(currentCharacter()))
-		throw ParserException(location(), "could not parse integer value");
+		throw TokenizerException(location(), "could not read integer value");

 	uint64_t value = 0;

@@ -421,51 +422,51 @@ uint64_t Parser<ParserPolicy>::parseIntegerBody()

 ////////////////////////////////////////////////////////////////////////////////////////////////////

-template<class ParserPolicy>
-int64_t Parser<ParserPolicy>::parseImpl(Tag<int64_t>)
+template<class TokenizerPolicy>
+int64_t Tokenizer<TokenizerPolicy>::getImpl(Tag<int64_t>)
 {
 	skipWhiteSpace();

 	bool positive = testAndSkip<char>('+') || !testAndSkip<char>('-');

-	const auto value = parseIntegerBody();
+	const auto value = getIntegerBody();

 	return (positive ? value : -value);
 }

 ////////////////////////////////////////////////////////////////////////////////////////////////////

-template<class ParserPolicy>
-uint64_t Parser<ParserPolicy>::parseImpl(Tag<uint64_t>)
+template<class TokenizerPolicy>
+uint64_t Tokenizer<TokenizerPolicy>::getImpl(Tag<uint64_t>)
 {
 	skipWhiteSpace();

 	if (currentCharacter() == '-')
-		throw ParserException(location(), "expected unsigned integer, got signed one");
+		throw TokenizerException(location(), "expected unsigned integer, got signed one");

-	return parseIntegerBody();
+	return getIntegerBody();
 }

 ////////////////////////////////////////////////////////////////////////////////////////////////////

-template<class ParserPolicy>
-int32_t Parser<ParserPolicy>::parseImpl(Tag<int32_t>)
+template<class TokenizerPolicy>
+int32_t Tokenizer<TokenizerPolicy>::getImpl(Tag<int32_t>)
 {
-	return static_cast<int32_t>(parseImpl(Tag<int64_t>()));
+	return static_cast<int32_t>(getImpl(Tag<int64_t>()));
 }

 ////////////////////////////////////////////////////////////////////////////////////////////////////

-template<class ParserPolicy>
-uint32_t Parser<ParserPolicy>::parseImpl(Tag<uint32_t>)
+template<class TokenizerPolicy>
+uint32_t Tokenizer<TokenizerPolicy>::getImpl(Tag<uint32_t>)
 {
-	return static_cast<uint32_t>(parseImpl(Tag<uint64_t>()));
+	return static_cast<uint32_t>(getImpl(Tag<uint64_t>()));
 }

 ////////////////////////////////////////////////////////////////////////////////////////////////////

-template<class ParserPolicy>
-bool Parser<ParserPolicy>::parseImpl(Tag<bool>)
+template<class TokenizerPolicy>
+bool Tokenizer<TokenizerPolicy>::getImpl(Tag<bool>)
 {
 	skipWhiteSpace();

@@ -475,15 +476,15 @@ bool Parser<ParserPolicy>::parseImpl(Tag<bool>)
 	if (testAndSkip<char>('1'))
 		return true;

-	throw ParserException(location(), "could not parse Boolean value");
+	throw TokenizerException(location(), "could not read Boolean value");
 }

 ////////////////////////////////////////////////////////////////////////////////////////////////////

-template<class ParserPolicy>
-bool Parser<ParserPolicy>::testImpl(const std::string &expectedValue)
+template<class TokenizerPolicy>
+bool Tokenizer<TokenizerPolicy>::testImpl(const std::string &expectedValue)
 {
-	if (!ParserPolicy::isWhiteSpaceCharacter(expectedValue.front()))
+	if (!TokenizerPolicy::isWhiteSpaceCharacter(expectedValue.front()))
 		skipWhiteSpace();

 	const auto match = std::find_if(expectedValue.cbegin(), expectedValue.cend(),
@@ -504,8 +505,8 @@ bool Parser<ParserPolicy>::testImpl(const std::string &expectedValue)

 ////////////////////////////////////////////////////////////////////////////////////////////////////

-template<class ParserPolicy>
-bool Parser<ParserPolicy>::testImpl(char expectedValue)
+template<class TokenizerPolicy>
+bool Tokenizer<TokenizerPolicy>::testImpl(char expectedValue)
 {
 	const auto result = (currentCharacter() == expectedValue);

@@ -516,46 +517,46 @@ bool Parser<ParserPolicy>::testImpl(char expectedValue)

 ////////////////////////////////////////////////////////////////////////////////////////////////////

-template<class ParserPolicy>
-bool Parser<ParserPolicy>::testImpl(int64_t expectedValue)
+template<class TokenizerPolicy>
+bool Tokenizer<TokenizerPolicy>::testImpl(int64_t expectedValue)
 {
-	const auto value = parseImpl(Tag<int64_t>());
+	const auto value = getImpl(Tag<int64_t>());

 	return (value == expectedValue);
 }

 ////////////////////////////////////////////////////////////////////////////////////////////////////

-template<class ParserPolicy>
-bool Parser<ParserPolicy>::testImpl(uint64_t expectedValue)
+template<class TokenizerPolicy>
+bool Tokenizer<TokenizerPolicy>::testImpl(uint64_t expectedValue)
 {
-	const auto value = parseImpl(Tag<uint64_t>());
+	const auto value = getImpl(Tag<uint64_t>());

 	return (value == expectedValue);
 }

 ////////////////////////////////////////////////////////////////////////////////////////////////////

-template<class ParserPolicy>
-bool Parser<ParserPolicy>::testImpl(int32_t expectedValue)
+template<class TokenizerPolicy>
+bool Tokenizer<TokenizerPolicy>::testImpl(int32_t expectedValue)
 {
 	return testImpl(static_cast<int64_t>(expectedValue));
 }

 ////////////////////////////////////////////////////////////////////////////////////////////////////

-template<class ParserPolicy>
-bool Parser<ParserPolicy>::testImpl(uint32_t expectedValue)
+template<class TokenizerPolicy>
+bool Tokenizer<TokenizerPolicy>::testImpl(uint32_t expectedValue)
 {
 	return testImpl(static_cast<uint64_t>(expectedValue));
 }

 ////////////////////////////////////////////////////////////////////////////////////////////////////

-template<class ParserPolicy>
-bool Parser<ParserPolicy>::testImpl(bool expectedValue)
+template<class TokenizerPolicy>
+bool Tokenizer<TokenizerPolicy>::testImpl(bool expectedValue)
 {
-	const auto value = parseImpl(Tag<bool>());
+	const auto value = getImpl(Tag<bool>());

 	return (value == expectedValue);
 }
--- a/lib/tokenize/include/tokenize/TokenizerException.h
+++ b/lib/tokenize/include/tokenize/TokenizerException.h
@@ -1,34 +1,34 @@
-#ifndef __PARSE_BASE__PARSER_EXCEPTION_H
-#define __PARSE_BASE__PARSER_EXCEPTION_H
+#ifndef __TOKENIZE__TOKENIZER_EXCEPTION_H
+#define __TOKENIZE__TOKENIZER_EXCEPTION_H

 #include <exception>
 #include <string>

-#include <parsebase/Location.h>
+#include <tokenize/Location.h>

-namespace parsebase
+namespace tokenize
 {

 ////////////////////////////////////////////////////////////////////////////////////////////////////
 //
-// ParserException
+// TokenizerException
 //
 ////////////////////////////////////////////////////////////////////////////////////////////////////

-class ParserException: public std::exception
+class TokenizerException: public std::exception
 {
 	public:
-		explicit ParserException(const Location &location)
-		:	ParserException(location, "unspecified parser error")
+		explicit TokenizerException(const Location &location)
+		:	TokenizerException(location, "unspecified tokenizer error")
 		{
 		}

-		explicit ParserException(const Location &location, const char *message)
-		:	ParserException(location, static_cast<std::string>(message))
+		explicit TokenizerException(const Location &location, const char *message)
+		:	TokenizerException(location, static_cast<std::string>(message))
 		{
 		}

-		explicit ParserException(const Location &location, const std::string &message)
+		explicit TokenizerException(const Location &location, const std::string &message)
 		:	m_location{location},
 			m_message{message},
 			// TODO: refactor
@@ -37,11 +37,9 @@ class ParserException: public std::exception
 		{
 		}

-		~ParserException() throw()
-		{
-		}
+		~TokenizerException() noexcept = default;

-		const char *what() const throw()
+		const char *what() const noexcept
 		{
 			return m_plainMessage.c_str();
 		}
--- a/lib/tokenize/include/tokenize/TokenizerPolicy.h
+++ b/lib/tokenize/include/tokenize/TokenizerPolicy.h
@@ -1,18 +1,18 @@
-#ifndef __PARSE_BASE__PARSER_POLICY_H
-#define __PARSE_BASE__PARSER_POLICY_H
+#ifndef __TOKENIZE__TOKENIZER_POLICY_H
+#define __TOKENIZE__TOKENIZER_POLICY_H

 #include <iostream>

-namespace parsebase
+namespace tokenize
 {

 ////////////////////////////////////////////////////////////////////////////////////////////////////
 //
-// ParserPolicy
+// TokenizerPolicy
 //
 ////////////////////////////////////////////////////////////////////////////////////////////////////

-struct CaseSensitiveParserPolicy
+struct CaseSensitiveTokenizerPolicy
 {
 	static constexpr char transformCharacter(char c) noexcept
 	{
@@ -37,7 +37,7 @@ struct CaseSensitiveParserPolicy

 ////////////////////////////////////////////////////////////////////////////////////////////////////

-struct CaseInsensitiveParserPolicy
+struct CaseInsensitiveTokenizerPolicy
 {
 	static char transformCharacter(char c) noexcept
 	{
--- a/lib/parsebase/src/CMakeLists.txt
+++ b/lib/parsebase/src/CMakeLists.txt
@@ -1,7 +1,7 @@
-set(target parsebase)
+set(target tokenize)

-file(GLOB core_sources "parsebase/*.cpp")
-file(GLOB core_headers "../include/parsebase/*.h")
+file(GLOB core_sources "tokenize/*.cpp")
+file(GLOB core_headers "../include/tokenize/*.h")

 set(includes
 	${PROJECT_SOURCE_DIR}/include
--- a/lib/parsebase/src/parsebase/Stream.cpp
+++ b/lib/parsebase/src/parsebase/Stream.cpp
@@ -1,11 +1,11 @@
-#include <parsebase/Stream.h>
+#include <tokenize/Stream.h>

 #include <algorithm>
 #include <fstream>

-#include <parsebase/ParserException.h>
+#include <tokenize/TokenizerException.h>

-namespace parsebase
+namespace tokenize
 {

 ////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -89,7 +89,7 @@ Location Stream::location() const
 			return currentPosition >= fileDelimiter.position;
 		});

-	// If the parser is at the end of the stream, still count from the beginning of the last section
+	// If the tokenizer is at the end of the stream, still count from the beginning of the last section
 	if (currentFile == m_delimiters.crend())
 		currentFile = m_delimiters.crbegin();

@@ -143,10 +143,10 @@ bool Stream::atEnd() const
 void Stream::check() const
 {
 	if (atEnd())
-		throw ParserException(location(), "reading past end of file");
+		throw TokenizerException(location(), "reading past end of file");

 	if (m_stream.fail())
-		throw ParserException(location());
+		throw TokenizerException(location());
 }

 ////////////////////////////////////////////////////////////////////////////////////////////////////
--- a/lib/parsebase/tests/CMakeLists.txt
+++ b/lib/parsebase/tests/CMakeLists.txt
@@ -1,22 +1,21 @@
-set(target parsebase-tests)
+set(target tokenize-tests)

 file(GLOB core_sources "*.cpp")

 set(includes
-	${Boost_INCLUDE_DIRS}
 	${PROJECT_SOURCE_DIR}/include
 	${PROJECT_SOURCE_DIR}/../../lib/catch/single_include
 )

 set(libraries
-	parsebase
+	tokenize
 )

 add_executable(${target} ${core_sources})
 target_include_directories(${target} PRIVATE ${includes})
 target_link_libraries(${target} ${libraries})

-add_custom_target(run-parsebase-tests
-	COMMAND ${CMAKE_BINARY_DIR}/bin/parsebase-tests
+add_custom_target(run-tokenize-tests
+	COMMAND ${CMAKE_BINARY_DIR}/bin/tokenize-tests
 	DEPENDS ${target}
 	WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/tests)
--- a/lib/tokenize/tests/TestTokenizer.cpp
+++ b/lib/tokenize/tests/TestTokenizer.cpp
@@ -1,33 +1,33 @@
 #include <catch.hpp>

-#include <parsebase/Parser.h>
-#include <parsebase/ParserException.h>
+#include <tokenize/Tokenizer.h>
+#include <tokenize/TokenizerException.h>

 ////////////////////////////////////////////////////////////////////////////////////////////////////

-TEST_CASE("[parser] Simple strings are parsed correctly", "[parser]")
+TEST_CASE("[tokenizer] Simple strings are tokenized correctly", "[tokenizer]")
 {
 	std::stringstream s("  identifier  5   \n-51\t 0 1 100 200 -300 -400");
-	parsebase::Parser<> p("input", s);
+	tokenize::Tokenizer<> p("input", s);

-	REQUIRE(p.parse<std::string>() == "identifier");
-	REQUIRE(p.parse<size_t>() == 5u);
-	REQUIRE(p.parse<int>() == -51);
-	REQUIRE(p.parse<bool>() == false);
-	REQUIRE(p.parse<bool>() == true);
+	REQUIRE(p.get<std::string>() == "identifier");
+	REQUIRE(p.get<size_t>() == 5u);
+	REQUIRE(p.get<int>() == -51);
+	REQUIRE(p.get<bool>() == false);
+	REQUIRE(p.get<bool>() == true);

-	REQUIRE(p.parse<int>() == 100);
-	REQUIRE(p.parse<size_t>() == 200u);
-	REQUIRE(p.parse<int>() == -300);
-	REQUIRE_THROWS_AS(p.parse<size_t>(), parsebase::ParserException);
+	REQUIRE(p.get<int>() == 100);
+	REQUIRE(p.get<size_t>() == 200u);
+	REQUIRE(p.get<int>() == -300);
+	REQUIRE_THROWS_AS(p.get<size_t>(), tokenize::TokenizerException);
 }

 ////////////////////////////////////////////////////////////////////////////////////////////////////

-TEST_CASE("[parser] Parsing exceptions are correctly reported", "[parser]")
+TEST_CASE("[tokenizer] Tokenizing exceptions are correctly reported", "[tokenizer]")
 {
 	std::stringstream s("  identifier  5   \n-51\t 0 1 100 200 -300 -400");
-	parsebase::Parser<> p("input", s);
+	tokenize::Tokenizer<> p("input", s);

 	REQUIRE_NOTHROW(p.expect<std::string>("identifier"));
 	REQUIRE_NOTHROW(p.expect<size_t>(5u));
@@ -38,41 +38,41 @@ TEST_CASE("[parser] Parsing exceptions are correctly reported", "[parser]")
 	REQUIRE_NOTHROW(p.expect<int>(100));
 	REQUIRE_NOTHROW(p.expect<size_t>(200u));
 	REQUIRE_NOTHROW(p.expect<int>(-300));
-	REQUIRE_THROWS_AS(p.expect<size_t>(-400), parsebase::ParserException);
+	REQUIRE_THROWS_AS(p.expect<size_t>(-400), tokenize::TokenizerException);

 	p.seek(0);
-	REQUIRE_THROWS_AS(p.expect<std::string>("error"), parsebase::ParserException);
+	REQUIRE_THROWS_AS(p.expect<std::string>("error"), tokenize::TokenizerException);

 	p.seek(14);
-	REQUIRE_THROWS_AS(p.expect<size_t>(6u), parsebase::ParserException);
+	REQUIRE_THROWS_AS(p.expect<size_t>(6u), tokenize::TokenizerException);

 	p.seek(17);
-	REQUIRE_THROWS_AS(p.expect<int>(-50), parsebase::ParserException);
+	REQUIRE_THROWS_AS(p.expect<int>(-50), tokenize::TokenizerException);

 	p.seek(24);
-	REQUIRE_THROWS_AS(p.expect<bool>(true), parsebase::ParserException);
+	REQUIRE_THROWS_AS(p.expect<bool>(true), tokenize::TokenizerException);

 	p.seek(26);
-	REQUIRE_THROWS_AS(p.expect<bool>(false), parsebase::ParserException);
+	REQUIRE_THROWS_AS(p.expect<bool>(false), tokenize::TokenizerException);

 	p.seek(28);
-	REQUIRE_THROWS_AS(p.expect<int>(101), parsebase::ParserException);
+	REQUIRE_THROWS_AS(p.expect<int>(101), tokenize::TokenizerException);

 	p.seek(31);
-	REQUIRE_THROWS_AS(p.expect<size_t>(201), parsebase::ParserException);
+	REQUIRE_THROWS_AS(p.expect<size_t>(201), tokenize::TokenizerException);

 	p.seek(34);
-	REQUIRE_THROWS_AS(p.expect<int>(-299), parsebase::ParserException);
+	REQUIRE_THROWS_AS(p.expect<int>(-299), tokenize::TokenizerException);
 }

 ////////////////////////////////////////////////////////////////////////////////////////////////////

-TEST_CASE("[parser] While parsing, the cursor position is as expected", "[parser]")
+TEST_CASE("[tokenizer] While tokenizing, the cursor position is as expected", "[tokenizer]")
 {
 	std::stringstream s("  identifier  5   \n-51\t 0 1");
-	parsebase::Parser<> p("input", s);
+	tokenize::Tokenizer<> p("input", s);

-	parsebase::Parser<>::Position pos;
+	tokenize::Tokenizer<>::Position pos;

 	pos = p.position();
 	REQUIRE(p.testAndReturn<std::string>("error") == false);
@@ -127,61 +127,61 @@ TEST_CASE("[parser] While parsing, the cursor position is as expected", "[parser

 ////////////////////////////////////////////////////////////////////////////////////////////////////

-TEST_CASE("[parser] The end of the input stream is correctly handled", "[parser]")
+TEST_CASE("[tokenizer] The end of the input stream is correctly handled", "[tokenizer]")
 {
 	std::stringstream s1("test");
-	parsebase::Parser<> p1("input", s1);
+	tokenize::Tokenizer<> p1("input", s1);

 	REQUIRE_NOTHROW(p1.expect<std::string>("test"));
-	REQUIRE_THROWS_AS(p1.parse<std::string>(), parsebase::ParserException);
+	REQUIRE_THROWS_AS(p1.get<std::string>(), tokenize::TokenizerException);

 	std::stringstream s2("test1 test2 test3");
-	parsebase::Parser<> p2("input", s2);
+	tokenize::Tokenizer<> p2("input", s2);

 	REQUIRE_NOTHROW(p2.expect<std::string>("test1"));
 	REQUIRE_NOTHROW(p2.expect<std::string>("test2"));
 	REQUIRE_NOTHROW(p2.expect<std::string>("test3"));
-	REQUIRE_THROWS_AS(p2.parse<std::string>(), parsebase::ParserException);
+	REQUIRE_THROWS_AS(p2.get<std::string>(), tokenize::TokenizerException);

 	std::stringstream s3("-127");
-	parsebase::Parser<> p3("input", s3);
+	tokenize::Tokenizer<> p3("input", s3);

 	p3.expect<int>(-127);
-	REQUIRE_THROWS_AS(p3.parse<int>(), parsebase::ParserException);
+	REQUIRE_THROWS_AS(p3.get<int>(), tokenize::TokenizerException);

 	std::stringstream s4("128 -1023 -4095");
-	parsebase::Parser<> p4("input", s4);
+	tokenize::Tokenizer<> p4("input", s4);

 	REQUIRE_NOTHROW(p4.expect<size_t>(128));
 	REQUIRE_NOTHROW(p4.expect<int>(-1023));
 	REQUIRE_NOTHROW(p4.expect<int>(-4095));
-	REQUIRE_THROWS_AS(p4.parse<int>(), parsebase::ParserException);
+	REQUIRE_THROWS_AS(p4.get<int>(), tokenize::TokenizerException);

 	std::stringstream s5("0");
-	parsebase::Parser<> p5("input", s5);
+	tokenize::Tokenizer<> p5("input", s5);

 	p5.expect<bool>(false);
-	REQUIRE_THROWS_AS(p5.parse<bool>(), parsebase::ParserException);
+	REQUIRE_THROWS_AS(p5.get<bool>(), tokenize::TokenizerException);

 	std::stringstream s6("0 1 0");
-	parsebase::Parser<> p6("input", s6);
+	tokenize::Tokenizer<> p6("input", s6);

 	REQUIRE_NOTHROW(p6.expect<bool>(false));
 	REQUIRE_NOTHROW(p6.expect<bool>(true));
 	REQUIRE_NOTHROW(p6.expect<bool>(false));
-	REQUIRE_THROWS_AS(p6.parse<bool>(), parsebase::ParserException);
+	REQUIRE_THROWS_AS(p6.get<bool>(), tokenize::TokenizerException);
 }

 ////////////////////////////////////////////////////////////////////////////////////////////////////

-TEST_CASE("[parser] While parsing, the cursor location is as expcected", "[parser]")
+TEST_CASE("[tokenizer] While tokenizing, the cursor location is as expcected", "[tokenizer]")
 {
 	std::stringstream s("123 \n4\ntest1\n test2\ntest3 \ntest4\n\n\n\n");
-	parsebase::Parser<> p("input", s);
+	tokenize::Tokenizer<> p("input", s);

 	const auto startPosition = p.position();

-	parsebase::Location l;
+	tokenize::Location l;

 	l = p.location();
 	REQUIRE(l.rowStart == 1u);
@@ -277,19 +277,19 @@ TEST_CASE("[parser] While parsing, the cursor location is as expcected", "[parse

 	REQUIRE_NOTHROW(p.expect<std::string>("test1"));

-	// TODO: test parser with multiple sections
+	// TODO: test tokenizer with multiple sections
 }

 ////////////////////////////////////////////////////////////////////////////////////////////////////

-TEST_CASE("[parser] Comments are correctly removed", "[parser]")
+TEST_CASE("[tokenizer] Comments are correctly removed", "[tokenizer]")
 {
 	std::stringstream s1("; comment at beginning\ntest1; comment in between\ntest2; comment at end");
-	parsebase::Parser<> p1("input", s1);
+	tokenize::Tokenizer<> p1("input", s1);

 	p1.removeComments(";", "\n", false);

-	parsebase::Location l;
+	tokenize::Location l;

 	REQUIRE_NOTHROW(p1.expect<std::string>("test1"));

@@ -308,7 +308,7 @@ TEST_CASE("[parser] Comments are correctly removed", "[parser]")
 	REQUIRE(p1.atEnd());

 	std::stringstream s2("test;");
-	parsebase::Parser<> p2("input", s2);
+	tokenize::Tokenizer<> p2("input", s2);

 	p2.removeComments(";", "\n", false);

@@ -319,7 +319,7 @@ TEST_CASE("[parser] Comments are correctly removed", "[parser]")
 	REQUIRE(p2.atEnd());

 	std::stringstream s3("/* comment at start */ test1 /* comment in between */ test2 /*");
-	parsebase::Parser<> p3("input", s3);
+	tokenize::Tokenizer<> p3("input", s3);

 	p3.removeComments("/*", "*/", true);

--- a/lib/parsebase/tests/main.cpp
+++ b/lib/parsebase/tests/main.cpp