diff --git a/lib/pddlparse/src/pddlparse/detail/parsing/Action.cpp b/lib/pddlparse/src/pddlparse/detail/parsing/Action.cpp index 351b0fb..1dfe499 100644 --- a/lib/pddlparse/src/pddlparse/detail/parsing/Action.cpp +++ b/lib/pddlparse/src/pddlparse/detail/parsing/Action.cpp @@ -23,10 +23,10 @@ namespace detail ActionParser::ActionParser(Context &context, ast::Domain &domain) : m_context{context}, m_domain{domain}, - m_parametersPosition{-1}, - m_preconditionPosition{-1}, - m_effectPosition{-1}, - m_varsPosition{-1} + m_parametersPosition{tokenize::Stream::InvalidPosition}, + m_preconditionPosition{tokenize::Stream::InvalidPosition}, + m_effectPosition{tokenize::Stream::InvalidPosition}, + m_varsPosition{tokenize::Stream::InvalidPosition} { } @@ -40,26 +40,26 @@ ast::ActionPointer ActionParser::parse() auto &tokenizer = m_context.tokenizer; - if (m_parametersPosition != -1) + if (m_parametersPosition != tokenize::Stream::InvalidPosition) { tokenizer.seek(m_parametersPosition); parseParameterSection(*action); } // For compatibility with old PDDL versions, vars sections are parsed in addition to parameters - if (m_varsPosition != -1) + if (m_varsPosition != tokenize::Stream::InvalidPosition) { tokenizer.seek(m_varsPosition); parseVarsSection(*action); } - if (m_preconditionPosition != -1) + if (m_preconditionPosition != tokenize::Stream::InvalidPosition) { tokenizer.seek(m_preconditionPosition); parsePreconditionSection(*action); } - if (m_effectPosition != -1) + if (m_effectPosition != tokenize::Stream::InvalidPosition) { tokenizer.seek(m_effectPosition); parseEffectSection(*action); @@ -84,7 +84,7 @@ void ActionParser::findSections(ast::Action &action) const auto setSectionPosition = [&](const std::string §ionName, auto §ionPosition, const auto value, bool unique = false) { - if (unique && sectionPosition != -1) + if (unique && sectionPosition != tokenize::Stream::InvalidPosition) { tokenizer.seek(value); throw ParserException(tokenizer.location(), "only one “:" + sectionName + "” section allowed"); diff --git a/lib/pddlparse/src/pddlparse/detail/parsing/ConstantDeclaration.cpp b/lib/pddlparse/src/pddlparse/detail/parsing/ConstantDeclaration.cpp index 1541362..423a622 100644 --- a/lib/pddlparse/src/pddlparse/detail/parsing/ConstantDeclaration.cpp +++ b/lib/pddlparse/src/pddlparse/detail/parsing/ConstantDeclaration.cpp @@ -19,8 +19,8 @@ namespace detail void parseAndAddUntypedConstantDeclaration(Context &context, ast::ConstantDeclarations &constantDeclarations) { auto &tokenizer = context.tokenizer; - auto constantName = tokenizer.getIdentifier(); + auto constantName = tokenizer.getIdentifier(); assert(constantName != "-"); constantDeclarations.emplace_back(std::make_unique(std::move(constantName))); diff --git a/lib/pddlparse/src/pddlparse/detail/parsing/Description.cpp b/lib/pddlparse/src/pddlparse/detail/parsing/Description.cpp index 9c4e4d7..7da8cf1 100644 --- a/lib/pddlparse/src/pddlparse/detail/parsing/Description.cpp +++ b/lib/pddlparse/src/pddlparse/detail/parsing/Description.cpp @@ -19,8 +19,8 @@ namespace detail DescriptionParser::DescriptionParser(Context &context) : m_context{context}, - m_domainPosition{-1}, - m_problemPosition{-1} + m_domainPosition{tokenize::Stream::InvalidPosition}, + m_problemPosition{tokenize::Stream::InvalidPosition} { } @@ -33,7 +33,7 @@ ast::Description DescriptionParser::parse() findSections(); - if (m_domainPosition == -1) + if (m_domainPosition == tokenize::Stream::InvalidPosition) throw ParserException("no PDDL domain specified"); tokenizer.seek(m_domainPosition); @@ -41,7 +41,7 @@ ast::Description DescriptionParser::parse() auto domain = DomainParser(m_context).parse(); // If no problem is given, return just the domain - if (m_problemPosition == -1) + if (m_problemPosition == tokenize::Stream::InvalidPosition) return {std::move(domain), std::experimental::nullopt}; tokenizer.seek(m_problemPosition); @@ -86,7 +86,7 @@ void DescriptionParser::findSections() if (tokenizer.testAndSkip("domain")) { - if (m_domainPosition != -1) + if (m_domainPosition != tokenize::Stream::InvalidPosition) throw ParserException(tokenizer.location(), "PDDL description may not contain two domains"); m_domainPosition = position; @@ -95,7 +95,7 @@ void DescriptionParser::findSections() } else if (m_context.tokenizer.testAndSkip("problem")) { - if (m_problemPosition != -1) + if (m_problemPosition != tokenize::Stream::InvalidPosition) throw ParserException("PDDL description may not contain two problems currently"); m_problemPosition = position; diff --git a/lib/pddlparse/src/pddlparse/detail/parsing/Domain.cpp b/lib/pddlparse/src/pddlparse/detail/parsing/Domain.cpp index 0f83876..e2e5ca3 100644 --- a/lib/pddlparse/src/pddlparse/detail/parsing/Domain.cpp +++ b/lib/pddlparse/src/pddlparse/detail/parsing/Domain.cpp @@ -22,10 +22,10 @@ namespace detail DomainParser::DomainParser(Context &context) : m_context{context}, - m_requirementsPosition{-1}, - m_typesPosition{-1}, - m_constantsPosition{-1}, - m_predicatesPosition{-1} + m_requirementsPosition{tokenize::Stream::InvalidPosition}, + m_typesPosition{tokenize::Stream::InvalidPosition}, + m_constantsPosition{tokenize::Stream::InvalidPosition}, + m_predicatesPosition{tokenize::Stream::InvalidPosition} { } @@ -39,32 +39,32 @@ ast::DomainPointer DomainParser::parse() auto &tokenizer = m_context.tokenizer; - if (m_requirementsPosition != -1) + if (m_requirementsPosition != tokenize::Stream::InvalidPosition) { tokenizer.seek(m_requirementsPosition); parseRequirementSection(*domain); } - if (m_typesPosition != -1) + if (m_typesPosition != tokenize::Stream::InvalidPosition) { tokenizer.seek(m_typesPosition); parseTypeSection(*domain); } - if (m_constantsPosition != -1) + if (m_constantsPosition != tokenize::Stream::InvalidPosition) { tokenizer.seek(m_constantsPosition); parseConstantSection(*domain); } - if (m_predicatesPosition != -1) + if (m_predicatesPosition != tokenize::Stream::InvalidPosition) { tokenizer.seek(m_predicatesPosition); parsePredicateSection(*domain); } for (size_t i = 0; i < m_actionPositions.size(); i++) - if (m_actionPositions[i] != -1) + if (m_actionPositions[i] != tokenize::Stream::InvalidPosition) { tokenizer.seek(m_actionPositions[i]); parseActionSection(*domain); @@ -93,7 +93,7 @@ void DomainParser::findSections(ast::Domain &domain) const auto setSectionPosition = [&](const std::string §ionName, auto §ionPosition, const auto value, bool unique = false) { - if (unique && sectionPosition != -1) + if (unique && sectionPosition != tokenize::Stream::InvalidPosition) { tokenizer.seek(value); throw ParserException(tokenizer.location(), "only one “:" + sectionName + "” section allowed"); @@ -125,7 +125,7 @@ void DomainParser::findSections(ast::Domain &domain) setSectionPosition("predicates", m_predicatesPosition, position, true); else if (tokenizer.testIdentifierAndSkip("action")) { - m_actionPositions.emplace_back(-1); + m_actionPositions.emplace_back(tokenize::Stream::InvalidPosition); setSectionPosition("action", m_actionPositions.back(), position); } else if (tokenizer.testIdentifierAndSkip("functions") diff --git a/lib/pddlparse/src/pddlparse/detail/parsing/PrimitiveType.cpp b/lib/pddlparse/src/pddlparse/detail/parsing/PrimitiveType.cpp index ef42997..872317c 100644 --- a/lib/pddlparse/src/pddlparse/detail/parsing/PrimitiveType.cpp +++ b/lib/pddlparse/src/pddlparse/detail/parsing/PrimitiveType.cpp @@ -19,8 +19,6 @@ ast::PrimitiveTypePointer parsePrimitiveType(Context &context, ast::Domain &doma auto &tokenizer = context.tokenizer; auto &types = domain.types; - tokenizer.skipWhiteSpace(); - auto typeName = tokenizer.getIdentifier(); if (typeName.empty()) diff --git a/lib/pddlparse/src/pddlparse/detail/parsing/Problem.cpp b/lib/pddlparse/src/pddlparse/detail/parsing/Problem.cpp index ba48558..84c7c28 100644 --- a/lib/pddlparse/src/pddlparse/detail/parsing/Problem.cpp +++ b/lib/pddlparse/src/pddlparse/detail/parsing/Problem.cpp @@ -22,11 +22,11 @@ namespace detail ProblemParser::ProblemParser(Context &context, ast::Domain &domain) : m_context{context}, m_domain{domain}, - m_domainPosition{-1}, - m_requirementsPosition{-1}, - m_objectsPosition{-1}, - m_initialStatePosition{-1}, - m_goalPosition{-1} + m_domainPosition{tokenize::Stream::InvalidPosition}, + m_requirementsPosition{tokenize::Stream::InvalidPosition}, + m_objectsPosition{tokenize::Stream::InvalidPosition}, + m_initialStatePosition{tokenize::Stream::InvalidPosition}, + m_goalPosition{tokenize::Stream::InvalidPosition} { } @@ -40,31 +40,31 @@ ast::ProblemPointer ProblemParser::parse() auto &tokenizer = m_context.tokenizer; - if (m_domainPosition == -1) + if (m_domainPosition == tokenize::Stream::InvalidPosition) throw ParserException(tokenizer.location(), "problem description does not specify a corresponding domain"); tokenizer.seek(m_domainPosition); parseDomainSection(*problem); - if (m_requirementsPosition != -1) + if (m_requirementsPosition != tokenize::Stream::InvalidPosition) { tokenizer.seek(m_requirementsPosition); parseRequirementSection(*problem); } - if (m_objectsPosition != -1) + if (m_objectsPosition != tokenize::Stream::InvalidPosition) { tokenizer.seek(m_objectsPosition); parseObjectSection(*problem); } - if (m_initialStatePosition == -1) + if (m_initialStatePosition == tokenize::Stream::InvalidPosition) throw ParserException(tokenizer.location(), "problem description does not specify an initial state"); tokenizer.seek(m_initialStatePosition); parseInitialStateSection(*problem); - if (m_goalPosition == -1) + if (m_goalPosition == tokenize::Stream::InvalidPosition) throw ParserException(tokenizer.location(), "problem description does not specify a goal"); tokenizer.seek(m_goalPosition); @@ -91,7 +91,7 @@ void ProblemParser::findSections(ast::Problem &problem) const auto setSectionPosition = [&](const std::string §ionName, auto §ionPosition, const auto value, bool unique = false) { - if (unique && sectionPosition != -1) + if (unique && sectionPosition != tokenize::Stream::InvalidPosition) { tokenizer.seek(value); throw ParserException(tokenizer.location(), "only one “:" + sectionName + "” section allowed"); diff --git a/lib/pddlparse/src/pddlparse/detail/parsing/VariableDeclaration.cpp b/lib/pddlparse/src/pddlparse/detail/parsing/VariableDeclaration.cpp index 5f75f6e..eca18d8 100644 --- a/lib/pddlparse/src/pddlparse/detail/parsing/VariableDeclaration.cpp +++ b/lib/pddlparse/src/pddlparse/detail/parsing/VariableDeclaration.cpp @@ -19,9 +19,9 @@ namespace detail void parseAndAddUntypedVariableDeclaration(Context &context, ast::VariableDeclarations &variableDeclarations) { auto &tokenizer = context.tokenizer; + tokenizer.expect("?"); auto variableName = tokenizer.getIdentifier(); - assert(variableName != "-"); variableDeclarations.emplace_back(std::make_unique(std::move(variableName))); diff --git a/lib/tokenize/include/tokenize/Stream.h b/lib/tokenize/include/tokenize/Stream.h index 22cbf42..30944f1 100644 --- a/lib/tokenize/include/tokenize/Stream.h +++ b/lib/tokenize/include/tokenize/Stream.h @@ -1,6 +1,7 @@ #ifndef __TOKENIZE__STREAM_H #define __TOKENIZE__STREAM_H +#include #include #include #include @@ -8,6 +9,7 @@ #include #include +#include namespace tokenize { @@ -21,7 +23,8 @@ namespace tokenize class Stream { public: - using Position = std::stringstream::pos_type; + using Position = size_t; + static const Position InvalidPosition; struct Delimiter { @@ -47,14 +50,34 @@ class Stream Position position() const; Location location() const; - char currentCharacter() const; - void advance(); - bool atEnd() const; + char currentCharacter() const + { + assert(m_position < m_stream.size()); - void check() const; + // TODO: check if this should be secured by check() + return m_stream[m_position]; + } + + void advance() + { + check(); + m_position++; + } + + bool atEnd() const + { + return m_position >= m_stream.size(); + } + + void check() const + { + if (atEnd()) + throw TokenizerException(location(), "reading past end of file"); + } protected: - mutable std::stringstream m_stream; + std::string m_stream; + mutable Position m_position; std::vector m_delimiters; }; diff --git a/lib/tokenize/include/tokenize/Tokenizer.h b/lib/tokenize/include/tokenize/Tokenizer.h index 0edea3b..5be10ec 100644 --- a/lib/tokenize/include/tokenize/Tokenizer.h +++ b/lib/tokenize/include/tokenize/Tokenizer.h @@ -45,8 +45,6 @@ class Tokenizer: public Stream, public TokenizerPolicy void removeComments(const std::string &startSequence, const std::string &endSequence, bool removeEnd); - char currentCharacter() const; - template Type get(); @@ -115,8 +113,6 @@ Tokenizer::Tokenizer(std::string streamName, std::istream &istr template void Tokenizer::skipWhiteSpace() { - check(); - while (!atEnd() && TokenizerPolicy::isWhiteSpaceCharacter(currentCharacter())) advance(); } @@ -126,8 +122,6 @@ void Tokenizer::skipWhiteSpace() template void Tokenizer::skipBlankSpace() { - check(); - while (!atEnd() && TokenizerPolicy::isBlankCharacter(currentCharacter())) advance(); } @@ -137,9 +131,7 @@ void Tokenizer::skipBlankSpace() template void Tokenizer::skipLine() { - check(); - - while (currentCharacter() != '\n') + while (!atEnd() && currentCharacter() != '\n') advance(); advance(); @@ -296,78 +288,51 @@ std::string Tokenizer::getLine() template void Tokenizer::removeComments(const std::string &startSequence, const std::string &endSequence, bool removeEnd) { - const auto inPosition = m_stream.tellg(); - const auto outPosition = m_stream.tellp(); - - m_stream.seekg(0); + // TODO: move to appropriate place + for (auto &character : m_stream) + character = TokenizerPolicy::transformCharacter(character); const auto removeRange = [&](const auto &start, const auto &end) { - assert(start != -1); + const auto previousPosition = m_position; - m_stream.clear(); - m_stream.seekp(start); - m_stream.seekg(start); + assert(start < m_stream.size()); - auto position = start; + m_position = start; - while (end == -1 || position < end) + while (m_position < end) { - m_stream.ignore(1); - if (atEnd()) return; - m_stream.put(' '); - position += static_cast(1); + m_stream[m_position] = ' '; + m_position++; } + + m_position = previousPosition; }; + m_position = 0; + while (!atEnd()) { - Position startPosition = m_stream.tellg(); - - while (!atEnd()) - { - startPosition = m_stream.tellg(); - - if (testAndSkip(startSequence)) - break; - + while (!atEnd() && !testAndSkip(startSequence)) advance(); - } - Position endPosition = m_stream.tellg(); - - while (!atEnd()) - { - endPosition = m_stream.tellg(); - - if (testAndSkip(endSequence)) - break; + auto startPosition = m_position - startSequence.size(); + while (!atEnd() && !testAndSkip(endSequence)) advance(); - } - if (removeEnd) - endPosition = m_stream.tellg(); + auto endPosition = (removeEnd) ? m_position : m_position - endSequence.size(); removeRange(startPosition, endPosition); + + m_position = endPosition + 1; } - m_stream.clear(); - - m_stream.seekg(inPosition); - m_stream.seekp(outPosition); -} - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template -char Tokenizer::currentCharacter() const -{ - return TokenizerPolicy::transformCharacter(Stream::currentCharacter()); + m_position = 0; } //////////////////////////////////////////////////////////////////////////////////////////////////// @@ -506,20 +471,20 @@ bool Tokenizer::testImpl(const std::string &expectedValue) if (!TokenizerPolicy::isWhiteSpaceCharacter(expectedValue.front())) skipWhiteSpace(); - const auto match = std::find_if(expectedValue.cbegin(), expectedValue.cend(), - [&](const auto &expectedCharacter) - { - const auto character = static_cast(this->currentCharacter()); - - if (character != expectedCharacter) - return true; - - this->advance(); - + for (size_t i = 0; i < expectedValue.size(); i++) + { + if (atEnd()) return false; - }); - return (match == expectedValue.cend()); + const auto character = currentCharacter(); + + if (character != expectedValue[i]) + return false; + + advance(); + } + + return true; } //////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/lib/tokenize/src/tokenize/Stream.cpp b/lib/tokenize/src/tokenize/Stream.cpp index 06450f1..9850882 100644 --- a/lib/tokenize/src/tokenize/Stream.cpp +++ b/lib/tokenize/src/tokenize/Stream.cpp @@ -3,8 +3,6 @@ #include #include -#include - namespace tokenize { @@ -14,12 +12,14 @@ namespace tokenize // //////////////////////////////////////////////////////////////////////////////////////////////////// +const Stream::Position Stream::InvalidPosition{std::numeric_limits::max()}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + Stream::Stream() +: m_position{0} { std::setlocale(LC_NUMERIC, "C"); - - // Don’t skip whitespace - m_stream.exceptions(std::istream::badbit); } //////////////////////////////////////////////////////////////////////////////////////////////////// @@ -34,11 +34,18 @@ Stream::Stream(std::string streamName, std::istream &istream) void Stream::read(std::string streamName, std::istream &istream) { // Store position of new section - const auto position = m_stream.tellp(); + const auto position = m_stream.size(); m_delimiters.push_back({position, streamName}); - m_stream << istream.rdbuf(); + istream.seekg(0, std::ios::end); + const auto streamSize = istream.tellg(); + istream.seekg(0, std::ios::beg); + + const auto startPosition = m_stream.size(); + + m_stream.resize(m_stream.size() + streamSize); + std::copy(std::istreambuf_iterator(istream), std::istreambuf_iterator(), m_stream.begin() + startPosition); } //////////////////////////////////////////////////////////////////////////////////////////////////// @@ -57,23 +64,21 @@ void Stream::read(const std::experimental::filesystem::path &path) void Stream::reset() { - m_stream.clear(); - seek(0); + m_position = 0; } //////////////////////////////////////////////////////////////////////////////////////////////////// void Stream::seek(Position position) { - m_stream.clear(); - m_stream.seekg(position); + m_position = position; } //////////////////////////////////////////////////////////////////////////////////////////////////// typename Stream::Position Stream::position() const { - return m_stream.tellg(); + return m_position; } //////////////////////////////////////////////////////////////////////////////////////////////////// @@ -94,8 +99,7 @@ Location Stream::location() const currentFile = m_delimiters.crbegin(); // Go back to beginning of section - m_stream.clear(); - m_stream.seekg(currentFile->position); + m_position = currentFile->position; size_t row = 1; size_t column = 1; @@ -103,9 +107,9 @@ Location Stream::location() const // Compute the location character by character while (true) { - if (currentPosition == -1 && atEnd()) + if (currentPosition >= m_stream.size() && atEnd()) break; - else if (currentPosition >= 0 && position() >= currentPosition) + else if (currentPosition < m_stream.size() && position() >= currentPosition) break; const auto character = currentCharacter(); @@ -118,7 +122,7 @@ Location Stream::location() const else if (std::isblank(character) || std::isprint(character)) column++; - m_stream.ignore(1); + m_position++; } return {currentFile->sectionName.c_str(), currentFile->sectionName.c_str(), row, row, column, column}; @@ -126,38 +130,4 @@ Location Stream::location() const //////////////////////////////////////////////////////////////////////////////////////////////////// -char Stream::currentCharacter() const -{ - // TODO: check if this should be secured by check() - return m_stream.peek(); -} - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -bool Stream::atEnd() const -{ - return position() == -1; -} - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -void Stream::check() const -{ - if (atEnd()) - throw TokenizerException(location(), "reading past end of file"); - - if (m_stream.fail()) - throw TokenizerException(location()); -} - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -void Stream::advance() -{ - check(); - m_stream.ignore(1); -} - -//////////////////////////////////////////////////////////////////////////////////////////////////// - }