Renamed tokenizing module for clarity.

This commit is contained in:
2017-05-12 14:17:57 +02:00
parent c10187f6ba
commit e312a91632
77 changed files with 854 additions and 861 deletions

View File

@@ -3,7 +3,7 @@
#include <plasp/Language.h>
#include <parsebase/Parser.h>
#include <tokenize/Tokenizer.h>
namespace plasp
{
@@ -14,32 +14,32 @@ namespace plasp
//
////////////////////////////////////////////////////////////////////////////////////////////////////
Language::Type detectLanguage(parsebase::Parser<parsebase::CaseInsensitiveParserPolicy> &parser)
Language::Type detectLanguage(tokenize::Tokenizer<tokenize::CaseInsensitiveTokenizerPolicy> &tokenizer)
{
parser.skipWhiteSpace();
tokenizer.skipWhiteSpace();
// SAS begins with "begin_version"
if (parser.testAndSkip<std::string>("begin"))
if (tokenizer.testAndSkip<std::string>("begin"))
{
parser.seek(0);
tokenizer.seek(0);
return Language::Type::SAS;
}
// Skip potential PDDL comments
while (parser.currentCharacter() == ';')
while (tokenizer.currentCharacter() == ';')
{
parser.skipLine();
parser.skipWhiteSpace();
tokenizer.skipLine();
tokenizer.skipWhiteSpace();
}
// PDDL contains sections starting with "(define"
if (parser.testAndSkip<std::string>("(") && parser.testAndSkip<std::string>("define"))
if (tokenizer.testAndSkip<std::string>("(") && tokenizer.testAndSkip<std::string>("define"))
{
parser.seek(std::ios::beg);
tokenizer.seek(std::ios::beg);
return Language::Type::PDDL;
}
parser.seek(std::ios::beg);
tokenizer.seek(std::ios::beg);
return Language::Type::Unknown;
}

View File

@@ -6,7 +6,7 @@
#include <plasp/output/ColorStream.h>
#include <plasp/output/Priority.h>
#include <parsebase/Location.h>
#include <tokenize/Location.h>
namespace plasp
{
@@ -37,8 +37,8 @@ class Logger
void log(Priority priority, const char *message);
void log(Priority priority, const std::string &message);
void log(Priority priority, const parsebase::Location &location, const char *message);
void log(Priority priority, const parsebase::Location &location, const std::string &message);
void log(Priority priority, const tokenize::Location &location, const char *message);
void log(Priority priority, const tokenize::Location &location, const std::string &message);
private:
ColorStream m_outputStream;

View File

@@ -6,7 +6,7 @@
#include <plasp/pddl/Expression.h>
#include <plasp/pddl/expressions/Variable.h>
#include <parsebase/Parser.h>
#include <tokenize/Tokenizer.h>
namespace plasp
{

View File

@@ -6,7 +6,7 @@
#include <vector>
#include <plasp/output/Logger.h>
#include <plasp/pddl/Parser.h>
#include <plasp/pddl/Tokenizer.h>
namespace plasp
{
@@ -25,8 +25,8 @@ class Context
Context() = default;
~Context() = default;
explicit Context(Parser &&otherParser, output::Logger &otherLogger)
: parser{std::move(otherParser)},
explicit Context(Tokenizer &&otherTokenizer, output::Logger &otherLogger)
: tokenizer{std::move(otherTokenizer)},
logger(otherLogger)
{
}
@@ -35,7 +35,7 @@ class Context
Context &operator=(const Context &other) = delete;
Context(Context &&other)
: parser(std::move(other.parser)),
: tokenizer(std::move(other.tokenizer)),
logger(other.logger)
{
}
@@ -47,7 +47,7 @@ class Context
return "__plasp_";
}
Parser parser;
Tokenizer tokenizer;
output::Logger &logger;
};

View File

@@ -4,7 +4,7 @@
#include <plasp/pddl/Domain.h>
#include <plasp/pddl/Problem.h>
#include <parsebase/Parser.h>
#include <tokenize/Tokenizer.h>
namespace plasp
{
@@ -46,9 +46,9 @@ class Description
Context &m_context;
parsebase::Stream::Position m_domainPosition;
tokenize::Stream::Position m_domainPosition;
std::unique_ptr<Domain> m_domain;
parsebase::Stream::Position m_problemPosition;
tokenize::Stream::Position m_problemPosition;
std::unique_ptr<Problem> m_problem;
};

View File

@@ -4,8 +4,8 @@
#include <plasp/pddl/Action.h>
#include <plasp/pddl/Context.h>
#include <plasp/pddl/Expression.h>
#include <plasp/pddl/Parser.h>
#include <plasp/pddl/Requirement.h>
#include <plasp/pddl/Tokenizer.h>
#include <plasp/pddl/expressions/Constant.h>
#include <plasp/pddl/expressions/DerivedPredicate.h>
#include <plasp/pddl/expressions/PredicateDeclaration.h>
@@ -75,19 +75,19 @@ class Domain
std::string m_name;
parsebase::Stream::Position m_requirementsPosition;
tokenize::Stream::Position m_requirementsPosition;
Requirements m_requirements;
parsebase::Stream::Position m_typesPosition;
tokenize::Stream::Position m_typesPosition;
expressions::PrimitiveTypes m_types;
parsebase::Stream::Position m_constantsPosition;
tokenize::Stream::Position m_constantsPosition;
expressions::Constants m_constants;
parsebase::Stream::Position m_predicatesPosition;
tokenize::Stream::Position m_predicatesPosition;
expressions::PredicateDeclarations m_predicates;
std::vector<parsebase::Stream::Position> m_actionPositions;
std::vector<tokenize::Stream::Position> m_actionPositions;
std::vector<std::unique_ptr<Action>> m_actions;
expressions::DerivedPredicates m_derivedPredicates;

View File

@@ -6,7 +6,7 @@
#include <boost/intrusive_ptr.hpp>
#include <parsebase/Parser.h>
#include <tokenize/Tokenizer.h>
namespace plasp
{

View File

@@ -3,7 +3,7 @@
#include <iostream>
#include <plasp/pddl/Parser.h>
#include <plasp/pddl/Tokenizer.h>
namespace plasp
{
@@ -16,14 +16,14 @@ namespace pddl
//
////////////////////////////////////////////////////////////////////////////////////////////////////
inline void skipSection(Parser &parser)
inline void skipSection(Tokenizer &tokenizer)
{
size_t openParentheses = 1;
while (true)
{
const auto character = parser.currentCharacter();
parser.advance();
const auto character = tokenizer.currentCharacter();
tokenizer.advance();
if (character == '(')
openParentheses++;

View File

@@ -4,8 +4,8 @@
#include <plasp/pddl/Context.h>
#include <plasp/pddl/Expression.h>
#include <plasp/pddl/InitialState.h>
#include <plasp/pddl/Parser.h>
#include <plasp/pddl/Requirement.h>
#include <plasp/pddl/Tokenizer.h>
namespace plasp
{
@@ -64,18 +64,18 @@ class Problem
std::string m_name;
parsebase::Stream::Position m_domainPosition;
tokenize::Stream::Position m_domainPosition;
parsebase::Stream::Position m_requirementsPosition;
tokenize::Stream::Position m_requirementsPosition;
Requirements m_requirements;
parsebase::Stream::Position m_objectsPosition;
tokenize::Stream::Position m_objectsPosition;
expressions::Constants m_objects;
parsebase::Stream::Position m_initialStatePosition;
tokenize::Stream::Position m_initialStatePosition;
std::unique_ptr<InitialState> m_initialState;
parsebase::Stream::Position m_goalPosition;
tokenize::Stream::Position m_goalPosition;
ExpressionPointer m_goal;
};

View File

@@ -1,7 +1,7 @@
#ifndef __PLASP__PDDL__PARSER_H
#define __PLASP__PDDL__PARSER_H
#ifndef __PLASP__PDDL__TOKENIZER_H
#define __PLASP__PDDL__TOKENIZER_H
#include <parsebase/Parser.h>
#include <tokenize/Tokenizer.h>
namespace plasp
{
@@ -10,11 +10,11 @@ namespace pddl
////////////////////////////////////////////////////////////////////////////////////////////////////
//
// Parser
// Tokenizer
//
////////////////////////////////////////////////////////////////////////////////////////////////////
class PDDLParserPolicy
class PDDLTokenizerPolicy
{
public:
static char transformCharacter(char c) noexcept
@@ -44,7 +44,7 @@ class PDDLParserPolicy
////////////////////////////////////////////////////////////////////////////////////////////////////
using Parser = parsebase::Parser<PDDLParserPolicy>;
using Tokenizer = tokenize::Tokenizer<PDDLTokenizerPolicy>;
////////////////////////////////////////////////////////////////////////////////////////////////////

View File

@@ -59,33 +59,33 @@ template<typename ExpressionParser>
AtPointer At::parse(Context &context, ExpressionContext &expressionContext,
ExpressionParser parseExpression)
{
auto &parser = context.parser;
auto &tokenizer = context.tokenizer;
const auto position = parser.position();
const auto position = tokenizer.position();
if (!parser.testAndSkip<std::string>("(")
|| !parser.testIdentifierAndSkip("at"))
if (!tokenizer.testAndSkip<std::string>("(")
|| !tokenizer.testIdentifierAndSkip("at"))
{
parser.seek(position);
tokenizer.seek(position);
return nullptr;
}
size_t timePoint;
const auto timePointPosition = parser.position();
const auto timePointPosition = tokenizer.position();
if (parser.testIdentifierAndSkip("start"))
if (tokenizer.testIdentifierAndSkip("start"))
timePoint = TimePointStart;
else if (parser.testIdentifierAndSkip("end"))
else if (tokenizer.testIdentifierAndSkip("end"))
timePoint = TimePointEnd;
else if (parser.probeNumber())
else if (tokenizer.probeNumber())
{
parser.seek(timePointPosition);
timePoint = parser.parse<size_t>();
tokenizer.seek(timePointPosition);
timePoint = tokenizer.get<size_t>();
}
else
{
parser.seek(position);
tokenizer.seek(position);
return nullptr;
}
@@ -93,12 +93,12 @@ AtPointer At::parse(Context &context, ExpressionContext &expressionContext,
expression->m_timePoint = timePoint;
context.parser.skipWhiteSpace();
tokenizer.skipWhiteSpace();
// Parse argument
expression->setArgument(parseExpression(context, expressionContext));
parser.expect<std::string>(")");
tokenizer.expect<std::string>(")");
return expression;
}

View File

@@ -54,14 +54,14 @@ template<typename ExpressionParser>
boost::intrusive_ptr<Derived> Binary<Derived>::parse(Context &context,
ExpressionContext &expressionContext, ExpressionParser parseExpression)
{
auto &parser = context.parser;
auto &tokenizer = context.tokenizer;
const auto position = parser.position();
const auto position = tokenizer.position();
if (!parser.testAndSkip<std::string>("(")
|| !parser.testIdentifierAndSkip(Derived::Identifier))
if (!tokenizer.testAndSkip<std::string>("(")
|| !tokenizer.testIdentifierAndSkip(Derived::Identifier))
{
parser.seek(position);
tokenizer.seek(position);
return nullptr;
}
@@ -71,7 +71,7 @@ boost::intrusive_ptr<Derived> Binary<Derived>::parse(Context &context,
expression->Binary<Derived>::setArgument(0, parseExpression(context, expressionContext));
expression->Binary<Derived>::setArgument(1, parseExpression(context, expressionContext));
parser.expect<std::string>(")");
tokenizer.expect<std::string>(")");
return expression;
}

View File

@@ -55,33 +55,33 @@ template<typename ExpressionParser>
boost::intrusive_ptr<Derived> NAry<Derived>::parse(Context &context,
ExpressionContext &expressionContext, ExpressionParser parseExpression)
{
auto &parser = context.parser;
auto &tokenizer = context.tokenizer;
const auto position = parser.position();
const auto position = tokenizer.position();
if (!parser.testAndSkip<std::string>("(")
|| !parser.testIdentifierAndSkip(Derived::Identifier))
if (!tokenizer.testAndSkip<std::string>("(")
|| !tokenizer.testIdentifierAndSkip(Derived::Identifier))
{
parser.seek(position);
tokenizer.seek(position);
return nullptr;
}
auto expression = boost::intrusive_ptr<Derived>(new Derived);
parser.skipWhiteSpace();
tokenizer.skipWhiteSpace();
// Parse arguments of the expression
while (parser.currentCharacter() != ')')
while (tokenizer.currentCharacter() != ')')
{
expression->addArgument(parseExpression(context, expressionContext));
parser.skipWhiteSpace();
tokenizer.skipWhiteSpace();
}
if (expression->m_arguments.empty())
context.logger.log(output::Priority::Warning, parser.location(), "" + Derived::Identifier + "” expressions should not be empty");
context.logger.log(output::Priority::Warning, tokenizer.location(), "" + Derived::Identifier + "” expressions should not be empty");
parser.expect<std::string>(")");
tokenizer.expect<std::string>(")");
return expression;
}

View File

@@ -53,25 +53,25 @@ template<typename ExpressionParser>
NotPointer Not::parse(Context &context, ExpressionContext &expressionContext,
ExpressionParser parseExpression)
{
auto &parser = context.parser;
auto &tokenizer = context.tokenizer;
const auto position = parser.position();
const auto position = tokenizer.position();
if (!parser.testAndSkip<std::string>("(")
|| !parser.testIdentifierAndSkip("not"))
if (!tokenizer.testAndSkip<std::string>("(")
|| !tokenizer.testIdentifierAndSkip("not"))
{
parser.seek(position);
tokenizer.seek(position);
return nullptr;
}
auto expression = NotPointer(new Not);
context.parser.skipWhiteSpace();
tokenizer.skipWhiteSpace();
// Parse argument
expression->setArgument(parseExpression(context, expressionContext));
parser.expect<std::string>(")");
tokenizer.expect<std::string>(")");
return expression;
}

View File

@@ -69,23 +69,23 @@ template<typename ExpressionParser>
boost::intrusive_ptr<Derived> QuantifiedCRTP<Derived>::parse(Context &context,
ExpressionContext &expressionContext, ExpressionParser parseExpression)
{
auto &parser = context.parser;
auto &tokenizer = context.tokenizer;
const auto position = parser.position();
const auto position = tokenizer.position();
if (!parser.testAndSkip<std::string>("(")
|| !parser.testIdentifierAndSkip(Derived::Identifier))
if (!tokenizer.testAndSkip<std::string>("(")
|| !tokenizer.testIdentifierAndSkip(Derived::Identifier))
{
parser.seek(position);
tokenizer.seek(position);
return nullptr;
}
auto expression = boost::intrusive_ptr<Derived>(new Derived);
// Parse variable list
parser.expect<std::string>("(");
tokenizer.expect<std::string>("(");
Variable::parseTypedDeclarations(context, expressionContext, expression->m_variables);
parser.expect<std::string>(")");
tokenizer.expect<std::string>(")");
// Push newly parsed variables to the stack
expressionContext.variables.push(&expression->m_variables);
@@ -96,7 +96,7 @@ boost::intrusive_ptr<Derived> QuantifiedCRTP<Derived>::parse(Context &context,
// Clean up variable stack
expressionContext.variables.pop();
parser.expect<std::string>(")");
tokenizer.expect<std::string>(")");
return expression;
}

View File

@@ -36,14 +36,14 @@ WhenPointer When::parse(Context &context, ExpressionContext &expressionContext,
ConditionExpressionParser parseConditionExpression,
ImplicationExpressionParser parseImplicationExpression)
{
auto &parser = context.parser;
auto &tokenizer = context.tokenizer;
const auto position = parser.position();
const auto position = tokenizer.position();
if (!parser.testAndSkip<std::string>("(")
|| !parser.testIdentifierAndSkip(Identifier))
if (!tokenizer.testAndSkip<std::string>("(")
|| !tokenizer.testIdentifierAndSkip(Identifier))
{
parser.seek(position);
tokenizer.seek(position);
return nullptr;
}
@@ -53,7 +53,7 @@ WhenPointer When::parse(Context &context, ExpressionContext &expressionContext,
expression->setArgument(0, parseConditionExpression(context, expressionContext));
expression->setArgument(1, parseImplicationExpression(context, expressionContext));
parser.expect<std::string>(")");
tokenizer.expect<std::string>(")");
return expression;
}

View File

@@ -7,7 +7,7 @@
#include <plasp/sas/Value.h>
#include <plasp/sas/Variable.h>
#include <parsebase/Parser.h>
#include <tokenize/Tokenizer.h>
namespace plasp
{
@@ -28,8 +28,8 @@ using AssignedVariables = std::vector<AssignedVariable>;
class AssignedVariable
{
public:
static AssignedVariable fromSAS(parsebase::Parser<> &parser, const Variables &variables);
static AssignedVariable fromSAS(parsebase::Parser<> &parser, const Variable &variable);
static AssignedVariable fromSAS(tokenize::Tokenizer<> &tokenizer, const Variables &variables);
static AssignedVariable fromSAS(tokenize::Tokenizer<> &tokenizer, const Variable &variable);
public:
explicit AssignedVariable(const Variable &variable, const Value &value);

View File

@@ -6,7 +6,7 @@
#include <plasp/sas/AssignedVariable.h>
#include <plasp/sas/Variable.h>
#include <parsebase/Parser.h>
#include <tokenize/Tokenizer.h>
namespace plasp
{
@@ -30,7 +30,7 @@ class AxiomRule
using Condition = AssignedVariable;
using Conditions = AssignedVariables;
static AxiomRule fromSAS(parsebase::Parser<> &parser, const Variables &variables);
static AxiomRule fromSAS(tokenize::Tokenizer<> &tokenizer, const Variables &variables);
public:
const Conditions &conditions() const;

View File

@@ -14,7 +14,7 @@
#include <plasp/sas/Operator.h>
#include <plasp/sas/Variable.h>
#include <parsebase/Parser.h>
#include <tokenize/Tokenizer.h>
namespace plasp
{
@@ -30,7 +30,7 @@ namespace sas
class Description
{
public:
static Description fromParser(parsebase::Parser<> &&parser);
static Description fromTokenizer(tokenize::Tokenizer<> &&tokenizer);
static Description fromStream(std::istream &istream);
static Description fromFile(const std::experimental::filesystem::path &path);
@@ -51,16 +51,16 @@ class Description
private:
Description();
void parseContent(parsebase::Parser<> &parser);
void parseContent(tokenize::Tokenizer<> &tokenizer);
void parseVersionSection(parsebase::Parser<> &parser) const;
void parseMetricSection(parsebase::Parser<> &parser);
void parseVariablesSection(parsebase::Parser<> &parser);
void parseMutexSection(parsebase::Parser<> &parser);
void parseInitialStateSection(parsebase::Parser<> &parser);
void parseGoalSection(parsebase::Parser<> &parser);
void parseOperatorSection(parsebase::Parser<> &parser);
void parseAxiomSection(parsebase::Parser<> &parser);
void parseVersionSection(tokenize::Tokenizer<> &tokenizer) const;
void parseMetricSection(tokenize::Tokenizer<> &tokenizer);
void parseVariablesSection(tokenize::Tokenizer<> &tokenizer);
void parseMutexSection(tokenize::Tokenizer<> &tokenizer);
void parseInitialStateSection(tokenize::Tokenizer<> &tokenizer);
void parseGoalSection(tokenize::Tokenizer<> &tokenizer);
void parseOperatorSection(tokenize::Tokenizer<> &tokenizer);
void parseAxiomSection(tokenize::Tokenizer<> &tokenizer);
bool m_usesActionCosts;

View File

@@ -6,7 +6,7 @@
#include <plasp/sas/AssignedVariable.h>
#include <plasp/sas/Variable.h>
#include <parsebase/Parser.h>
#include <tokenize/Tokenizer.h>
namespace plasp
{
@@ -30,7 +30,7 @@ class Effect
using Condition = AssignedVariable;
using Conditions = AssignedVariables;
static Effect fromSAS(parsebase::Parser<> &parser, const Variables &variables, Conditions &preconditions);
static Effect fromSAS(tokenize::Tokenizer<> &tokenizer, const Variables &variables, Conditions &preconditions);
public:
const Conditions &conditions() const;

View File

@@ -3,7 +3,7 @@
#include <plasp/sas/AssignedVariable.h>
#include <parsebase/Parser.h>
#include <tokenize/Tokenizer.h>
namespace plasp
{
@@ -22,7 +22,7 @@ class Goal
using Fact = AssignedVariable;
using Facts = AssignedVariables;
static Goal fromSAS(parsebase::Parser<> &parser, const Variables &variables);
static Goal fromSAS(tokenize::Tokenizer<> &tokenizer, const Variables &variables);
public:
const Facts &facts() const;

View File

@@ -3,7 +3,7 @@
#include <plasp/sas/AssignedVariable.h>
#include <parsebase/Parser.h>
#include <tokenize/Tokenizer.h>
namespace plasp
{
@@ -22,7 +22,7 @@ class InitialState
using Fact = AssignedVariable;
using Facts = AssignedVariables;
static InitialState fromSAS(parsebase::Parser<> &parser, const Variables &variables);
static InitialState fromSAS(tokenize::Tokenizer<> &tokenizer, const Variables &variables);
public:
const Facts &facts() const;

View File

@@ -5,7 +5,7 @@
#include <plasp/sas/AssignedVariable.h>
#include <parsebase/Parser.h>
#include <tokenize/Tokenizer.h>
namespace plasp
{
@@ -29,7 +29,7 @@ class MutexGroup
using Fact = AssignedVariable;
using Facts = AssignedVariables;
static MutexGroup fromSAS(parsebase::Parser<> &parser, const Variables &variables);
static MutexGroup fromSAS(tokenize::Tokenizer<> &tokenizer, const Variables &variables);
public:
const Facts &facts() const;

View File

@@ -10,7 +10,7 @@
#include <plasp/sas/Predicate.h>
#include <plasp/sas/Variable.h>
#include <parsebase/Parser.h>
#include <tokenize/Tokenizer.h>
namespace plasp
{
@@ -34,7 +34,7 @@ class Operator
using Condition = AssignedVariable;
using Conditions = AssignedVariables;
static Operator fromSAS(parsebase::Parser<> &parser, const Variables &variables);
static Operator fromSAS(tokenize::Tokenizer<> &tokenizer, const Variables &variables);
public:
void printPredicateAsASP(output::ColorStream &stream) const;

View File

@@ -7,7 +7,7 @@
#include <plasp/output/ColorStream.h>
#include <parsebase/Parser.h>
#include <tokenize/Tokenizer.h>
namespace plasp
{
@@ -23,7 +23,7 @@ namespace sas
class Predicate
{
public:
static Predicate fromSAS(parsebase::Parser<> &parser);
static Predicate fromSAS(tokenize::Tokenizer<> &tokenizer);
using Arguments = std::vector<std::string>;

View File

@@ -7,7 +7,7 @@
#include <plasp/output/ColorStream.h>
#include <parsebase/Parser.h>
#include <tokenize/Tokenizer.h>
namespace plasp
{
@@ -40,8 +40,8 @@ struct Value
static const Value Any;
static const Value None;
static Value fromSAS(parsebase::Parser<> &parser);
static const Value &referenceFromSAS(parsebase::Parser<> &parser, const Variable &variable);
static Value fromSAS(tokenize::Tokenizer<> &tokenizer);
static const Value &referenceFromSAS(tokenize::Tokenizer<> &tokenizer, const Variable &variable);
public:
Value negated() const;

View File

@@ -8,7 +8,7 @@
#include <plasp/output/ColorStream.h>
#include <plasp/sas/Value.h>
#include <parsebase/Parser.h>
#include <tokenize/Tokenizer.h>
namespace plasp
{
@@ -29,8 +29,8 @@ using Variables = std::vector<Variable>;
class Variable
{
public:
static Variable fromSAS(parsebase::Parser<> &parser);
static const Variable &referenceFromSAS(parsebase::Parser<> &parser, const Variables &variables);
static Variable fromSAS(tokenize::Tokenizer<> &tokenizer);
static const Variable &referenceFromSAS(tokenize::Tokenizer<> &tokenizer, const Variables &variables);
public:
void printNameAsASPPredicate(output::ColorStream &outputStream) const;

View File

@@ -6,7 +6,7 @@
#include <plasp/sas/Value.h>
#include <plasp/sas/Variable.h>
#include <parsebase/Parser.h>
#include <tokenize/Tokenizer.h>
namespace plasp
{
@@ -27,7 +27,7 @@ using VariableTransitions = std::vector<VariableTransition>;
class VariableTransition
{
public:
static VariableTransition fromSAS(parsebase::Parser<> &parser, const Variables &variables);
static VariableTransition fromSAS(tokenize::Tokenizer<> &tokenizer, const Variables &variables);
public:
const Variable &variable() const;