No OneTemporary
Actions

Authored By

Unknown

Size

19 KB

Referenced Files

None

Subscribers

None

View Options

	diff --git a/util/token.cpp b/util/token.cpp
	index 807331e6..5c6a2365 100644
	--- a/util/token.cpp
	+++ b/util/token.cpp
	@@ -1,427 +1,428 @@
	#include "token.h"
	#include "token_exception.h"
	#include <string>
	#include <vector>
	#include <ostream>
	#include <sstream>
	#include <iostream>

	using namespace std;

	Token::Token():
	num_token(1),
	parent( NULL ),
	own(true){
	name = "HEAD";
	}

	Token::Token( string tok, bool parse ):
	num_token( 1 ),
	parent( NULL ),
	own(true){

	/* legacy code, not used much */
	if ( !parse ){
	name = tok;
	while ( name.find(' ' ) == 0 )
	name.erase( 0, 1 );
	// lowerCase( name );
	return;
	}

	}

	Token::Token(Token const & copy):
	num_token(1),
	parent(copy.parent),
	own(false){
	this->tokens = copy.tokens;
	this->name = copy.name;
	this->filename = copy.filename;
	}

	/* Dump token to the screen */
	void Token::print( const string space ){
	cout<<space<<"Token: "<< getName() << endl;
	for ( signed int i = 0; i < numTokens(); i++ ){
	Token * x = getToken( i );
	x->print( space + " \|--" );
	}
	}

	void Token::toString(ostream & stream, const string & space){
	if ( numTokens() == -1 ){
	stream << getName();
	} else {
	stream << endl;
	stream << space << "(" << getName();
	for ( signed int i = 0; i < numTokens(); i++ ){
	Token * x = getToken(i);
	stream << " ";
	x->toString( stream, space + " " );
	}
	stream << ")";
	}
	}

	/* helper function */
	string Token::lowerCase( const string & s ){
	string ret;
	for ( unsigned int q = 0; q < s.length(); q++ ){
	if ( s[q] >= 'A' && s[q] <= 'Z' ){
	ret += s[q] - 'A' + 'a';
	} else {
	ret += s[q];
	}
	}
	return ret;
	}

	/* Return next token and increment the internal position
	* of the current token
	*/
	Token * Token::readToken(){
	if ( num_token < tokens.size() ){
	return tokens[ num_token++ ];
	}
	return NULL;
	}

	bool Token::hasTokens(){
	return num_token < tokens.size();
	}

	/*
	vector<Token*> Token::findTokens(const string & path){
	vector<Token*> whoba;
	cout << "whoba has " << whoba.size() << endl;
	whoba.push_back(this);
	if (whoba[0] != this){
	cout << "complete failure!!!" << endl;
	}
	int x = 2;
	x = x + 12;
	return whoba;
	}
	*/

	vector<Token *> Token::findTokens(const string & path){
	vector<Token *> found;
	if (path == ""){
	return found;
	}

	size_t find = path.find('/');
	string self;
	if (find == string::npos){
	self = path;
	} else {
	self = path.substr(0, find);
	}

	/* a name of `_' means succeed with the current token no matter
	* what its called.
	*/
	if (self == "_" \|\| *this == self){
	if (find == string::npos){
	found.push_back(this);
	if (found[0] != this){
	cout << "internal consistency error!!!!" << endl;
	throw exception();
	}
	} else {
	string rest = path.substr(find+1);
	for (int i = 0; i < numTokens(); i++){
	Token * next = getToken(i);
	if (next != NULL){
	vector<Token *> more = next->findTokens(rest);
	found.insert(found.end(), more.begin(), more.end());
	}
	}
	}
	}

	return found;
	}

	TokenMatcher Token::getMatcher(const std::string & subject){
	TokenMatcher matcher(findTokens(subject));
	return matcher;
	}

	Token * Token::findToken(const string & path){
	vector<Token *> all = findTokens(path);
	if (all.size() == 0){
	return NULL;
	}
	return all[0];
	}

	/*
	Token * Token::findToken(const string & path){
	if (path == ""){
	return NULL;
	}

	size_t find = path.find('/');
	string self;
	if (find == string::npos){
	self = path;
	} else {
	self = path.substr(0, find);
	}
	if (*this == self){
	if (find == string::npos){
	return this;
	} else {
	for (int i = 0; i < numTokens(); i++){
	Token * next = getToken(i);
	if (next != NULL){
	Token * ok = next->findToken(path.substr(find+1));
	if (ok != NULL){
	return ok;
	}
	}
	}
	}
	}
	return NULL;
	}
	*/

	Token * Token::getToken( unsigned int n ) const {
	int q = numTokens();
	if ( q == -1 ) return NULL;
	if ( n >= 0 && (signed int)n < q )
	return tokens[n+1];
	return NULL;
	}

	/* If the token has children then the name of this token
	* is the name of the first child token.
	* Otherwise, the name is this token's name
	*/
	const string & Token::getName() const {
	if ( numTokens() != -1 ){
	return tokens[0]->_getName();
	}
	// cout<<"No tokens!!"<<endl;
	return name;
	}

	const Token * Token::getParent() const {
	return this->parent;
	}

	const string Token::getLineage() const {
	if ( getParent() != NULL ){
	return getParent()->getLineage() + " -> " + getName();
	}

	return getName();
	}

	/* A token's identity is its name
	*/
	bool Token::operator== ( const string & rhs ){
	return lowerCase( getName() ) == lowerCase( rhs );
	}

	bool Token::operator!= ( const string & rhs ){
	return !( *this == rhs );
	}

	/* extracting operators */
	Token & Token::operator>>( Token * & rhs ) throw( TokenException ){
	Token * x = readToken();
	if ( x == NULL ){
	throw TokenException(__FILE__, __LINE__, getFileName() + ": " + string("Tried to read a token from ") + this->getName() + string(" but there are no more elements") );
	}
	rhs = x;
	return *this;
	}

	void Token::setFile( const string & s ){
	filename = s;
	}

	const string Token::getFileName() const {
	if ( parent ){
	return parent->getFileName();
	} else {
	return filename;
	}
	}

	Token & Token::operator>>( string & rhs ) throw( TokenException ){
	Token * token = readToken();
	if (token == NULL){
	throw TokenException(__FILE__, __LINE__, getFileName() + ":" + string("Tried to read a string from '") + this->getLineage() + string("' but there no more elements") );
	}
	if (!token->isData()){
	ostringstream out;
	out << getFileName() << ": Element is not a string '";
	token->toString(out, "");
	// throw TokenException(__FILE__, __LINE__, getFileName() + ":" + string(" Element is not a string: "));
	throw TokenException(__FILE__, __LINE__, out.str());
	}

	rhs = token->getName();

	// rhs = getName();

	return *this;
	}

	Token & Token::operator>>( int & rhs ) throw( TokenException ){
	Token * l = readToken();
	if ( l == NULL ){
	throw TokenException(__FILE__, __LINE__, getFileName() + ": " + string("Tried to read an int from ") + this->getLineage() + string(" but there are no more elements") );
	}
	if (!l->isData()){
	throw TokenException(__FILE__, __LINE__, getFileName() + ":" + string(" Element is not a string"));
	}
	istringstream is ( l->getName() );
	is >> rhs;
	return *this;
	}

	Token & Token::operator>>( double & rhs ) throw( TokenException ){
	Token * l = readToken();
	if ( l == NULL ){
	throw TokenException(__FILE__, __LINE__, getFileName() + ": " + string("Tried to read a double from ") + this->getLineage() + string(" but there no more elements") );
	}
	if (!l->isData()){
	throw TokenException(__FILE__, __LINE__, getFileName() + ":" + string(" Element is not a string"));
	}
	istringstream is ( l->getName() );
	is >> rhs;
	return *this;
	}

	Token & Token::operator>>( bool & rhs ) throw( TokenException ){
	Token * l = readToken();
	if ( l == NULL ){
	throw TokenException(__FILE__, __LINE__, getFileName() + ": " + string("Tried to read a bool from ") + this->getLineage() + string(" but there no more elements") );
	}
	if (!l->isData()){
	throw TokenException(__FILE__, __LINE__, getFileName() + ":" + string(" Element is not a string"));
	}
	istringstream is ( l->getName() );
	is >> rhs;
	return *this;
	}

	void Token::addToken(Token * t){
	/*
	if (!own){
	throw TokenException(__FILE__, __LINE__, "This token does not own its own tokens, so you cannot add tokens to it");
	}
	*/
	tokens.push_back( t );
	}

	Token * Token::newToken(){
	Token * token = new Token();
	addToken(token);
	return token;
	}

	/* put quotes around a string if there are spaces in it */
	static string quoteify(const string & rhs){
	if (rhs.find(' ') != string::npos){
	return "\"" + rhs + "\"";
	}
	+
	return rhs;
	}

	Token & Token::operator<<(Token * token){
	if (!own){
	throw TokenException(__FILE__, __LINE__, "Cannot add tokens to a token you don't own");
	}
	this->addToken(token);
	return *this;
	}

	Token & Token::operator<<( const string rhs ){
	if (!own){
	throw TokenException(__FILE__, __LINE__, "Cannot add raw strings to a token you don't own");
	}

	Token * n = new Token(quoteify(rhs), false );
	this->addToken(n);
	return *this;
	}

	Token & Token::operator<<( const int rhs ){
	if (!own){
	throw TokenException(__FILE__, __LINE__, "Cannot add raw integers to a token you don't own");
	}

	ostringstream o;
	o << rhs;
	return *this << o.str();
	}

	Token & Token::operator<<( const double rhs ){
	if (!own){
	throw TokenException(__FILE__, __LINE__, "Cannot add raw doubles to a token you don't own");
	}

	ostringstream o;
	o << rhs;
	return *this << o.str();
	}

	Token * Token::copy(){
	Token * token = new Token();
	token->filename = this->filename;
	token->name = this->name;
	for (vector<Token *>::iterator it = this->tokens.begin(); it != this->tokens.end(); it++){
	Token * him = (*it)->copy();
	him->setParent(token);
	token->addToken(him);
	}
	return token;
	}

	/* Delete tokens that are commented.
	* A commented token has a '!' character as the first
	* character in the name, e.g:
	* (!a_token (child_token 2))
	*/
	void Token::finalize(){
	for ( vector< Token * >::iterator it = tokens.begin(); it != tokens.end(); ){
	Token * t = *it;
	if ( t->getName().find('!') == 0 ){
	delete t;
	it = tokens.erase( it );
	} else {
	t->finalize();
	it++;
	}
	}
	}

	Token::~Token(){
	if (own){
	for ( vector< Token * >::iterator it = tokens.begin(); it != tokens.end(); it++ ){
	delete *it;
	}
	}
	}

	TokenMatcher::TokenMatcher(){
	throw std::exception();
	}

	TokenMatcher & TokenMatcher::operator=(const TokenMatcher & matcher){
	tokens = matcher.tokens;
	current = tokens.begin();
	return *this;
	}

	TokenMatcher::TokenMatcher(std::vector<Token*> tokens):
	tokens(tokens){
	current = this->tokens.begin();
	}
	diff --git a/util/tokenreader.cpp b/util/tokenreader.cpp
	index f01b0a4d..44eb9309 100644
	--- a/util/tokenreader.cpp
	+++ b/util/tokenreader.cpp
	@@ -1,248 +1,255 @@
	#include <fstream>
	#include <iostream>
	#include <string>
	#include <cstring>
	#include <sstream>
	#include <vector>

	#include "token.h"
	#include "token_exception.h"
	#include "tokenreader.h"

	using namespace std;

	/* tokenreader reads a file formatted with s-expressions. examples:
	* (hello)
	* (hello world)
	* (hello "world")
	* (hello (world))
	* (hello (world hi))
	*/

	TokenReader::TokenReader(){
	}

	TokenReader::TokenReader(const char * file){
	readTokenFromFile(file);
	}

	TokenReader::TokenReader(const string & file){
	readTokenFromFile(file.c_str());
	}

	Token * TokenReader::readTokenFromFile(const char * path){
	ifstream file(path);
	if (!file){
	ostringstream out;
	out << "Could not open '" << path << "'";
	throw TokenException(__FILE__, __LINE__, out.str());
	}
	file >> noskipws;
	readTokens(file);
	file.close();
	if (my_tokens.size() > 0){
	return my_tokens[0];
	}
	ostringstream out;
	out << "No tokens read from " << path;
	throw TokenException(__FILE__, __LINE__, out.str());
	}

	Token * TokenReader::readToken(){
	if (my_tokens.size() > 0){
	return my_tokens[0];
	}
	throw TokenException(__FILE__, __LINE__, "No tokens read");
	}

	Token * TokenReader::readToken(const string & path) throw (TokenException){
	return readTokenFromFile(path.c_str());
	}

	Token * TokenReader::readToken(const char * path) throw (TokenException){
	return readTokenFromFile(path);
	}

	Token * TokenReader::readTokenFromString(const string & stuff) throw (TokenException){
	istringstream input(stuff);
	input >> noskipws;
	readTokens(input);
	if (my_tokens.size() > 0){
	return my_tokens[0];
	}
	throw TokenException(__FILE__, __LINE__, "No tokens read");
	}

	TokenReader::~TokenReader(){
	// ifile.close();

	/* tokenreader giveth, and tokenreader taketh */
	for ( vector< Token * >::iterator it = my_tokens.begin(); it != my_tokens.end(); it++ ){
	delete *it;
	}
	}

	void TokenReader::readTokens(istream & input) throw (TokenException){

	/*
	if ( !ifile ){
	ostringstream out;
	out << "Could not open '" << myfile << "'";
	throw TokenException(__FILE__, __LINE__, out.str());
	}
	*/
	// Token * t;

	// string token_string;

	char open_paren = 'x';
	int parens = 1;
	+ int position = 0;
	while (input.good() && open_paren != '('){
	input >> open_paren;
	+ position += 1;
	}
	// token_string += '(';

	Token * cur_token = new Token();
	// cur_token->setFile(myfile);
	my_tokens.push_back( cur_token );
	Token * first = cur_token;
	vector< Token * > token_stack;
	/* tokens that were ignored using ;@, and should be deleted */
	vector<Token*> ignore_list;
	token_stack.push_back( cur_token );

	/* when a ;@ is seen, read the next s-expression but throw it away */
	bool do_ignore = false;

	unsigned char n;
	string cur_string = "";

	/* in_quote is true if a " is read and before another " is read */
	bool in_quote = false;

	/* escaped unconditionally adds the next character to the string */
	bool escaped = false;
	while ( !token_stack.empty() ){
	if ( !input ){
	// cout<<__FILE__<<": "<<myfile<<" is bad. Open parens "<<parens<<endl;
	// cout<<"Dump: "<< token_string << "Last token = [" << n << "]" << (int)n << endl;
	// first->print( " " );
	ostringstream failure;
	failure << "Wrong number of parentheses. Open parens is " << parens;
	throw TokenException(__FILE__, __LINE__, failure.str());
	}
	// char n;
	// slow as we go
	input >> n;
	+ position += 1;
	// printf("Read character '%c' %d\n", n, n);

	const char * alpha = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789./-_!:";
	const char * nonalpha = " ;()#\"";
	// cout<<"Alpha char: "<<n<<endl;

	if (escaped){
	switch (n){
	case 'n' : {
	cur_string += "\n";
	break;
	}
	default : {
	cur_string += n;
	break;
	}
	}
	escaped = false;
	continue;
	}

	if ( n == '\\' ){
	escaped = true;
	continue;
	}

	if ( in_quote ){
	if ( n == '"' ){
	in_quote = false;

	Token * sub = new Token( cur_string, false );
	sub->setParent( cur_token );
	if (do_ignore){
	ignore_list.push_back(sub);
	do_ignore = false;
	} else {
	cur_token->addToken( sub );
	}
	cur_string = "";

	} else {
	cur_string += n;
	}
	} else {
	- if ( n == '"' )
	- in_quote = true;
	+ /* not in a quote */

	- if (strchr(alpha, n) != NULL){
	+ if (n == '"'){
	+ in_quote = true;
	+ } else if (strchr(alpha, n) != NULL){
	cur_string += n;
	} else if (cur_string != "" && strchr(nonalpha, n) != NULL){
	// cout<<"Made new token "<<cur_string<<endl;
	Token * sub = new Token( cur_string, false );
	sub->setParent( cur_token );
	if (do_ignore){
	do_ignore = false;
	ignore_list.push_back(sub);
	} else {
	cur_token->addToken( sub );
	}
	cur_string = "";
	}
	- }

	- if ( n == '#' \|\| n == ';' ){
	- input >> n;
	- if (n == '@'){
	- do_ignore = true;
	- } else {
	- while ( n != '\n' && !input.eof() ){
	- input >> n;
	+ if ( n == '#' \|\| n == ';' ){
	+ input >> n;
	+
	+ /* if the next character is a @ then ignore the next entire s-expression
	+ * otherwise just ignore the current line
	+ */
	+ if (n == '@'){
	+ do_ignore = true;
	+ } else {
	+ while ( n != '\n' && !input.eof() ){
	+ input >> n;
	+ }
	+ continue;
	}
	- continue;
	- }
	- } else if ( n == '(' ){
	- Token * another = new Token();
	- another->setParent( cur_token );
	+ } else if ( n == '(' ){
	+ Token * another = new Token();
	+ another->setParent( cur_token );

	- if (do_ignore){
	- ignore_list.push_back(another);
	- do_ignore = false;
	- } else {
	- cur_token->addToken(another);
	- }
	+ if (do_ignore){
	+ ignore_list.push_back(another);
	+ do_ignore = false;
	+ } else {
	+ cur_token->addToken(another);
	+ }

	- cur_token = another;
	- token_stack.push_back( cur_token );
	- /*
	- parens++;
	- cout<<"Inc Parens is "<<parens<<endl;
	- */
	- } else if ( n == ')' ){
	-
	- if ( token_stack.empty() ){
	- cout<<"Stack is empty"<<endl;
	- throw TokenException(__FILE__, __LINE__, "Stack is empty");
	- }
	- token_stack.pop_back();
	+ cur_token = another;
	+ token_stack.push_back( cur_token );
	+ /*
	+ parens++;
	+ cout<<"Inc Parens is "<<parens<<endl;
	+ */
	+ } else if ( n == ')' ){
	+ if ( token_stack.empty() ){
	+ cout<<"Stack is empty"<<endl;
	+ throw TokenException(__FILE__, __LINE__, "Stack is empty");
	+ }
	+ token_stack.pop_back();

	- if ( ! token_stack.empty() ){
	- cur_token = token_stack.back();
	+ if ( ! token_stack.empty() ){
	+ cur_token = token_stack.back();
	+ }
	}
	}
	}

	for (vector<Token*>::iterator it = ignore_list.begin(); it != ignore_list.end(); it++){
	delete (*it);
	}

	// first->print("");
	first->finalize();
	// return first;
	}

File Metadata

Mime Type: text/x-diff
Expires: Thu, Jun 11, 12:11 PM (3 w, 4 d ago)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 68970
Default Alt Text: (19 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions