Page MenuHomePhabricator (Chris)
Files F118263

No OneTemporary
Actions

Authored By
Unknown
Size
45 KB
Referenced Files
None
Subscribers
None

View Options

diff --git a/src/FakeLuaLexer.cpp b/src/FakeLuaLexer.cpp
new file mode 100644
index 0000000..dd4be19
--- /dev/null
+++ b/src/FakeLuaLexer.cpp
@@ -0,0 +1,524 @@
+/*
+ * Copyright (C) 2019 Me and My Shadow
+ *
+ * This file is part of Me and My Shadow.
+ *
+ * Me and My Shadow is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Me and My Shadow is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Me and My Shadow. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "FakeLuaLexer.h"
+#include <string.h>
+#include <stdio.h>
+
+FakeLuaLexer::FakeLuaLexer()
+ : buf(NULL)
+ , posStart(ITreeStorageBuilder::FilePosition{ 1, 1 })
+ , pos(ITreeStorageBuilder::FilePosition{ 1, 1 })
+ , tokenType(EndOfFile)
+ , storedByPOASerializer(false)
+{
+}
+
+bool FakeLuaLexer::getNextToken() {
+ char c;
+
+ // skip whitespaces
+ for (;;) {
+ c = *buf;
+ if (c == ' ' || c == '\r' || c == '\n' || c == '\t') {
+ buf++; advanceByCharacter(c);
+ } else break;
+ }
+
+ if (c == '_' || (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z')) {
+ // it's identifier
+ posStart = pos;
+ for (int i = 1;; i++) {
+ c = buf[i];
+ if (c == '_' || (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9')) {
+ // do nothing
+ } else {
+ tokenType = Identifier;
+ token.assign(buf, buf + i);
+ buf += i; pos.column += i;
+ return true;
+ }
+ }
+ } else if (c >= '0' && c <= '9') {
+ // it's a number
+ return parseNumber();
+ }
+
+ int length = 0;
+
+ switch (c) {
+ case '\0':
+ // EOF
+ tokenType = EndOfFile;
+ token.clear();
+ return false;
+ break;
+ case '.':
+ if (buf[1] >= '0' && buf[1] <= '9') {
+ // it's a number
+ return parseNumber();
+ } else {
+ length = (buf[1] == '.') ? (buf[2] == '.' ? 3 : 2) : 1;
+ }
+ break;
+ case '\'': case '\"':
+ // short string
+ buf++; advanceByCharacter(c);
+ return parseShortString(c);
+ break;
+ case '-':
+ // check if it's the beginning of a comment
+ if (buf[1] == '-') {
+ buf += 2; pos.column += 2;
+ return parseComment();
+ } else {
+ length = 1;
+ }
+ break;
+ case '+': case '*': case '%': case '^': case '#': case '&': case '|':
+ case '(': case ')': case '{': case '}': case ']': case ';': case ',':
+ length = 1;
+ break;
+ case '/': case ':':
+ length = (buf[1] == c) ? 2 : 1;
+ break;
+ case '<': case '>':
+ length = (buf[1] == c || buf[1] == '=') ? 2 : 1;
+ break;
+ case '~': case '=':
+ length = (buf[1] == '=') ? 2 : 1;
+ break;
+ case '[':
+ // check if it's the beginning of a long string
+ length = checkOpeningLongBracket();
+ if (length >= 0) {
+ buf += length + 2; pos.column += length + 2;
+ return parseLongString(length);
+ } else {
+ length = 1;
+ }
+ break;
+ default:
+ // invalid character
+ break;
+ }
+
+ if (length > 0) {
+ tokenType = Operator;
+ token.assign(buf, buf + length);
+ posStart = pos;
+ buf += length; pos.column += length;
+ return true;
+ } else {
+ tokenType = EndOfFile;
+ token.clear();
+ error = "Invalid character: '";
+ error.push_back(c);
+ error.push_back('\'');
+ return false;
+ }
+}
+
+// Parse number which is of form
+// ([0-9]+([.][0-9]*)?|[.][0-9]+)([Ee][+-]?[0-9]+)?
+// or
+// 0[Xx]([0-9A-Fa-f]+([.][0-9A-Fa-f]*)?|[.][0-9A-Fa-f]+)([Pp][+-]?[0-9]+)?
+// returns true if it succeded,
+bool FakeLuaLexer::parseNumber() {
+ char c;
+ int i = 0;
+ bool isHex = false;
+
+ c = buf[0];
+ if (c == '0' && (buf[1] == 'X' || buf[1] == 'x')) {
+ isHex = true;
+ i = 2;
+ }
+
+ int len1 = 0, len2 = 0;
+
+ // read the numbers before '.'
+ for (;;) {
+ c = buf[i];
+ if ((c >= '0' && c <= '9') ||
+ (isHex && ((c >= 'A' && c <= 'F') || (c >= 'a' && c <= 'f')))
+ )
+ {
+ i++;
+ len1++;
+ } else {
+ break;
+ }
+ }
+
+ if (c == '.') {
+ // read the numbers after '.'
+ i++;
+ for (;;) {
+ c = buf[i];
+ if ((c >= '0' && c <= '9') ||
+ (isHex && ((c >= 'A' && c <= 'F') || (c >= 'a' && c <= 'f')))
+ )
+ {
+ i++;
+ len2++;
+ } else {
+ break;
+ }
+ }
+ }
+
+ if (len1 == 0 && len2 == 0) {
+ // invalid
+ tokenType = EndOfFile;
+ token.clear();
+ error = "At least one digits are expected";
+ buf += i; pos.column += i;
+ return false;
+ }
+
+ if (isHex ? (c == 'P' || c == 'p') : (c == 'E' || c == 'e')) {
+ // read the exponents
+ i++;
+ c = buf[i];
+ if (c == '+' || c == '-') {
+ i++;
+ c = buf[i];
+ }
+
+ int len3 = 0;
+ for (;;) {
+ c = buf[i];
+ if (c >= '0' && c <= '9') {
+ i++;
+ len3++;
+ } else {
+ break;
+ }
+ }
+
+ if (len3 == 0) {
+ // invalid
+ tokenType = EndOfFile;
+ token.clear();
+ error = "At least one digits are expected";
+ buf += i; pos.column += i;
+ return false;
+ }
+ }
+
+ // done
+ tokenType = NumberLiteral;
+ token.assign(buf, buf + i);
+ posStart = pos;
+ buf += i; pos.column += i;
+ return true;
+}
+
+#define U8_ENCODE(CH,OPERATION) \
+ if(CH<0x80){ \
+ OPERATION(CH); \
+ }else if(CH<0x800){ \
+ OPERATION(0xC0 | (CH>>6)); \
+ OPERATION(0x80 | (CH & 0x3F)); \
+ }else if(CH<0x10000){ \
+ OPERATION(0xE0 | (CH>>12)); \
+ OPERATION(0x80 | ((CH>>6) & 0x3F)); \
+ OPERATION(0x80 | (CH & 0x3F)); \
+ }else{ \
+ OPERATION(0xF0 | (CH>>18)); \
+ OPERATION(0x80 | ((CH>>12) & 0x3F)); \
+ OPERATION(0x80 | ((CH>>6) & 0x3F)); \
+ OPERATION(0x80 | (CH & 0x3F)); \
+ }
+
+// parse a short string which should ends with delim.
+// before calling this function the buf should point to the first character of this string.
+bool FakeLuaLexer::parseShortString(char delim) {
+ tokenType = StringLiteral;
+ token.clear();
+
+ posStart = pos;
+
+ char c;
+
+ for (;;) {
+ c = *buf;
+ if (c == delim) {
+ // over
+ buf++; advanceByCharacter(c);
+ return true;
+ } else if (c == '\\') {
+ buf++; pos.column++;
+ c = *buf;
+ switch (c) {
+ case 'a': buf++; pos.column++; token.push_back('\a'); break;
+ case 'b': buf++; pos.column++; token.push_back('\b'); break;
+ case 'f': buf++; pos.column++; token.push_back('\f'); break;
+ case 'n': buf++; pos.column++; token.push_back('\n'); break;
+ case 'r': buf++; pos.column++; token.push_back('\r'); break;
+ case 't': buf++; pos.column++; token.push_back('\t'); break;
+ case 'v': buf++; pos.column++; token.push_back('\v'); break;
+ case '\\': buf++; pos.column++; token.push_back('\\'); break;
+ case '\"': buf++; advanceByCharacter(c); token.push_back('\"'); break;
+ case '\'': buf++; pos.column++; token.push_back('\''); break;
+ case '\r': case '\n':
+ buf++; advanceByCharacter(c);
+ if ((buf[0] == '\r' || buf[0] == '\n') && buf[0] != c) {
+ c = buf[0];
+ buf++; advanceByCharacter(c);
+ }
+ token.push_back('\n');
+ break;
+ case 'z':
+ buf++; pos.column++;
+ for (;;) {
+ c = *buf;
+ if (c == '\r' || c == '\n' || c == ' ' || c == '\t') {
+ buf++; advanceByCharacter(c);
+ } else {
+ break;
+ }
+ }
+ break;
+ case 'u':
+ {
+ buf++; pos.column++;
+ if (buf[0] != '{') {
+ tokenType = EndOfFile;
+ token.clear();
+ error = "'{' expected";
+ return false;
+ }
+ buf++; pos.column++;
+ int ch = 0;
+ for (;;) {
+ int tmp = checkDigit(true);
+ if (tmp < 0) {
+ if (buf[0] == '}') {
+ buf++; pos.column++;
+ break;
+ }
+ tokenType = EndOfFile;
+ token.clear();
+ error = "Hexadecimal digit or '}' expected";
+ return false;
+ }
+ tmp |= (ch << 4);
+ if (tmp >= 0x110000) {
+ tokenType = EndOfFile;
+ token.clear();
+ error = "Out of Unicode range";
+ return false;
+ }
+ buf++; pos.column++;
+ ch = tmp;
+ }
+ U8_ENCODE(ch, token.push_back);
+ break;
+ }
+ case 'x':
+ {
+ buf++; pos.column++;
+ int ch = 0;
+ for (int i = 0; i < 2; i++) {
+ int tmp = checkDigit(true);
+ if (tmp < 0) {
+ tokenType = EndOfFile;
+ token.clear();
+ error = "Hexadecimal digit expected";
+ return false;
+ }
+ buf++; pos.column++;
+ ch = (ch << 4) | tmp;
+ }
+ token.push_back(ch);
+ break;
+ }
+ default:
+ if (c >= '0' && c <= '9') {
+ int ch = 0;
+ for (int i = 0; i < 3; i++) {
+ int tmp = checkDigit(false);
+ if (tmp < 0) break;
+ tmp += ch * 10;
+ if (tmp > 255) break;
+ buf++; pos.column++;
+ ch = tmp;
+ }
+ token.push_back(ch);
+ } else {
+ // invalid character
+ tokenType = EndOfFile;
+ token.clear();
+ error = "Invalid character: '";
+ error.push_back(c);
+ error.push_back('\'');
+ return false;
+ }
+ break;
+ }
+ } else if (c == '\r' || c == '\n' || c == '\0') {
+ tokenType = EndOfFile;
+ token.clear();
+ error = "Unexpected end of string literal";
+ return false;
+ } else {
+ buf++; advanceByCharacter(c);
+ token.push_back(c);
+ }
+ }
+}
+
+// parse a long string which should ends with closing long bracket of given level.
+// before calling this function the buf should point to the first character of this string.
+bool FakeLuaLexer::parseLongString(int level) {
+ tokenType = StringLiteral;
+ token.clear();
+
+ posStart = pos;
+
+ char c;
+
+ // skip initial newline
+ c = *buf;
+ if (c == '\r' || c == '\n') {
+ buf++; advanceByCharacter(c);
+ if ((buf[0] == '\r' || buf[0] == '\n') && buf[0] != c) {
+ c = buf[0];
+ buf++; advanceByCharacter(c);
+ }
+ }
+
+ for (;;) {
+ c = *buf;
+ if (c == ']' && checkClosingLongBracket() == level) {
+ // over
+ buf += level + 2; pos.column += level + 2;
+ return true;
+ } else if (c == '\r' || c == '\n') {
+ buf++; advanceByCharacter(c);
+ if ((buf[0] == '\r' || buf[0] == '\n') && buf[0] != c) {
+ c = buf[0];
+ buf++; advanceByCharacter(c);
+ }
+ token.push_back('\n');
+ } else if (c == '\0') {
+ tokenType = EndOfFile;
+ token.clear();
+ error = "Unexpected end of string literal";
+ return false;
+ } else {
+ buf++; advanceByCharacter(c);
+ token.push_back(c);
+ }
+ }
+}
+
+// parse a comment.
+// before calling this function the buf should point to the first character of this comment.
+bool FakeLuaLexer::parseComment() {
+ int level = checkOpeningLongBracket();
+ if (level >= 0) {
+ buf += level + 2; pos.column += level + 2;
+ if (parseLongString(level)) {
+ tokenType = Comment;
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ tokenType = Comment;
+ token.clear();
+
+ posStart = pos;
+
+ char c;
+
+ for (;;) {
+ c = *buf;
+ if (c == '\r' || c == '\n' || c == '\0') {
+ // over
+ return true;
+ } else {
+ buf++; advanceByCharacter(c);
+ token.push_back(c);
+ }
+ }
+}
+
+// check if the beginning of buf+offset is an opening long bracket.
+// if yes it returns the level, otherwise it returns -1.
+int FakeLuaLexer::checkOpeningLongBracket(int offset) {
+ char c;
+
+ c = buf[offset];
+ if (c == '[') {
+ for (int i = 1;; i++) {
+ c = buf[offset + i];
+ if (c == '[') return i - 1;
+ else if (c != '=') return -1;
+ }
+ }
+
+ return -1;
+}
+
+// check if the beginning of buf+offset is a closing long bracket.
+// if yes it returns the level, otherwise it returns -1.
+int FakeLuaLexer::checkClosingLongBracket(int offset) {
+ char c;
+
+ c = buf[offset];
+ if (c == ']') {
+ for (int i = 1;; i++) {
+ c = buf[offset + i];
+ if (c == ']') return i - 1;
+ else if (c != '=') return -1;
+ }
+ }
+
+ return -1;
+}
+
+// check if buf+offset is a digit.
+// if yes it returns the number representation of the digit, otherwise it returns -1.
+int FakeLuaLexer::checkDigit(bool isHex, int offset) {
+ char c = buf[offset];
+
+ if (c >= '0' && c <= '9') {
+ return c - '0';
+ } else {
+ if (isHex) {
+ if (c >= 'A' && c <= 'F') {
+ return c - ('A' - 10);
+ } else if (c >= 'a' && c <= 'f') {
+ return c - ('a' - 10);
+ }
+ }
+ return -1;
+ }
+}
+
+void FakeLuaLexer::advanceByCharacter(char c) {
+ // we need to advance by 2 since the \" stored by POASerializer is escaped
+ if (c == '\"') pos.column += storedByPOASerializer ? 2 : 1;
+ else pos.advanceByCharacter((int)(unsigned char)c);
+}
diff --git a/src/FakeLuaLexer.h b/src/FakeLuaLexer.h
new file mode 100644
index 0000000..d69dda9
--- /dev/null
+++ b/src/FakeLuaLexer.h
@@ -0,0 +1,65 @@
+/*
+ * Copyright (C) 2019 Me and My Shadow
+ *
+ * This file is part of Me and My Shadow.
+ *
+ * Me and My Shadow is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Me and My Shadow is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Me and My Shadow. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef FAKELUALEXER_H
+#define FAKELUALEXER_H
+
+#include "ITreeStorage.h"
+#include <string>
+
+// A simple and fake Lua lexer.
+class FakeLuaLexer {
+public:
+ enum TokenType {
+ EndOfFile,
+ Identifier,
+ Operator,
+ StringLiteral,
+ NumberLiteral,
+ Comment,
+ };
+
+ const char* buf; // The input buffer, will be advanced after get next token
+ ITreeStorageBuilder::FilePosition posStart; // The start position of the token
+ ITreeStorageBuilder::FilePosition pos; // The position, will be advanced after get next token (usually the end position of the token)
+ TokenType tokenType; // The token type
+ std::string token; // The (unescaped) string containing the contents of the token
+
+ std::string error; // The error message if it is not empty.
+
+ bool storedByPOASerializer; // True if it's a string stored in a file generated by POASerializer.
+
+ FakeLuaLexer();
+
+ // Get next token, returns true if it succeded, false if it's EOF or an error occured.
+ bool getNextToken();
+
+private:
+ bool parseNumber();
+ bool parseShortString(char delim);
+ bool parseLongString(int level);
+ bool parseComment();
+ int checkOpeningLongBracket(int offset = 0);
+ int checkClosingLongBracket(int offset = 0);
+ int checkDigit(bool isHex, int offset = 0);
+ void advanceByCharacter(char c);
+};
+
+#endif
+
diff --git a/src/LevelPackPOTExporter.cpp b/src/LevelPackPOTExporter.cpp
index c674c15..8099eb0 100644
--- a/src/LevelPackPOTExporter.cpp
+++ b/src/LevelPackPOTExporter.cpp
@@ -1,344 +1,499 @@
/*
* Copyright (C) 2019 Me and My Shadow
*
* This file is part of Me and My Shadow.
*
* Me and My Shadow is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Me and My Shadow is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Me and My Shadow. If not, see <http://www.gnu.org/licenses/>.
*/
#include "LevelPackPOTExporter.h"
#include "Functions.h"
#include "FileManager.h"
#include "POASerializer.h"
#include "TreeStorageNode.h"
+#include "FakeLuaLexer.h"
#include <string.h>
#include <time.h>
#include <stdio.h>
#include <string>
#include <vector>
#include <map>
#include <fstream>
#include <algorithm>
#include <iostream>
struct POTFileEntry {
std::string msgid, msgid_plural;
std::string comments, sources;
};
//Check if a string contains c-format specifier.
//Currently we only support %[+-]?[0-9]*[.]?[0-9]*[diufFeEgGaAxXoscp]
static bool isCFormat(const std::string& s) {
for (int i = 0, m = s.size(); i < m; i++) {
if (s[i] == '%') {
i++;
if (i < m) {
switch (s[i]) {
case '+': case '-':
i++;
break;
}
}
int dotCount = 0;
while (i < m) {
switch (s[i]) {
case '0': case '1': case '2': case '3': case '4':
case '5': case '6': case '7': case '8': case '9':
break;
case '.':
dotCount++;
break;
default:
dotCount = 2;
break;
}
if (dotCount >= 2) break;
i++;
}
if (i < m) {
switch (s[i]) {
case 'd': case 'i': case 'u': case 'f': case 'F':
case 'e': case 'E': case 'g': case 'G': case 'a': case 'A':
case 'x': case 'X': case 'o': case 's': case 'c': case 'p':
return true;
}
}
}
}
return false;
}
static void writeComment(std::ostream& fout, const std::string& comments, const std::string& prefix) {
std::string message;
for (auto c : comments) {
if (c != '\r') message.push_back(c);
}
//Trim the message.
{
size_t lps = message.find_first_not_of('\n'), lpe = message.find_last_not_of("\n \t");
if (lps == std::string::npos || lpe == std::string::npos || lps > lpe) {
message.clear(); // it's completely empty
} else {
message = message.substr(lps, lpe - lps + 1);
}
}
if (!message.empty()) {
message.push_back('\0');
//Split the message into lines.
for (int lps = 0;;) {
// determine the end of line
int lpe = lps;
for (; message[lpe] != '\n' && message[lpe] != '\0'; lpe++);
// output the line
fout << prefix << message.substr(lps, lpe - lps) << std::endl;
// break if the string ends
if (message[lpe] == '\0') break;
// point to the start of next line
lps = lpe + 1;
}
}
}
static void writeHeader(std::ostream& fout) {
time_t rawtime;
struct tm *timeinfo;
char buffer[256];
time(&rawtime);
timeinfo = gmtime(&rawtime);
//FIXME: I can't make %z print timezone under Windows
strftime(buffer, sizeof(buffer), "\"POT-Creation-Date: %Y-%m-%d %H:%M+0000\\n\"", timeinfo);
fout << "# SOME DESCRIPTIVE TITLE." << std::endl;
fout << "# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER" << std::endl;
fout << "# This file is distributed under the same license as the PACKAGE package." << std::endl;
fout << "# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR." << std::endl;
fout << "#" << std::endl;
fout << "#, fuzzy" << std::endl;
fout << "msgid \"\"" << std::endl;
fout << "msgstr \"\"" << std::endl;
fout << "\"Project-Id-Version: PACKAGE VERSION\\n\"" << std::endl;
fout << "\"Report-Msgid-Bugs-To: \\n\"" << std::endl;
fout << buffer << std::endl;
fout << "\"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\\n\"" << std::endl;
fout << "\"Last-Translator: FULL NAME <EMAIL@ADDRESS>\\n\"" << std::endl;
fout << "\"Language-Team: LANGUAGE <LL@li.org>\\n\"" << std::endl;
fout << "\"Language: \\n\"" << std::endl;
fout << "\"MIME-Version: 1.0\\n\"" << std::endl;
fout << "\"Content-Type: text/plain; charset=UTF-8\\n\"" << std::endl;
fout << "\"Content-Transfer-Encoding: 8bit\\n\"" << std::endl;
fout << "\"Plural-Forms: nplurals=INTEGER; plural=EXPRESSION;\\n\"" << std::endl;
fout << std::endl;
}
class POTFileEntries {
public:
std::vector<POTFileEntry> entries;
std::map<std::string, int> lookupTable;
public:
//Add a msgid to the entries.
void addEntry(const std::string& msgid, const std::string& msgid_plural, const std::string& comments, const std::string& sources) {
if (msgid.empty()) return;
int index;
{
std::string key = msgid + "\x01" + msgid_plural;
auto it = lookupTable.find(key);
if (it == lookupTable.end()) {
entries.emplace_back();
lookupTable[key] = index = entries.size() - 1;
entries[index].msgid = msgid;
entries[index].msgid_plural = msgid_plural;
} else {
index = it->second;
}
}
if (!comments.empty()) {
if (comments.back() == '\n') {
entries[index].comments += comments;
} else {
entries[index].comments += comments + "\n";
}
}
if (!sources.empty()) {
if (sources.back() == '\n') {
entries[index].sources += sources;
} else {
entries[index].sources += sources + "\n";
}
}
}
//Write all the entries to file.
void writeEntry(std::ostream& fout) {
for (auto &entry : entries) {
//Write comments.
writeComment(fout, entry.comments, "# ");
//Write sources.
writeComment(fout, entry.sources, "#: ");
//Check if it's c-format.
if (isCFormat(entry.msgid) || isCFormat(entry.msgid_plural)) {
fout << "#, c-format" << std::endl;
}
//Write msgids.
fout << "msgid \"" << escapeCString(entry.msgid) << "\"" << std::endl;
if (entry.msgid_plural.empty()) {
fout << "msgstr \"\"" << std::endl;
} else {
fout << "msgid_plural \"" << escapeCString(entry.msgid_plural) << "\"" << std::endl;
fout << "msgstr[0] \"\"" << std::endl;
fout << "msgstr[1] \"\"" << std::endl;
}
//Add a newline.
fout << std::endl;
}
}
};
-static std::string formatSource(const std::string& fileName, const ITreeStorageBuilder::FilePosition& pos) {
+static std::string formatSource(const std::string& fileName, const ITreeStorageBuilder::FilePosition& pos, bool showColumn = false) {
char s[32];
- sprintf(s, "%d", pos.row);
- return fileName + ":" + s;
+ if (showColumn) {
+ sprintf(s, ":%d:%d", pos.row, pos.column);
+ } else {
+ sprintf(s, ":%d", pos.row);
+ }
+ return fileName + s;
}
class LoadLevelListTreeStorageNode : public TreeStorageNode {
public:
POTFileEntries *pot;
public:
LoadLevelListTreeStorageNode(POTFileEntries *pot) : TreeStorageNode(), pot(pot) {}
virtual bool newAttribute(const std::string& name, const std::vector<std::string>& value, const FilePosition& namePos, const std::vector<FilePosition>& valuePos) override {
//Do our own stuff first.
if (name == "name" && value.size() >= 1) {
pot->addEntry(value[0], "",
"TRANSLATORS: This is the name of the level pack.",
formatSource("levels.lst", valuePos[0]));
} else if (name == "description" && value.size() >= 1) {
pot->addEntry(value[0], "",
"TRANSLATORS: This is the description of the level pack.",
formatSource("levels.lst", valuePos[0]));
} else if (name == "congratulations" && value.size() >= 1) {
pot->addEntry(value[0], "",
"TRANSLATORS: This will be shown when all the levels in the pack are finished.",
formatSource("levels.lst", valuePos[0]));
}
//Do default stuff.
return TreeStorageNode::newAttribute(name, value, namePos, valuePos);
}
};
+class FakeLuaParser {
+public:
+ POTFileEntries *pot;
+ FakeLuaLexer& lexer;
+ const std::string& fileName;
+public:
+ FakeLuaParser(POTFileEntries *pot, FakeLuaLexer& lexer, const std::string& fileName)
+ : pot(pot), lexer(lexer), fileName(fileName)
+ {
+ }
+
+ void parse() {
+ comment.clear();
+
+ bool skipOnce = false;
+
+ for (;;) {
+ if (skipOnce) skipOnce = false;
+ else if (!getNextNonCommentToken()) return;
+
+ // we only parse the following format
+ // ( '_' | '__' | 'gettext' ) ( <string> | '(' <string> ')' )
+ // or
+ // 'ngettext' '(' <string> ',' <string> ','
+
+ if (lexer.tokenType == FakeLuaLexer::Identifier) {
+ if (lexer.token == "_" || lexer.token == "__" || lexer.token == "gettext") {
+ if (!getNextNonCommentToken()) return;
+
+ std::string msgid;
+ ITreeStorageBuilder::FilePosition pos;
+
+ if (lexer.tokenType == FakeLuaLexer::StringLiteral) {
+ msgid = lexer.token;
+ pos = lexer.posStart;
+ } else if (lexer.tokenType == FakeLuaLexer::Operator && lexer.token == "(") {
+ if (!getNextNonCommentToken()) return;
+ if (lexer.tokenType == FakeLuaLexer::StringLiteral) {
+ msgid = lexer.token;
+ pos = lexer.posStart;
+ } else {
+ skipOnce = true;
+ continue;
+ }
+ if (!getNextNonCommentToken()) return;
+ if (lexer.tokenType == FakeLuaLexer::Operator && lexer.token == ")") {
+ // do nothing
+ } else {
+ skipOnce = true;
+ continue;
+ }
+ } else {
+ skipOnce = true;
+ continue;
+ }
+
+ pot->addEntry(msgid, "", comment, formatSource(fileName, pos));
+ comment.clear();
+ } else if (lexer.token == "ngettext") {
+ if (!getNextNonCommentToken()) return;
+ if (lexer.tokenType == FakeLuaLexer::Operator && lexer.token == "(") {
+ // do nothing
+ } else {
+ skipOnce = true;
+ continue;
+ }
+
+ std::string msgid, msgid_plural;
+ ITreeStorageBuilder::FilePosition pos;
+
+ if (!getNextNonCommentToken()) return;
+ if (lexer.tokenType == FakeLuaLexer::StringLiteral) {
+ msgid = lexer.token;
+ pos = lexer.posStart;
+ } else {
+ skipOnce = true;
+ continue;
+ }
+
+ if (!getNextNonCommentToken()) return;
+ if (lexer.tokenType == FakeLuaLexer::Operator && lexer.token == ",") {
+ // do nothing
+ } else {
+ skipOnce = true;
+ continue;
+ }
+
+ if (!getNextNonCommentToken()) return;
+ if (lexer.tokenType == FakeLuaLexer::StringLiteral) {
+ msgid_plural = lexer.token;
+ } else {
+ skipOnce = true;
+ continue;
+ }
+
+ if (!getNextNonCommentToken()) return;
+ if (lexer.tokenType == FakeLuaLexer::Operator && lexer.token == ",") {
+ // do nothing
+ } else {
+ skipOnce = true;
+ continue;
+ }
+
+ pot->addEntry(msgid, msgid_plural, comment, formatSource(fileName, pos));
+ comment.clear();
+ }
+ }
+ }
+ }
+
+ bool getNextNonCommentToken() {
+ bool isTranslatorsComment = false;
+
+ for (;;) {
+ if (!lexer.getNextToken()) {
+ if (!lexer.error.empty()) {
+ //Show error message.
+ std::cerr << formatSource(fileName, lexer.pos, true) << ": ERROR: " << lexer.error << std::endl;
+ }
+ return false;
+ }
+
+ if (lexer.tokenType == FakeLuaLexer::Comment) {
+ if (lexer.token.empty() || lexer.token.back() != '\n') lexer.token.push_back('\n');
+ if (!isTranslatorsComment) {
+ if (lexer.token.find("TRANSLATORS:") != std::string::npos) {
+ isTranslatorsComment = true;
+ comment.clear(); // remove last translators comment
+ }
+ }
+ if (isTranslatorsComment) comment += lexer.token;
+ } else {
+ return true;
+ }
+ }
+ }
+
+private:
+ std::string comment;
+};
+
class LoadLevelMessageTreeStorageNode : public TreeStorageNode {
public:
POTFileEntries *pot;
- std::string fileName;
+ const std::string& fileName;
public:
LoadLevelMessageTreeStorageNode(POTFileEntries *pot, const std::string& fileName) : TreeStorageNode(), pot(pot), fileName(fileName) {}
virtual ITreeStorageBuilder* newNode() override {
return new LoadLevelMessageTreeStorageNode(pot, fileName);
}
virtual bool newAttribute(const std::string& name, const std::vector<std::string>& value, const FilePosition& namePos, const std::vector<FilePosition>& valuePos) override {
//Do our own stuff first.
if (name == "name" && value.size() >= 1) {
pot->addEntry(value[0], "",
"TRANSLATORS: This is the name of a level.",
formatSource(fileName, valuePos[0]));
} else if (name == "message" && value.size() >= 1) {
if (this->name == "tile" && this->value.size() >= 1 && this->value[0] == "NotificationBlock") {
pot->addEntry(unescapeNewline(value[0]), "", "",
formatSource(fileName, valuePos[0]));
} else {
pot->addEntry(value[0], "", "",
formatSource(fileName, valuePos[0]));
}
- }
+ } else if (name == "script" && value.size() >= 1) {
+ //Now we extract strings from script.
+ FakeLuaLexer lexer;
- //TODO: extract strings from script.
+ lexer.buf = value[0].c_str();
+ lexer.pos = valuePos[0];
+ lexer.storedByPOASerializer = true;
+
+ FakeLuaParser parser(pot, lexer, fileName);
+
+ parser.parse();
+ }
//Do default stuff.
return TreeStorageNode::newAttribute(name, value, namePos, valuePos);
}
};
bool LevelPackPOTExporter::exportPOT(const std::string& levelpackPath) {
//Open the level list.
std::string levelListFile = levelpackPath + "levels.lst";
std::ifstream fin(levelListFile.c_str());
if (!fin) {
std::cerr << "ERROR: Can't load level list " << levelListFile << std::endl;
return false;
}
//Create the entries.
POTFileEntries pot;
//Load the level list file.
LoadLevelListTreeStorageNode obj(&pot);
if (!POASerializer().readNode(fin, &obj, true)){
std::cerr << "ERROR: Invalid file format of level list " << levelListFile << std::endl;
return false;
}
//Loop through the level list entries.
for (unsigned int i = 0; i<obj.subNodes.size(); i++){
TreeStorageNode* obj1 = obj.subNodes[i];
if (obj1 == NULL)
continue;
if (!obj1->value.empty() && obj1->name == "levelfile") {
std::string fileName = obj1->value[0];
//The path to the file to open.
std::string levelFile = levelpackPath + fileName;
//Open the level file.
LoadLevelMessageTreeStorageNode obj(&pot, fileName);
if (!POASerializer().loadNodeFromFile(levelFile.c_str(), &obj, true)) {
return false;
}
}
}
//Create the directory.
createDirectory((levelpackPath + "locale").c_str());
//Create the messages.pot
std::string potFile = levelpackPath + "locale/messages.pot";
std::ofstream fout(potFile.c_str());
if (!fout) {
std::cerr << "ERROR: Can't open the file " << potFile << " for save" << std::endl;
return false;
}
//Write entries.
writeHeader(fout);
pot.writeEntry(fout);
//Over.
return true;
}
diff --git a/src/POASerializer.cpp b/src/POASerializer.cpp
index 7b82b9f..ed1d157 100644
--- a/src/POASerializer.cpp
+++ b/src/POASerializer.cpp
@@ -1,551 +1,552 @@
/*
* Copyright (C) 2011-2012 Me and My Shadow
*
* This file is part of Me and My Shadow.
*
* Me and My Shadow is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Me and My Shadow is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Me and My Shadow. If not, see <http://www.gnu.org/licenses/>.
*/
#include "POASerializer.h"
#include <sstream>
using namespace std;
//This method is used for reading a string from an input stream.
//fin: The input stream to read from.
//string: String to place the result in.
static void readString(std::istream& fin, std::string& string, ITreeStorageBuilder::FilePosition& pos) {
//The current character.
int c;
c = fin.get();
ITreeStorageBuilder::FilePosition lastPos = pos; pos.advanceByCharacter(c);
//Check if there's a '"'.
if(c=='\"'){
//There's a '"' so place every character we encounter in the string without parsing.
while((!fin.eof()) & (!fin.fail())){
//First we get the next character to prevent putting the '"' in the string.
c=fin.get();
lastPos = pos; pos.advanceByCharacter(c);
//Check if there's a '"' since that could mean the end of the string.
if(c=='\"'){
//Get the next character and check if that's also an '"'.
c=fin.get();
lastPos = pos; pos.advanceByCharacter(c);
if (c != '\"') {
//We have two '"' after each other meaning an escaped '"'.
//We unget one so there will be one '"' placed in the string.
fin.unget();
pos = lastPos;
return;
}
}
//Every other character can be put in the string.
string.push_back(c);
}
}else{
//There are no quotes around the string so we need to be carefull detecting if the string has ended.
do{
switch(c){
//Check for characters that mean the end of the string.
case EOF:
case ' ':
case '\r':
case '\n':
case '\t':
return;
//Check for characters that are part of the POA file format.
//If so we first unget one character to prevent problems parsing the rest of the file.
case ',':
case '=':
case '(':
case ')':
case '{':
case '}':
case '#':
fin.unget();
pos = lastPos;
return;
default:
//In any other case the character is normal so we put it in the string.
string.push_back(c);
}
//Get the next character.
c=fin.get();
lastPos = pos; pos.advanceByCharacter(c);
}while((!fin.eof()) & (!fin.fail()));
}
}
//This function will read from the input stream until there's something else than whitespaces.
//fin: The input stream to read from.
static void skipWhitespaces(std::istream& fin, ITreeStorageBuilder::FilePosition& pos) {
//The current character.
int c;
ITreeStorageBuilder::FilePosition lastPos;
while((!fin.eof()) & (!fin.fail())){
//Get the character.
c=fin.get();
lastPos = pos; pos.advanceByCharacter(c);
//Check if it's one of the whitespace characters.
switch(c){
case EOF:
case ' ':
case '\r':
case '\n':
case '\t':
break;
default:
//Anything other means that the whitespaces have ended.
//Unget the last character and return.
fin.unget();
pos = lastPos;
return;
}
}
}
//This function will read from the input stream until the end of a line (also end of the comment).
//fin: The input stream to read from.
static void skipComment(std::istream& fin, ITreeStorageBuilder::FilePosition& pos) {
//The current character.
int c;
ITreeStorageBuilder::FilePosition lastPos;
while ((!fin.eof()) & (!fin.fail())){
//Get the character.
c=fin.get();
lastPos = pos; pos.advanceByCharacter(c);
//Check if it's a new line (end of comment).
if(c=='\r'||c=='\n'){
fin.unget();
pos = lastPos;
break;
}
}
}
bool POASerializer::readNode(std::istream& fin,ITreeStorageBuilder* objOut,bool loadSubNodeOnly){
//The current file position.
ITreeStorageBuilder::FilePosition pos = { 1, 1 }, lastPos, tempPos;
//The current character.
int c;
//The current mode of reading.
enum ReadMode {
ReadName = 0,
ReadAttributeValue = 1,
ReadSubnodeValue = 2,
AddFirst = 16,
AddAttribute = 16,
AddSubnode = 17,
} mode = ReadName;
//Before reading make sure that the input stream isn't null.
if(!fin) return false;
//Vector containing the stack of TreeStorageNodes.
vector<ITreeStorageBuilder*> stack;
//A vector for the names and a vector for the values.
vector<string> names,values;
//Positions of the names and values.
vector<ITreeStorageBuilder::FilePosition> namePos, valuePos;
//Check if we only need to load subNodes.
//If so then put the objOut as the first TreeStorageNode.
if(loadSubNodeOnly) stack.push_back(objOut);
//Loop through the files.
while((!fin.eof()) && (!fin.fail())){
//Get a character.
c=fin.get();
lastPos = pos; pos.advanceByCharacter(c);
//Check what it is and what to do with that character.
switch(c){
case EOF:
case ' ':
case '\r':
case '\n':
case '\t':
//We skip whitespaces.
break;
case '#':
//A comment so skip it.
skipComment(fin, pos);
break;
case '}':
//A closing bracket so do one step back in the stack.
//There must be a TreeStorageNode left if not return false.
if(stack.empty()) return false;
//Remove the last entry of the stack.
stack.pop_back();
//Check if the stack is empty, if so than the reading of the node is done.
if(stack.empty()) return true;
objOut=stack.back();
break;
default:
//It isn't a special character but part of a name/value, so unget it.
fin.unget();
pos = lastPos;
{
//Clear the names and values vectors, start reading new names/values.
names.clear();
values.clear();
namePos.clear();
valuePos.clear();
//Set the mode to the read name mode.
mode=ReadName;
//Keep reading characters, until we break out the while loop or there's an error.
while((!fin.eof()) & (!fin.fail())){
//The string containing the name.
string s;
//First skip the whiteSpaces.
skipWhitespaces(fin,pos);
//Now get the string.
tempPos = pos;
+ if (fin.peek() == '\"') tempPos.column++;
readString(fin,s,pos);
//Check the mode.
switch(mode){
case ReadName:
//Mode is 0(read names) so put the string in the names vector.
names.push_back(s);
namePos.push_back(tempPos);
break;
case ReadAttributeValue:
case ReadSubnodeValue:
//Mode is 1 or 2 so put the string in the values vector.
values.push_back(s);
valuePos.push_back(tempPos);
break;
}
//Again skip whitespaces.
skipWhitespaces(fin,pos);
//Now read the next character.
c=fin.get();
lastPos = pos; pos.advanceByCharacter(c);
switch (c) {
case ',':
//A comma means one more name or value.
break;
case '=':
//An '=' can only occur after a name (mode=0).
if(mode==ReadName){
//The next string will be a value so set mode to 1.
mode=ReadAttributeValue;
}else{
//In any other case there's something wrong so return false.
return false;
}
break;
case '(':
//An '(' can only occur after a name (mode=0).
if(mode==ReadName){
//The next string will be a value of a block so set mode to 2.
mode=ReadSubnodeValue;
}else{
//In any other case there's something wrong so return false.
return false;
}
break;
case ')':
//A ')' can only occur after an attribute (mode=2).
if(mode==ReadSubnodeValue){
//The next will be a new subNode so set mode to 17.
mode=AddSubnode;
}else{
//In any other case there's something wrong so return false.
return false;
}
break;
case '{':
//A '{' can only mean a new subNode (mode=17).
fin.unget();
pos = lastPos;
mode=AddSubnode;
break;
default:
//The character is not special so unget it.
fin.unget();
pos = lastPos;
mode=AddAttribute;
break;
}
//We only need to break out if the mode is 16(add attribute) or 17(add subnode)
if(mode>=AddFirst) break;
}
//Check the mode.
switch(mode){
case AddAttribute:
//The mode is 16 so we need to change the names and values into attributes.
//The stack mustn't be empty.
if(stack.empty()) return false;
//Make sure that the result TreeStorageNode isn't null.
if(objOut!=NULL){
//Check if the names vector is empty, if so add an empty name.
if (names.empty()) {
names.push_back("");
namePos.push_back(pos);
}
//Put an empty value for every valueless name.
while (values.size() < names.size()) {
values.push_back("");
valuePos.push_back(pos);
}
//Now loop through the names.
for(unsigned int i=0;i<names.size()-1;i++){
//Temp vector that will contain the values.
vector<string> v;
v.push_back(values[i]);
//Temp vector that will contain the positions of values.
vector<ITreeStorageBuilder::FilePosition> vPos;
vPos.push_back(valuePos[i]);
//And add the attribute.
if (objOut->newAttribute(names[i], v, namePos[i], vPos)) {
//Early exit.
return true;
}
}
if (names.size() > 1) {
values.erase(values.begin(), values.begin() + (names.size() - 1));
valuePos.erase(valuePos.begin(), valuePos.begin() + (names.size() - 1));
}
if (objOut->newAttribute(names.back(), values, namePos.back(), valuePos)) {
//Early exit.
return true;
}
}
break;
case AddSubnode:
//The mode is 17 so we need to add a subNode.
{
//Check if the names vector is empty, if so add an empty name.
if (names.empty()) {
names.push_back("");
namePos.push_back(pos);
} else if (names.size() > 1){
if(stack.empty()) return false;
while (values.size() < names.size()) {
values.push_back("");
valuePos.push_back(pos);
}
for(unsigned int i=0;i<names.size()-1;i++){
vector<string> v;
v.push_back(values[i]);
vector<ITreeStorageBuilder::FilePosition> vPos;
vPos.push_back(valuePos[i]);
if (objOut->newAttribute(names[i], v, namePos[i], vPos)) {
//Early exit.
return true;
}
}
values.erase(values.begin(), values.begin() + (names.size() - 1));
valuePos.erase(valuePos.begin(), valuePos.begin() + (names.size() - 1));
}
//Create a new subNode.
ITreeStorageBuilder* objNew=NULL;
//If the stack is empty the new subNode will be the result TreeStorageNode.
if(stack.empty()) objNew=objOut;
//If not the new subNode will be a subNode of the result TreeStorageNode.
else if (objOut != NULL) {
objNew = objOut->newNode();
if (objNew == NULL) {
//Early exit.
return true;
}
}
//Add it to the stack.
stack.push_back(objNew);
if(objNew!=NULL){
//Add the name and the values.
if (objNew->setName(names.back(), namePos.back()) || objNew->setValue(values, valuePos)) {
//Early exit.
return true;
}
}
objOut=objNew;
//Skip the whitespaces.
skipWhitespaces(fin,pos);
//And get the next character.
c=fin.get();
lastPos = pos; pos.advanceByCharacter(c);
if(c!='{'){
//The character isn't a '{' meaning the block hasn't got a body.
fin.unget();
pos = lastPos;
stack.pop_back();
//Check if perhaps we're done, stack=empty.
if(stack.empty()) return true;
objOut=stack.back();
}
}
break;
default:
//The mode isn't 16 or 17 but still broke out the while loop.
//Something's wrong so return false.
return false;
}
}
break;
}
}
return true;
}
static void writeString(std::ostream& fout,std::string& s){
//This method will write a string.
//fout: The output stream to write to.
//s: The string to write.
//new: check if the string is empty
if(s.empty()){
//because of the new changes of loader, we should output 2 quotes '""'
fout<<"\"\"";
}else
//Check if the string contains any special character that needs escaping.
if(s.find_first_of(" \r\n\t,=(){}#\"")!=string::npos){
//It does so we put '"' around them.
fout<<'\"';
//The current character.
int c;
//Loop through the characters.
for(unsigned int i=0;i<s.size();i++){
c=s[i];
//If there's a '"' character it needs to be counter escaped. ("")
if(c=='\"'){
fout<<"\"\"";
}else{
//If it isn't we can just write away the character.
fout<<(char)c;
}
}
fout<<'\"';
}else{
//It doesn't contain any special characters so we can write it away.
fout<<s;
}
}
static void writeStringArray(std::ostream& fout,std::vector<std::string>& s){
//This method will write a away an array of strings.
//fout: The output stream to write to.
//s: Vector containing the strings to write.
//Loop the strings.
for(unsigned int i=0;i<s.size();i++){
//If it's the second or more there must be a ",".
if(i>0) fout<<',';
//Now write the string.
writeString(fout,s[i]);
}
}
static void pWriteNode(ITreeStorageReader* obj,std::ostream& fout,int indent,bool saveSubNodeOnly){
//Write the TreeStorageNode to the given output stream.
//obj: The TreeStorageNode to write away.
//fout: The output stream to write to.
//indent: Integer containing the number of indentations are needed.
//saveSubNodeOnly: Boolean if only the subNodes need to be saved.
//Boolean if the node has subNodes.
bool haveSubNodes=false;
void* lpUserData=NULL;
ITreeStorageReader* objSubNode=NULL;
string s;
vector<string> v;
//---
if(obj==NULL) return;
//---
if(!saveSubNodeOnly){
for(int i=0;i<indent;i++) fout<<'\t';
s.clear();
obj->getName(s);
writeString(fout,s);
fout<<'(';
v.clear();
obj->getValue(v);
writeStringArray(fout,v);
fout<<')';
indent++;
}
//attributes
lpUserData=NULL;
for(;;){
s.clear();
v.clear();
lpUserData=obj->getNextAttribute(lpUserData,s,v);
if(lpUserData==NULL) break;
if(!haveSubNodes && !saveSubNodeOnly) fout<<"{\n";
haveSubNodes=true;
for(int i=0;i<indent;i++) fout<<'\t';
writeString(fout,s);
fout<<'=';
writeStringArray(fout,v);
fout<<'\n';
}
//subnodes
lpUserData=NULL;
for(;;){
lpUserData=obj->getNextNode(lpUserData,objSubNode);
if(lpUserData==NULL) break;
if(objSubNode!=NULL){
if(!haveSubNodes && !saveSubNodeOnly) fout<<"{\n";
haveSubNodes=true;
pWriteNode(objSubNode,fout,indent,false);
}
}
//---
if(!saveSubNodeOnly){
indent--;
if(haveSubNodes){
for(int i=0;i<indent;i++) fout<<'\t';
fout<<'}';
}
fout<<'\n';
}
}
void POASerializer::writeNode(ITreeStorageReader* obj,std::ostream& fout,bool writeHeader,bool saveSubNodeOnly){
//Make sure that the output stream isn't null.
if(!fout) return;
//It isn't so start writing the node.
pWriteNode(obj,fout,0,saveSubNodeOnly);
}

File Metadata

Mime Type
text/x-diff
Expires
Sat, May 16, 5:12 AM (13 h, 54 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
62948
Default Alt Text
(45 KB)

Event Timeline