source/ps/XMB/XMBStorage.cpp

   1 /* Copyright (C) 2022 Wildfire Games.
   2  * This file is part of 0 A.D.
   3  *
   4  * 0 A.D. is free software: you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License as published by
   6  * the Free Software Foundation, either version 2 of the License, or
   7  * (at your option) any later version.
   8  *
   9  * 0 A.D. is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with 0 A.D.  If not, see <http://www.gnu.org/licenses/>.
  16  */
  17
  18 #include "precompiled.h"
  19
  20 #include "XMBStorage.h"
  21
  22 #include "lib/file/io/write_buffer.h"
  23 #include "lib/file/vfs/vfs.h"
  24 #include "ps/CLogger.h"
  25 #include "scriptinterface/Object.h"
  26 #include "scriptinterface/ScriptConversions.h"
  27 #include "scriptinterface/ScriptExtraHeaders.h"
  28 #include "scriptinterface/ScriptInterface.h"
  29
  30 #include <libxml/parser.h>
  31 #include <unordered_map>
  32
  33 const char* XMBStorage::HeaderMagicStr = "XMB0";
  34 const char* XMBStorage::UnfinishedHeaderMagicStr = "XMBu";
  35 // Arbitrary version number - change this if we update the code and
  36 // need to invalidate old users' caches
  37 const u32 XMBStorage::XMBVersion = 4;
  38
  39 namespace
  40 {
  41 class XMBStorageWriter
  42 {
  43 public:
  44         template<typename ...Args>
  45         bool Load(WriteBuffer& writeBuffer, Args&&... args);
  46
  47         int GetElementName(const std::string& name) { return GetName(m_ElementSize, m_ElementIDs, name); }
  48         int GetAttributeName(const std::string& name) { return GetName(m_AttributeSize, m_AttributeIDs, name); }
  49
  50 protected:
  51         int GetName(int& totalSize, std::unordered_map<std::string, int>& names, const std::string& name)
  52         {
  53                 int nameIdx = totalSize;
  54                 auto [iterator, inserted] = names.try_emplace(name, nameIdx);
  55                 if (inserted)
  56                         totalSize += name.size() + 5; // Add 1 for the null terminator & 4 for the size int.
  57                 return iterator->second;
  58         }
  59
  60         void OutputNames(WriteBuffer& writeBuffer, const std::unordered_map<std::string, int>& names) const;
  61
  62         template<typename ...Args>
  63         bool OutputElements(WriteBuffer&, Args...)
  64         {
  65                 static_assert(sizeof...(Args) != sizeof...(Args), "OutputElements must be specialized.");
  66                 return false;
  67         }
  68
  69         int m_ElementSize = 0;
  70         int m_AttributeSize = 0;
  71         std::unordered_map<std::string, int> m_ElementIDs;
  72         std::unordered_map<std::string, int> m_AttributeIDs;
  73 };
  74
  75 // Output text, prefixed by length in bytes (including null-terminator)
  76 void WriteStringAndLineNumber(WriteBuffer& writeBuffer, const std::string& text, int lineNumber)
  77 {
  78         if (text.empty())
  79         {
  80                 // No text; don't write much
  81                 writeBuffer.Append("\0\0\0\0", 4);
  82         }
  83         else
  84         {
  85                 // Write length and line number and null-terminated text
  86                 u32 nodeLen = u32(4 + text.length() + 1);
  87                 writeBuffer.Append(&nodeLen, 4);
  88                 writeBuffer.Append(&lineNumber, 4);
  89                 writeBuffer.Append((void*)text.c_str(), nodeLen-4);
  90         }
  91 }
  92
  93 template<typename ...Args>
  94 bool XMBStorageWriter::Load(WriteBuffer& writeBuffer, Args&&... args)
  95 {
  96         // Header
  97         writeBuffer.Append(XMBStorage::UnfinishedHeaderMagicStr, 4);
  98         // Version
  99         writeBuffer.Append(&XMBStorage::XMBVersion, 4);
 100
 101         // Filled in below.
 102         size_t elementPtr = writeBuffer.Size();
 103         writeBuffer.Append("????????", 8);
 104         // Likewise with attributes.
 105         size_t attributePtr = writeBuffer.Size();
 106         writeBuffer.Append("????????", 8);
 107
 108         if (!OutputElements<Args&&...>(writeBuffer, std::forward<Args>(args)...))
 109                 return false;
 110
 111         u32 data = writeBuffer.Size();
 112         writeBuffer.Overwrite(&data, 4, elementPtr);
 113         data = m_ElementIDs.size();
 114         writeBuffer.Overwrite(&data, 4, elementPtr + 4);
 115         OutputNames(writeBuffer, m_ElementIDs);
 116
 117         data = writeBuffer.Size();
 118         writeBuffer.Overwrite(&data, 4, attributePtr);
 119         data = m_AttributeIDs.size();
 120         writeBuffer.Overwrite(&data, 4, attributePtr + 4);
 121         OutputNames(writeBuffer, m_AttributeIDs);
 122
 123         // File is now valid, so insert correct magic string.
 124         writeBuffer.Overwrite(XMBStorage::HeaderMagicStr, 4, 0);
 125
 126         return true;
 127 }
 128
 129 void XMBStorageWriter::OutputNames(WriteBuffer& writeBuffer, const std::unordered_map<std::string, int>& names) const
 130 {
 131         std::vector<std::pair<std::string, int>> orderedElements;
 132         for (const std::pair<const std::string, int>& n : names)
 133                 orderedElements.emplace_back(n);
 134         std::sort(orderedElements.begin(), orderedElements.end(), [](const auto& a, const auto&b) { return a.second < b.second; });
 135         for (const std::pair<std::string, int>& n : orderedElements)
 136         {
 137                 u32 textLen = (u32)n.first.length() + 1;
 138                 writeBuffer.Append(&textLen, 4);
 139                 writeBuffer.Append((void*)n.first.c_str(), textLen);
 140         }
 141 }
 142
 143 class JSNodeData
 144 {
 145 public:
 146         JSNodeData(const ScriptInterface& s) : scriptInterface(s), rq(s) {}
 147
 148         bool Setup(XMBStorageWriter& xmb, JS::HandleValue value);
 149         bool Output(WriteBuffer& writeBuffer, JS::HandleValue value) const;
 150
 151         std::vector<std::pair<u32, std::string>> m_Attributes;
 152         std::vector<std::pair<u32, JS::Heap<JS::Value>>> m_Children;
 153
 154         const ScriptInterface& scriptInterface;
 155         const ScriptRequest rq;
 156 };
 157
 158 template<>
 159 bool XMBStorageWriter::OutputElements<JSNodeData&, const u32&, JS::HandleValue&&>(WriteBuffer& writeBuffer, JSNodeData& data, const u32& nodeName, JS::HandleValue&& value)
 160 {
 161         // Set up variables.
 162         if (!data.Setup(*this, value))
 163                 return false;
 164
 165         size_t posLength = writeBuffer.Size();
 166         // Filled in later with the length of the element
 167         writeBuffer.Append("????", 4);
 168
 169         writeBuffer.Append(&nodeName, 4);
 170
 171         u32 attrCount = static_cast<u32>(data.m_Attributes.size());
 172         writeBuffer.Append(&attrCount, 4);
 173
 174         u32 childCount = data.m_Children.size();
 175         writeBuffer.Append(&childCount, 4);
 176
 177         // Filled in later with the offset to the list of child elements
 178         size_t posChildrenOffset = writeBuffer.Size();
 179         writeBuffer.Append("????", 4);
 180
 181         data.Output(writeBuffer, value);
 182
 183         // Output attributes
 184         for (const std::pair<const u32, std::string> attr : data.m_Attributes)
 185         {
 186                 writeBuffer.Append(&attr.first, 4);
 187                 u32 attrLen = u32(attr.second.size())+1;
 188                 writeBuffer.Append(&attrLen, 4);
 189                 writeBuffer.Append((void*)attr.second.c_str(), attrLen);
 190         }
 191
 192         // Go back and fill in the child-element offset
 193         u32 childrenOffset = (u32)(writeBuffer.Size() - (posChildrenOffset+4));
 194         writeBuffer.Overwrite(&childrenOffset, 4, posChildrenOffset);
 195
 196         // Output all child elements, making a copy since data will be overwritten.
 197         std::vector<std::pair<u32, JS::Heap<JS::Value>>> children = data.m_Children;
 198         for (const std::pair<u32, JS::Heap<JS::Value>>& child : children)
 199         {
 200                 JS::RootedValue val(data.rq.cx, child.second);
 201                 if (!OutputElements<JSNodeData&, const u32&, JS::HandleValue&&>(writeBuffer, data, child.first, val))
 202                         return false;
 203         }
 204
 205         // Go back and fill in the length
 206         u32 length = (u32)(writeBuffer.Size() - posLength);
 207         writeBuffer.Overwrite(&length, 4, posLength);
 208
 209         return true;
 210 }
 211
 212 bool JSNodeData::Setup(XMBStorageWriter& xmb, JS::HandleValue value)
 213 {
 214         m_Attributes.clear();
 215         m_Children.clear();
 216         JSType valType = JS_TypeOfValue(rq.cx, value);
 217         if (valType != JSTYPE_OBJECT)
 218                 return true;
 219
 220         std::vector<std::string> props;
 221         if (!Script::EnumeratePropertyNames(rq, value, true, props))
 222         {
 223                 LOGERROR("Failed to enumerate component properties.");
 224                 return false;
 225         }
 226
 227         for (const std::string& prop : props)
 228         {
 229                 // Special 'value' key.
 230                 if (prop == "_string")
 231                         continue;
 232
 233                 bool attrib = !prop.empty() && prop.front() == '@';
 234
 235                 std::string_view name = prop;
 236                 if (!attrib && !prop.empty() && prop.back() == '@')
 237                 {
 238                         size_t idx = prop.substr(0, prop.size()-1).find_last_of('@');
 239                         if (idx == std::string::npos)
 240                         {
 241                                 LOGERROR("Object key name cannot end with an '@' unless it is an index specifier.");
 242                                 return false;
 243                         }
 244                         name = std::string_view(prop.c_str(), idx);
 245                 }
 246                 else if (attrib)
 247                         name = std::string_view(prop.c_str()+1, prop.length()-1);
 248
 249                 JS::RootedValue child(rq.cx);
 250                 if (!Script::GetProperty(rq, value, prop.c_str(), &child))
 251                         return false;
 252
 253                 if (attrib)
 254                 {
 255                         std::string attrVal;
 256                         if (!Script::FromJSVal(rq, child, attrVal))
 257                         {
 258                                 LOGERROR("Attributes must be convertible to string");
 259                                 return false;
 260                         }
 261                         m_Attributes.emplace_back(xmb.GetAttributeName(std::string(name)), attrVal);
 262                         continue;
 263                 }
 264
 265                 bool isArray = false;
 266                 if (!JS::IsArrayObject(rq.cx, child, &isArray))
 267                         return false;
 268                 if (!isArray)
 269                 {
 270                         m_Children.emplace_back(xmb.GetElementName(std::string(name)), child);
 271                         continue;
 272                 }
 273
 274                 // Parse each array object as a child.
 275                 JS::RootedObject obj(rq.cx);
 276                 JS_ValueToObject(rq.cx, child, &obj);
 277                 u32 length;
 278                 JS::GetArrayLength(rq.cx, obj, &length);
 279                 for (size_t i = 0; i < length; ++i)
 280                 {
 281                         JS::RootedValue arrayChild(rq.cx);
 282                         Script::GetPropertyInt(rq, child, i, &arrayChild);
 283                         m_Children.emplace_back(xmb.GetElementName(std::string(name)), arrayChild);
 284                 }
 285         }
 286         return true;
 287 }
 288
 289 bool JSNodeData::Output(WriteBuffer& writeBuffer, JS::HandleValue value) const
 290 {
 291         switch (JS_TypeOfValue(rq.cx, value))
 292         {
 293                 case JSTYPE_UNDEFINED:
 294                 case JSTYPE_NULL:
 295                 {
 296                         writeBuffer.Append("\0\0\0\0", 4);
 297                         break;
 298                 }
 299                 case JSTYPE_OBJECT:
 300                 {
 301                         if (!Script::HasProperty(rq, value, "_string"))
 302                         {
 303                                 writeBuffer.Append("\0\0\0\0", 4);
 304                                 break;
 305                         }
 306                         JS::RootedValue actualValue(rq.cx);
 307                         if (!Script::GetProperty(rq, value, "_string", &actualValue))
 308                                 return false;
 309                         std::string strVal;
 310                         if (!Script::FromJSVal(rq, actualValue, strVal))
 311                         {
 312                                 LOGERROR("'_string' value must be convertible to string");
 313                                 return false;
 314                         }
 315                         WriteStringAndLineNumber(writeBuffer, strVal, 0);
 316                         break;
 317                 }
 318                 case JSTYPE_STRING:
 319                 case JSTYPE_NUMBER:
 320                 {
 321                         std::string strVal;
 322                         if (!Script::FromJSVal(rq, value, strVal))
 323                                 return false;
 324
 325                         WriteStringAndLineNumber(writeBuffer, strVal, 0);
 326                         break;
 327                 }
 328                 default:
 329                 {
 330                         LOGERROR("Unsupported JS construct when parsing ParamNode");
 331                         return false;
 332                 }
 333         }
 334         return true;
 335 }
 336
 337 template<>
 338 bool XMBStorageWriter::OutputElements<xmlNodePtr&&>(WriteBuffer& writeBuffer, xmlNodePtr&& node)
 339 {
 340         // Filled in later with the length of the element
 341         size_t posLength = writeBuffer.Size();
 342         writeBuffer.Append("????", 4);
 343
 344         u32 name = GetElementName((const char*)node->name);
 345         writeBuffer.Append(&name, 4);
 346
 347         u32 attrCount = 0;
 348         for (xmlAttrPtr attr = node->properties; attr; attr = attr->next)
 349                 ++attrCount;
 350         writeBuffer.Append(&attrCount, 4);
 351
 352         u32 childCount = 0;
 353         for (xmlNodePtr child = node->children; child; child = child->next)
 354                 if (child->type == XML_ELEMENT_NODE)
 355                         ++childCount;
 356         writeBuffer.Append(&childCount, 4);
 357
 358         // Filled in later with the offset to the list of child elements
 359         size_t posChildrenOffset = writeBuffer.Size();
 360         writeBuffer.Append("????", 4);
 361
 362
 363         // Trim excess whitespace in the entity's text, while counting
 364         // the number of newlines trimmed (so that JS error reporting
 365         // can give the correct line number within the script)
 366
 367         std::string whitespace = " \t\r\n";
 368         std::string text;
 369         for (xmlNodePtr child = node->children; child; child = child->next)
 370         {
 371                 if (child->type == XML_TEXT_NODE)
 372                 {
 373                         xmlChar* content = xmlNodeGetContent(child);
 374                         text += std::string((const char*)content);
 375                         xmlFree(content);
 376                 }
 377         }
 378
 379         u32 linenum = xmlGetLineNo(node);
 380
 381         // Find the start of the non-whitespace section
 382         size_t first = text.find_first_not_of(whitespace);
 383
 384         if (first == text.npos)
 385                 // Entirely whitespace - easy to handle
 386                 text = "";
 387
 388         else
 389         {
 390                 // Count the number of \n being cut off,
 391                 // and add them to the line number
 392                 std::string trimmed (text.begin(), text.begin()+first);
 393                 linenum += std::count(trimmed.begin(), trimmed.end(), '\n');
 394
 395                 // Find the end of the non-whitespace section,
 396                 // and trim off everything else
 397                 size_t last = text.find_last_not_of(whitespace);
 398                 text = text.substr(first, 1+last-first);
 399         }
 400
 401
 402         // Output text, prefixed by length in bytes
 403         WriteStringAndLineNumber(writeBuffer, text, linenum);
 404
 405         // Output attributes
 406         for (xmlAttrPtr attr = node->properties; attr; attr = attr->next)
 407         {
 408                 u32 attrName = GetAttributeName((const char*)attr->name);
 409                 writeBuffer.Append(&attrName, 4);
 410
 411                 xmlChar* value = xmlNodeGetContent(attr->children);
 412                 u32 attrLen = u32(xmlStrlen(value)+1);
 413                 writeBuffer.Append(&attrLen, 4);
 414                 writeBuffer.Append((void*)value, attrLen);
 415                 xmlFree(value);
 416         }
 417
 418         // Go back and fill in the child-element offset
 419         u32 childrenOffset = (u32)(writeBuffer.Size() - (posChildrenOffset+4));
 420         writeBuffer.Overwrite(&childrenOffset, 4, posChildrenOffset);
 421
 422         // Output all child elements
 423         for (xmlNodePtr child = node->children; child; child = child->next)
 424                 if (child->type == XML_ELEMENT_NODE)
 425                         OutputElements<xmlNodePtr&&>(writeBuffer, std::move(child));
 426
 427         // Go back and fill in the length
 428         u32 length = (u32)(writeBuffer.Size() - posLength);
 429         writeBuffer.Overwrite(&length, 4, posLength);
 430
 431         return true;
 432 }
 433 } // anonymous namespace
 434
 435 bool XMBStorage::ReadFromFile(const PIVFS& vfs, const VfsPath& filename)
 436 {
 437         if(vfs->LoadFile(filename, m_Buffer, m_Size) < 0)
 438                 return false;
 439         // if the game crashes during loading, (e.g. due to driver bugs),
 440         // it sometimes leaves empty XMB files in the cache.
 441         // reporting failure will cause our caller to re-generate the XMB.
 442         if (m_Size == 0)
 443                 return false;
 444         ENSURE(m_Size >= 4); // make sure it's at least got the initial header
 445         return true;
 446 }
 447
 448 bool XMBStorage::LoadXMLDoc(const xmlDocPtr doc)
 449 {
 450         WriteBuffer writeBuffer;
 451
 452         XMBStorageWriter writer;
 453         if (!writer.Load(writeBuffer, std::move(xmlDocGetRootElement(doc))))
 454                 return false;
 455
 456         m_Buffer = writeBuffer.Data(); // add a reference
 457         m_Size = writeBuffer.Size();
 458         return true;
 459 }
 460
 461 bool XMBStorage::LoadJSValue(const ScriptInterface& scriptInterface, JS::HandleValue value, const std::string& rootName)
 462 {
 463         WriteBuffer writeBuffer;
 464
 465         XMBStorageWriter writer;
 466         const u32 name = writer.GetElementName(rootName);
 467         JSNodeData data(scriptInterface);
 468         if (!writer.Load(writeBuffer, data, name, std::move(value)))
 469                 return false;
 470
 471         m_Buffer = writeBuffer.Data(); // add a reference
 472         m_Size = writeBuffer.Size();
 473         return true;
 474 }