BVB Source Codes

CRYENGINE Show XMLCPB_Writer.cpp Source code

Return Download CRYENGINE: download XMLCPB_Writer.cpp Source code - Download CRYENGINE Source code - Type:.cpp
  1. // Copyright 2001-2016 Crytek GmbH / Crytek Group. All rights reserved.
  2.  
  3. /*************************************************************************
  4. *************************************************************************/
  5.  
  6. // overwiew of the writting process:
  7. //
  8. // - When a node is created, it is temporarily added to the "LiveNodes" list.
  9. //
  10. // - As nodes are being finished, their data is appended to the main buffer, and their spot in the live list is freed to be ready for another new node
  11. //
  12. // - if streaming is used, sub-buffers of the main buffer are written into the file as soon as they are filled.
  13. //
  14. // - tags, attribute names, and attribute string data is added to the StringTableWriter objects when every node/attr is created. They are written into the file only at the end of the process.
  15. //
  16.  
  17. #include "StdAfx.h"
  18. #include "XMLCPB_Writer.h"
  19. #include "XMLCPB_ZLibCompressor.h"
  20. #include "CryActionCVars.h"
  21. #include "../XMLCPB_Utils.h"
  22.  
  23. namespace XMLCPB
  24. {
  25.  
  26. uint64* AttrStringAllocatorImpl::s_buckets[AttrStringAllocatorImpl::k_maxItems / AttrStringAllocatorImpl::k_numPerBucket];
  27. uint64* AttrStringAllocatorImpl::s_ptr;
  28. uint64* AttrStringAllocatorImpl::s_end;
  29. int AttrStringAllocatorImpl::s_currentBucket;
  30. int AttrStringAllocatorImpl::s_poolInUse;
  31.  
  32. //////////////////////////////////////////////////////////////////////////
  33.  
  34. CWriter::CWriter()
  35.         : m_compressor(nullptr)
  36.         , m_firstFreeLiveNode(0)
  37.         , m_tableTags(MAX_NUM_TAGS, 4096)
  38.         , m_tableAttrNames(MAX_NUM_NAMES, 4096)
  39.         , m_tableStrData(MAX_NUM_STRDATA, 32 * 1024)
  40.         , m_tableStrDataConstants(DT_NUM_CONST_STR, 2048)
  41.         , m_tableAttrSets(BIT(16), 4096)
  42.         , m_mainBuffer(MAIN_BUFFER_SIZE_INCREMENT)
  43.         , m_safecheckIDCounter(0)
  44.         , m_isSavingIntoFile(false)
  45.         , m_counterCompactedNodes(0)
  46.         , m_hasInternalError(false)
  47. {
  48.         m_liveNodes.resize(MAX_NUM_LIVE_NODES);
  49.         AttrStringAllocatorImpl::LockPool();
  50. }
  51.  
  52. void CWriter::Init(const char* pNameRootNode, const char* pFileName)
  53. {
  54.         m_isSavingIntoFile = pFileName != NULL;
  55.         bool useStreaming = m_isSavingIntoFile;
  56.         m_mainBuffer.Init(this, useStreaming);
  57.         m_tableTags.Init(this);
  58.         m_tableAttrNames.Init(this);
  59.         m_tableStrData.Init(this);
  60.         m_tableStrDataConstants.Init(this);
  61.         m_tableAttrSets.Init(this);
  62.         if (m_isSavingIntoFile)
  63.         {
  64. #ifdef XMLCPB_DEBUGUTILS
  65.                 CDebugUtils::SetLastFileNameSaved(pFileName);
  66. #endif
  67. #ifdef XMLCPB_COLLECT_STATS
  68.                 CNodeLiveWriter::m_stats.Reset();
  69.                 CAttrWriter::m_statistics.Reset();
  70. #endif
  71.                 m_tableStrDataConstants.CreateStringsFromConstants();
  72.                 {
  73.                         m_compressor = new CZLibCompressor(pFileName); // Don't delete this when finished, it is held by the compressor thread until finished with and destroyed by it.
  74.                 }
  75.         }
  76.  
  77.         CNodeLiveWriter* pRoot = CreateAndAddLiveNode(pNameRootNode);
  78.         assert(pRoot->GetID() == XMLCPB_ROOTNODE_ID);
  79. }
  80.  
  81. //////////////////////////////////////////////////////////////////////////
  82.  
  83. CWriter::~CWriter()
  84. {
  85. #ifdef XMLCPB_COLLECT_STATS
  86.         if (m_isSavingIntoFile)
  87.         {
  88.                 LogStatistics();
  89.                 CAttrWriter::WriteFileStatistics(m_tableStrData);
  90.         }
  91. #endif
  92. }
  93.  
  94. //////////////////////////////////////////////////////////////////////////
  95.  
  96. CWriterBase::~CWriterBase()
  97. {
  98.         AttrStringAllocatorImpl::CleanPool();
  99. }
  100.  
  101. //////////////////////////////////////////////////////////////////////////
  102.  
  103. CNodeLiveWriterRef CWriter::GetRoot()
  104. {
  105.         return CNodeLiveWriterRef(*this, XMLCPB_ROOTNODE_ID);
  106. }
  107.  
  108. //////////////////////////////////////////////////////////////////////////
  109.  
  110. CNodeLiveWriter* CWriter::CreateAndAddLiveNode(const char* pName)
  111. {
  112.         assert(m_firstFreeLiveNode < m_liveNodes.size());
  113.  
  114.         // create and place in vector
  115.         NodeLiveID ID = m_firstFreeLiveNode;
  116.         StringID stringID = m_tableTags.GetStringID(pName);
  117.         CNodeLiveWriter* pNode = m_liveNodes[ID];
  118.         if (!pNode)
  119.         {
  120.                 pNode = new CNodeLiveWriter(*this, ID, stringID, m_safecheckIDCounter);
  121.                 m_liveNodes[ID] = pNode;
  122.         }
  123.         else
  124.                 pNode->Reuse(stringID, m_safecheckIDCounter);
  125.  
  126.         m_safecheckIDCounter++;
  127.  
  128.         // find the now first free live node
  129.         bool found = false;
  130.         for (int i = m_firstFreeLiveNode + 1; i < m_liveNodes.size(); ++i)
  131.         {
  132.                 CNodeLiveWriter* pNodeIter = m_liveNodes[i];
  133.                 if (pNodeIter == NULL || !pNodeIter->IsValid())
  134.                 {
  135.                         found = true;
  136.                         m_firstFreeLiveNode = i;
  137.                         break;
  138.                 }
  139.         }
  140.  
  141.         assert(found);
  142.  
  143.         return pNode;
  144. }
  145.  
  146. //////////////////////////////////////////////////////////////////////////
  147.  
  148. CNodeLiveWriter* CWriter::GetNodeLive(NodeLiveID nodeId)
  149. {
  150.         assert(nodeId < m_liveNodes.size());
  151.  
  152.         return m_liveNodes[nodeId];
  153. }
  154.  
  155. //////////////////////////////////////////////////////////////////////////
  156.  
  157. void CWriter::FreeNodeLive(NodeLiveID nodeId)
  158. {
  159.         if (nodeId < m_firstFreeLiveNode)
  160.                 m_firstFreeLiveNode = nodeId;
  161. }
  162.  
  163. //////////////////////////////////////////////////////////////////////////
  164.  
  165. void CWriter::Done()
  166. {
  167.         CNodeLiveWriter* pRoot = GetNodeLive(XMLCPB_ROOTNODE_ID);
  168.         if (!pRoot->IsDone())
  169.                 pRoot->Done();
  170. }
  171.  
  172. //////////////////////////////////////////////////////////////////////////
  173.  
  174. void CWriter::NotifyRootNodeStartCompact()
  175. {
  176.         assert(!m_rootAddr.IsValid());    // can only be one
  177.  
  178.         m_mainBuffer.GetCurrentAddr(m_rootAddr);
  179. }
  180.  
  181. //////////////////////////////////////////////////////////////////////////
  182. // for debug/statistics only.
  183. #ifdef XMLCPB_COLLECT_STATS
  184.  
  185. void CWriter::LogStatistics()
  186. {
  187.         CryLog("-----------Binary SaveGame Writer statistics--------------");
  188.  
  189.         //// live nodes info
  190.         {
  191.                 int nodesCreated = 0;
  192.                 int children = 0;
  193.                 int attrs = 0;
  194.                 for (int i = 0; i < m_liveNodes.size(); ++i)
  195.                 {
  196.                         CNodeLiveWriter* pNode = m_liveNodes[i];
  197.                         if (pNode)
  198.                         {
  199.                                 assert(!pNode->IsValid());
  200.                                 nodesCreated++;
  201.                                 children += pNode->m_children.capacity();
  202.                                 attrs += pNode->m_attrs.capacity();
  203.                         }
  204.                 }
  205.                 CryLog("live nodes.  created: %d/%d   children: %d    attrs: %d", nodesCreated, MAX_NUM_LIVE_NODES, children, attrs);
  206.         }
  207.  
  208.         CryLog("stringtables.    Tags: %d/%d (%d kb)    AttrNames: %d/%d  (%d kb)    stringsData: %d/%d  (%d kb)   total memory string tables: %d kb",
  209.                m_tableTags.GetNumStrings(), MAX_NUM_TAGS, m_tableTags.GetDataSize() / 1024,
  210.                m_tableAttrNames.GetNumStrings(), MAX_NUM_NAMES, m_tableAttrNames.GetDataSize() / 1024,
  211.                m_tableStrData.GetNumStrings(), MAX_NUM_STRDATA, m_tableStrData.GetDataSize() / 1024,
  212.                (m_tableTags.GetDataSize() + m_tableAttrNames.GetDataSize() + m_tableStrData.GetDataSize()) / 1024
  213.                );
  214.  
  215.         CryLog("Nodes.  total: %d   maxNumChildrenPerNode: %d  total children: %d   maxAttrsPerNode:%d  total attrs: %d  AttrSets: %d (%d kb)",
  216.                CNodeLiveWriter::m_stats.m_totalNodesCreated, CNodeLiveWriter::m_stats.m_maxNumChildren, CNodeLiveWriter::m_stats.m_totalChildren,
  217.                CNodeLiveWriter::m_stats.m_maxNumAttrs, CNodeLiveWriter::m_stats.m_totalAttrs,
  218.                m_tableAttrSets.GetNumSets(), m_tableAttrSets.GetDataSize() / 1024);
  219.  
  220.         {
  221.                 uint32 totalSize = m_mainBuffer.GetDataSize() + m_tableAttrNames.GetDataSize() + m_tableStrData.GetDataSize() + m_tableTags.GetDataSize() + sizeof(SFileHeader) + m_tableAttrSets.GetDataSize();
  222.                 CryLog("size:  total: %d (%d kb)  nodes basic info: %d  attr: %d   tagsStringTable: %d   attrStringTable: %d  dataStringTable: %d   attrSets: %d",
  223.                        totalSize, totalSize / 1024, CNodeLiveWriter::m_stats.m_totalSizeNodeData, CNodeLiveWriter::m_stats.m_totalSizeAttrData, m_tableTags.GetDataSize(), m_tableAttrNames.GetDataSize(), m_tableStrData.GetDataSize(), m_tableAttrSets.GetDataSize());
  224.         }
  225.  
  226.         CryLog("-------------------");
  227. }
  228. #endif
  229.  
  230. //////////////////////////////////////////////////////////////////////////
  231. // high level function for writing data into a file. it will use (or not) zlib compression depending on the cvar
  232. void CWriter::WriteDataIntoFile(void* pSrc, uint32 numBytes)
  233. {
  234.         m_compressor->WriteDataIntoFile(pSrc, numBytes);
  235. }
  236.  
  237. //////////////////////////////////////////////////////////////////////////
  238.  
  239. void CWriter::WriteDataIntoMemory(uint8*& rpData, void* pSrc, uint32 numBytes, uint32& outWriteLoc)
  240. {
  241.         assert(!m_isSavingIntoFile);
  242.         assert(rpData);
  243.         assert(pSrc);
  244.         assert(numBytes > 0);
  245.  
  246.         if (rpData && pSrc && numBytes > 0)
  247.         {
  248.                 memcpy(&rpData[outWriteLoc], pSrc, numBytes);
  249.                 outWriteLoc += numBytes;
  250.         }
  251. }
  252.  
  253. //////////////////////////////////////////////////////////////////////////
  254. // file structure:
  255. //
  256. // - header
  257. // - tagsTable
  258. // - attrNamesTable
  259. // - strDataTable
  260. // - nodes data
  261.  
  262. bool CWriter::FinishWritingFile()
  263. {
  264.         // if this happens, most likely is an overflow of some of the hard coded limits ( which usually is caused by an error in game side ).
  265.         if (m_hasInternalError)
  266.                 CryWarning(VALIDATOR_MODULE_SYSTEM, VALIDATOR_ERROR, "XMLCPB: ERROR in binary save generation. The savegame is corrupted.");
  267.  
  268.         if (!m_compressor->m_errorWritingIntoFile)
  269.         {
  270.                 if (!m_rootAddr.IsValid())
  271.                         Done();
  272.  
  273.                 assert(m_rootAddr.IsValid());
  274.  
  275.                 m_mainBuffer.WriteToFile(); // this actually writes only the last data remains, because it has been writing all along the process.
  276.                 m_tableTags.WriteToFile();
  277.                 m_tableAttrNames.WriteToFile();
  278.                 m_tableStrData.WriteToFile();
  279.                 m_tableAttrSets.WriteToFile();
  280.  
  281.                 CreateFileHeader(m_compressor->GetFileHeader());
  282.  
  283.                 m_compressor->FlushZLibBuffer();
  284.         }
  285.  
  286.         return !m_compressor->m_errorWritingIntoFile;
  287. }
  288.  
  289. //////////////////////////////////////////////////////////////////////////
  290.  
  291. bool CWriter::WriteAllIntoMemory(uint8*& rpData, uint32& outSize)
  292. {
  293.         if (!m_rootAddr.IsValid())
  294.                 Done();
  295.  
  296.         SFileHeader fileHeader;
  297.         CreateFileHeader(fileHeader);
  298.  
  299.         outSize = sizeof(fileHeader);
  300.         outSize += m_tableTags.GetDataSize();
  301.         outSize += m_tableAttrNames.GetDataSize();
  302.         outSize += m_tableStrData.GetDataSize();
  303.         outSize += m_tableAttrSets.GetDataSize();
  304.         outSize += m_mainBuffer.GetDataSize();
  305.  
  306.         if (outSize > 0)
  307.         {
  308.                 rpData = (uint8*)realloc((void*)rpData, outSize * sizeof(uint8));
  309.  
  310.                 uint32 uWriteLoc = 0;
  311.                 WriteDataIntoMemory(rpData, &fileHeader, sizeof(fileHeader), uWriteLoc);
  312.                 m_mainBuffer.WriteToMemory(rpData, uWriteLoc);
  313.                 m_tableTags.WriteToMemory(rpData, uWriteLoc);
  314.                 m_tableAttrNames.WriteToMemory(rpData, uWriteLoc);
  315.                 m_tableStrData.WriteToMemory(rpData, uWriteLoc);
  316.                 m_tableAttrSets.WriteToMemory(rpData, uWriteLoc);
  317.         }
  318.  
  319.         if (m_hasInternalError)
  320.                 CryWarning(VALIDATOR_MODULE_SYSTEM, VALIDATOR_ERROR, "XMLCPB: ERROR in binary save-into-memory generation. (probably something wrong in a pooled entity). The data will be corrupted");
  321.  
  322.         return (outSize > 0);
  323. }
  324.  
  325. //////////////////////////////////////////////////////////////////////////
  326. // a bit misleading, because the header is used when writing into memory too.
  327.  
  328. void CWriter::CreateFileHeader(SFileHeader& fileHeader)
  329. {
  330.         assert(m_rootAddr.IsValid());
  331.  
  332.         m_tableTags.FillFileHeaderInfo(fileHeader.m_tags);
  333.         m_tableAttrNames.FillFileHeaderInfo(fileHeader.m_attrNames);
  334.         m_tableStrData.FillFileHeaderInfo(fileHeader.m_strData);
  335.  
  336.         fileHeader.m_numAttrSets = m_tableAttrSets.GetNumSets();
  337.         fileHeader.m_sizeAttrSets = m_tableAttrSets.GetSetsDataSize();
  338.  
  339.         fileHeader.m_sizeNodes = m_mainBuffer.GetUsedMemory();
  340.         fileHeader.m_numNodes = m_counterCompactedNodes;
  341.  
  342.         fileHeader.m_hasInternalError = m_hasInternalError;
  343. }
  344.  
  345. } // namespace XMLCPB
  346.  
downloadXMLCPB_Writer.cpp Source code - Download CRYENGINE Source code
Related Source Codes/Software:
postal - 2017-06-11
reactide - Reactide is the first dedicated IDE for React web ... 2017-06-11
rkt - rkt is a pod-native container engine for Linux. It... 2017-06-11
uWebSockets - Tiny WebSockets https://for... 2017-06-11
realworld - TodoMVC for the RealWorld - Exemplary fullstack Me... 2017-06-11
CRYENGINE - CRYENGINE is a powerful real-time game development... 2017-06-11
goreplay - GoReplay is an open-source tool for capturing and ... 2017-06-10
pyenv - Simple Python version management 2017-06-10
redux-saga - An alternative side effect model for Redux apps ... 2017-06-10
angular-starter - 2017-06-10

 Back to top