Update README.md (#139)

* Update README.md

* Update README.md

Co-authored-by: Hyper <34012267+hyperbx@users.noreply.github.com>

---------

Co-authored-by: Hyper <34012267+hyperbx@users.noreply.github.com>

README.md

@@ -4,6 +4,8 @@ XenonRecomp is a tool that converts Xbox 360 executables into C++ code, which ca
 
 This project was heavily inspired by [N64: Recompiled](https://github.com/N64Recomp/N64Recomp), a similar tool for N64 executables.
 
+**DISCLAIMER:** This project does not provide a runtime implementation. It only converts the game code to C++, which is not going to function correctly without a runtime backing it. **Making the game work is your responsibility.**
+
 ## Implementation Details
 
 ### Instructions

XenonRecomp/recompiler.cpp

@@ -2001,7 +2001,7 @@ bool Recompiler::Recompile(
         switch (insn.operands[2])
         {
         case 0: // D3D color
-            if (insn.operands[3] != 1 || insn.operands[4] != 3)
+            if (insn.operands[3] != 1)
                 fmt::println("Unexpected D3D color pack instruction at {:X}", base);
 
             for (size_t i = 0; i < 4; i++)
@@ -2011,7 +2011,29 @@ bool Recompiler::Recompile(
                 println("\t{}.f32[{}] = {}.f32[{}] < 3.0f ? 3.0f : ({}.f32[{}] > {}.f32[{}] ? {}.f32[{}] : {}.f32[{}]);", vTemp(), i, v(insn.operands[1]), i, v(insn.operands[1]), i, vTemp(), i, vTemp(), i, v(insn.operands[1]), i);
                 println("\t{}.u32 {}= uint32_t({}.u8[{}]) << {};", temp(), i == 0 ? "" : "|", vTemp(), i * 4, indices[i] * 8);
             }
-            println("\t{}.u32[3] = {}.u32;", v(insn.operands[0]), temp());
+            println("\t{}.u32[{}] = {}.u32;", v(insn.operands[0]), insn.operands[4], temp());
+            break;
+
+        case 5: // float16_4
+            if (insn.operands[3] != 2 || insn.operands[4] > 2)
+                fmt::println("Unexpected float16_4 pack instruction at {:X}", base);
+
+            for (size_t i = 0; i < 4; i++)
+            {
+        		// Strip sign from source
+        		println("\t{}.u32 = ({}.u32[{}]&0x7FFFFFFF);", temp(), v(insn.operands[1]), i);
+        		// If |source| is > 65504, clamp output to 0x7FFF, else save 8 exponent bits 
+        		println("\t{0}.u8[0] = ({1}.f32 != {1}.f32) || ({1}.f32 > 65504.0f) ? 0xFF : (({2}.u32[{3}]&0x7f800000)>>23);", vTemp(), temp(), v(insn.operands[1]), i);
+        		// If 8 exponent bits were saved, it can only be 0x8E at most
+        		// If saved, save first 10 bits of mantissa
+        		println("\t{}.u16 = {}.u8[0] != 0xFF ? (({}.u32[{}]&0x7FE000)>>13) : 0x0;", temp(), vTemp(), v(insn.operands[1]), i);
+        		// If saved and > 127-15, exponent is converted from 8 to 5-bit by subtracting 0x70
+        		// If saved but not > 127-15, clamp exponent at 0, add 0x400 to mantissa and shift right by (0x71-exponent)
+        		// If right shift is greater than 31 bits, manually clamp mantissa to 0 or else the output of the shift will be wrong
+        		println("\t{0}.u16[{1}] = {2}.u8[0] != 0xFF ? ({2}.u8[0] > 0x70 ? ((({2}.u8[0]-0x70)<<10)+{3}.u16) : (0x71-{2}.u8[0] > 31 ? 0x0 : ((0x400+{3}.u16)>>(0x71-{2}.u8[0])))) : 0x7FFF;", v(insn.operands[0]), i+(2*insn.operands[4]), vTemp(), temp());
+        		// Add back original sign
+        		println("\t{}.u16[{}] |= (({}.u32[{}]&0x80000000)>>16);", v(insn.operands[0]), i+(2*insn.operands[4]), v(insn.operands[1]), i);
+            }
             break;
 
         default:

XenonUtils/xex.cpp

@@ -5,6 +5,8 @@
 #include <vector>
 #include <unordered_map>
 #include <aes.hpp>
+#include <TinySHA1.hpp>
+#include <xex_patcher.h>
 
 #define STRINGIFY(X) #X
 #define XE_EXPORT(MODULE, ORDINAL, NAME, TYPE) { (ORDINAL), "__imp__" STRINGIFY(NAME) }
@@ -135,7 +137,7 @@ Image Xex2LoadImage(const uint8_t* data, size_t dataSize)
     // Decompress image
     if (fileFormatInfo != nullptr)
     {
-        assert(fileFormatInfo->compressionType <= XEX_COMPRESSION_BASIC);
+        assert(fileFormatInfo->compressionType <= XEX_COMPRESSION_NORMAL);
 
         std::unique_ptr<uint8_t[]> decryptedData;
         const uint8_t* srcData = nullptr;
@@ -192,6 +194,67 @@ Image Xex2LoadImage(const uint8_t* data, size_t dataSize)
                 destData += blocks[i].zeroSize;
             }
         }
+        else if (fileFormatInfo->compressionType == XEX_COMPRESSION_NORMAL)
+        {
+            result = std::make_unique<uint8_t[]>(imageSize);
+            auto* destData = result.get();
+
+            const Xex2CompressedBlockInfo* blocks = &((const Xex2FileNormalCompressionInfo*)(fileFormatInfo + 1))->firstBlock;
+            const uint32_t headerSize = header->headerSize.get();
+
+            const uint32_t exeLength = dataSize - headerSize;
+            const uint8_t* exeBuffer = srcData;
+
+            auto compressBuffer = std::make_unique<uint8_t[]>(exeLength);
+            const uint8_t* p = NULL;
+            uint8_t* d = NULL;
+            sha1::SHA1 s;
+
+            p = exeBuffer;
+            d = compressBuffer.get();
+
+            uint8_t blockCalcedDigest[0x14];
+            while (blocks->blockSize) 
+            {
+                const uint8_t* pNext = p + blocks->blockSize;
+                const auto* nextBlock = (const Xex2CompressedBlockInfo*)p;
+
+                s.reset();
+                s.processBytes(p, blocks->blockSize);
+                s.finalize(blockCalcedDigest);
+
+                if (memcmp(blockCalcedDigest, blocks->blockHash, 0x14) != 0)
+                    return {};
+
+                p += 4;
+                p += 20;
+
+                while (true) 
+                {
+                    const size_t chunkSize = (p[0] << 8) | p[1];
+                    p += 2;
+
+                    if (!chunkSize)
+                        break;
+
+                    memcpy(d, p, chunkSize);
+                    p += chunkSize;
+                    d += chunkSize;
+                }
+
+                p = pNext;
+                blocks = nextBlock;
+            }
+
+            int resultCode = 0;
+            uint32_t uncompressedSize = security->imageSize;
+            uint8_t* buffer = destData;
+
+            resultCode = lzxDecompress(compressBuffer.get(), d - compressBuffer.get(), buffer, uncompressedSize, ((const Xex2FileNormalCompressionInfo*)(fileFormatInfo + 1))->windowSize, nullptr, 0);
+
+            if (resultCode)
+                return {};
+        }
     }
 
     image.data = std::move(result);
@@ -201,8 +264,17 @@ Image Xex2LoadImage(const uint8_t* data, size_t dataSize)
     const auto* dosHeader = reinterpret_cast<IMAGE_DOS_HEADER*>(image.data.get());
     const auto* ntHeaders = reinterpret_cast<IMAGE_NT_HEADERS32*>(image.data.get() + dosHeader->e_lfanew);
 
-    image.base = ntHeaders->OptionalHeader.ImageBase;
+    image.base = security->loadAddress;
-    image.entry_point = image.base + ntHeaders->OptionalHeader.AddressOfEntryPoint;
+    const void* xex2BaseAddressPtr = getOptHeaderPtr(data, XEX_HEADER_IMAGE_BASE_ADDRESS);
+    if (xex2BaseAddressPtr != nullptr)
+    {
+        image.base = *reinterpret_cast<const be<uint32_t>*>(xex2BaseAddressPtr);
+    }
+    const void* xex2EntryPointPtr = getOptHeaderPtr(data, XEX_HEADER_ENTRY_POINT);
+    if (xex2EntryPointPtr != nullptr)
+    {
+        image.entry_point = *reinterpret_cast<const be<uint32_t>*>(xex2EntryPointPtr);
+    }
 
     const auto numSections = ntHeaders->FileHeader.NumberOfSections;
     const auto* sections = reinterpret_cast<const IMAGE_SECTION_HEADER*>(ntHeaders + 1);
@@ -227,10 +299,13 @@ Image Xex2LoadImage(const uint8_t* data, size_t dataSize)
         std::vector<std::string_view> stringTable;
         auto* pStrTable = reinterpret_cast<const char*>(imports + 1);
 
+        size_t paddedStringOffset = 0;
         for (size_t i = 0; i < imports->numImports; i++)
         {
-            stringTable.emplace_back(pStrTable);
+            stringTable.emplace_back(pStrTable + paddedStringOffset);
-            pStrTable += strlen(pStrTable) + 1;
+            
+            // pad the offset to the next multiple of 4
+            paddedStringOffset += ((stringTable.back().length() + 1) + 3) & ~3;
         }
 
         auto* library = (Xex2ImportLibrary*)(((char*)imports) + sizeof(Xex2ImportHeader) + imports->sizeOfStringTable);

XenonUtils/xex.h

@@ -247,11 +247,15 @@ inline const void* getOptHeaderPtr(const uint8_t* moduleBytes, uint32_t headerKe
         {
             if((headerKey & 0xFF) == 0)
             {
-                return &optHeader.value;
+                return reinterpret_cast<const uint32_t *>(&optHeader.value);
+            }
+            else if ((headerKey & 0xFF) == 1)
+            {
+                return reinterpret_cast<const void *>(&optHeader.value);
             }
             else
             {
-                return &moduleBytes[optHeader.offset];
+                return reinterpret_cast<const void *>(reinterpret_cast<uintptr_t>(moduleBytes) + optHeader.offset);
             }
         }
     }

XenonUtils/xex_patcher.cpp

@@ -403,7 +403,63 @@ XexPatcher::Result XexPatcher::apply(const uint8_t* xexBytes, size_t xexBytesSiz
             memmove(outDataCursor, srcDataCursor, blocks[i].dataSize);
         }
     }
-    else if (fileFormatInfo->compressionType == XEX_COMPRESSION_NORMAL || fileFormatInfo->compressionType == XEX_COMPRESSION_DELTA)
+    else if (fileFormatInfo->compressionType == XEX_COMPRESSION_NORMAL)
+    {
+        const Xex2CompressedBlockInfo* blocks = &((const Xex2FileNormalCompressionInfo*)(fileFormatInfo + 1))->firstBlock;
+        const uint32_t exeLength = xexBytesSize - xexHeader->headerSize.get();
+        const uint8_t* exeBuffer = &outBytes[headerTargetSize];
+
+        auto compressBuffer = std::make_unique<uint8_t[]>(exeLength);
+        const uint8_t* p = NULL;
+        uint8_t* d = NULL;
+        sha1::SHA1 s;
+
+        p = exeBuffer;
+        d = compressBuffer.get();
+
+        uint8_t blockCalcedDigest[0x14];
+        while (blocks->blockSize) 
+        {
+            const uint8_t* pNext = p + blocks->blockSize;
+            const auto* nextBlock = (const Xex2CompressedBlockInfo*)p;
+
+            s.reset();
+            s.processBytes(p, blocks->blockSize);
+            s.finalize(blockCalcedDigest);
+
+            if (memcmp(blockCalcedDigest, blocks->blockHash, 0x14) != 0)
+                return Result::PatchFailed;
+
+            p += 4;
+            p += 20;
+
+            while (true) 
+            {
+                const size_t chunkSize = (p[0] << 8) | p[1];
+                p += 2;
+
+                if (!chunkSize)
+                    break;
+
+                memcpy(d, p, chunkSize);
+                p += chunkSize;
+                d += chunkSize;
+            }
+
+            p = pNext;
+            blocks = nextBlock;
+        }
+
+        int resultCode = 0;
+        uint32_t uncompressedSize = originalSecurityInfo->imageSize;
+        uint8_t* buffer = outBytes.data() + newXexHeaderSize;
+
+        resultCode = lzxDecompress(compressBuffer.get(), d - compressBuffer.get(), buffer, uncompressedSize, ((const Xex2FileNormalCompressionInfo*)(fileFormatInfo + 1))->windowSize, nullptr, 0);
+
+        if (resultCode)
+            return Result::PatchFailed;
+    }
+    else if (fileFormatInfo->compressionType == XEX_COMPRESSION_DELTA)
     {
         return Result::XexFileUnsupported;
     }

XenonUtils/xex_patcher.h

@@ -16,6 +16,8 @@
 #include <span>
 #include <vector>
 
+extern int lzxDecompress(const void* lzxData, size_t lzxLength, void* dst, size_t dstLength, uint32_t windowSize, void* windowData, size_t windowDataLength);
+
 struct XexPatcher
 {
     enum class Result {