Add files via upload

progressDisplay.py

@@ -0,0 +1,40 @@
+# credits to WinEunuuchs2Unix
+# https://stackoverflow.com/a/70586588
+
+def percent_complete(step, total_steps, bar_width=60, title="", print_perc=True):
+    import sys
+
+    # UTF-8 left blocks: 1, 1/8, 1/4, 3/8, 1/2, 5/8, 3/4, 7/8
+    utf_8s = ["█", "▏", "▎", "▍", "▌", "▋", "▊", "█"]
+    perc = 100 * float(step) / float(total_steps)
+    max_ticks = bar_width * 8
+    num_ticks = int(round(perc / 100 * max_ticks))
+    full_ticks = num_ticks / 8      # Number of full blocks
+    part_ticks = num_ticks % 8      # Size of partial block (array index)
+    
+    disp = bar = ""                 # Blank out variables
+    bar += utf_8s[0] * int(full_ticks)  # Add full blocks into Progress Bar
+    
+    # If part_ticks is zero, then no partial block, else append part char
+    if part_ticks > 0:
+        bar += utf_8s[part_ticks]
+    
+    # Pad Progress Bar with fill character
+    bar += "▒" * int((max_ticks/8 - float(num_ticks)/8.0))
+    
+    if len(title) > 0:
+        disp = title + ": "         # Optional title to progress display
+    
+    # Print progress bar in green: https://stackoverflow.com/a/21786287/6929343
+    disp += "\x1b[0;32m"            # Color Green
+    disp += bar                     # Progress bar to progress display
+    disp += "\x1b[0m"               # Color Reset
+    if print_perc:
+        # If requested, append percentage complete to progress display
+        if perc > 100.0:
+            perc = 100.0            # Fix "100.04 %" rounding error
+        disp += " {:6.2f}".format(perc) + " %"
+    
+    # Output to terminal repetitively over the same line using '\r'.
+    sys.stdout.write("\r" + disp)
+    sys.stdout.flush()

rpak.py

@@ -0,0 +1,336 @@
+import struct
+from enum import Enum
+from progressDisplay import percent_complete
+from texFormat import DDS
+
+virtualSegmentSize = 0x10
+pageInfoSize = 0xC
+descriptorSize = 0x8
+assetEntrySize = 0x48
+guidDescriptorSize = 0x8
+fileRelationSize = 0x4
+externalAssetSize = 0x4
+
+def getStringFromPage(bytes:bytes, offset:int):
+    endOfString = False
+    string = ''
+    while(endOfString == False):
+        data = bytes[offset:offset+1]
+        if(data == b'\x00'): endOfString = True
+        string += data.decode()
+        offset += 1
+    return string[:-1]
+        
+class ShaderTypes(Enum):
+     PIXEL = 0
+     VERTEX = 1
+     GEOMETRY = 2
+     HARDWARE = 3
+     DOMAIN = 4
+     COMPUTE = 5 
+
+# Rpak
+class Descriptor:
+     def __init__(self, data:bytes):
+          self.index = int.from_bytes(data[0:4], byteorder='little')
+          self.offset = int.from_bytes(data[4:8], byteorder='little')
+
+class PageInfo:
+     def __init__(self, data:bytes, offset:int = 0):
+          self.segmentIndex = int.from_bytes(data[0x0:0x4], byteorder='little')
+          self.align = int.from_bytes(data[0x4:0x8], byteorder='little')
+          self.dataSize = int.from_bytes(data[0x8:0xC], byteorder='little')
+          self.offset = offset
+          self.bytes = data
+
+class AssetEntry:
+     def __init__(self, data:bytes):
+          self.nameHash = int.from_bytes(data[0:8], byteorder='little')
+          self.padding = int.from_bytes(data[8:0x10], byteorder='little')
+          self.SubHeaderPageIndex = int.from_bytes(data[0x10:0x14], byteorder='little')
+          self.SubHeaderPageOffset = int.from_bytes(data[0x14:0x18], byteorder='little', signed=True)
+          self.RawDataPageIndex = int.from_bytes(data[0x18:0x1C], byteorder='little', signed=True)
+          self.RawDataPageOffset = int.from_bytes(data[0x1C:0x20], byteorder='little', signed=True)
+          self.StarpakOffset = int.from_bytes(data[0x20:0x28], byteorder='little', signed=True)
+          self.HighestPageNum = int.from_bytes(data[0x28:0x2A], byteorder='little')
+        #   self.Un2 = int.from_bytes(data[0x2A:0x2C], byteorder='little')
+          self.RelationsStartIndex = int.from_bytes(data[0x2C:0x30], byteorder='little')
+          self.UsesStartIndex = int.from_bytes(data[0x30:0x34], byteorder='little')
+          self.RelationsCount = int.from_bytes(data[0x34:0x38], byteorder='little')
+          self.UsesCount = int.from_bytes(data[0x38:0x3A], byteorder='little')
+        #   self.unk = int.from_bytes(data[0x3A:0x3C], byteorder='little')
+          self.SubHeaderSize = int.from_bytes(data[0x3C:0x40], byteorder='little')
+          self.Version = int.from_bytes(data[0x40:0x44], byteorder='little')
+          self.Magic = data[0x44:0x48].decode()
+# Pages
+# Assets
+class ShaderHeader:
+    def __init__(self, data:bytes, pages: list[bytes]):
+        pNameIndex = int.from_bytes(data[0:4], byteorder='little')
+        pNameOffset = int.from_bytes(data[4:8], byteorder='little')
+        self.name = getStringFromPage(pages[pNameIndex], pNameOffset)
+        self.ShaderType = ShaderTypes(int.from_bytes(data[8:9], byteorder='little'))
+        self.padd = struct.unpack('ccc',data[0x9:0xC])
+        self.dataCount = int.from_bytes(data[0xC:0x10], byteorder='little')
+        # 16 bytes unknown data
+
+class ShaderSetHeader:
+    def __init__(self, data:bytes, pages: list[bytes]):
+        # self.reservedVtbl = int.from_bytes(data[0:8], byteorder='little')
+        pNameIndex = int.from_bytes(data[8:0xC], byteorder='little')
+        pNameOffset = int.from_bytes(data[0xC:0x10], byteorder='little')
+        self.name = getStringFromPage(pages[pNameIndex], pNameOffset)
+        # 8 bytes unknown
+        # self.count1 = int.from_bytes(data[0x18:0x1A], byteorder='little')
+        self.textureInputCount = int.from_bytes(data[0x1A:0x1C], byteorder='little')
+        # self.count3 = int.from_bytes(data[0x1C:0x1E], byteorder='little')
+        # self.byte1 = int.from_bytes(data[0x1E:0x1F], byteorder='little')
+        # self.byte2 = int.from_bytes(data[0x1F:0x20], byteorder='little')
+        # 8 bytes unknown
+        # self.unk1 = int.from_bytes(data[0x28:0x30], byteorder='little')
+        # self.unk2 = int.from_bytes(data[0x30:0x38], byteorder='little')
+        # self.unk3 = int.from_bytes(data[0x38:0x40], byteorder='little')
+        # self.unk4 = int.from_bytes(data[0x40:0x48], byteorder='little')
+        self.vertexShaderGuid = int.from_bytes(data[0x48:0x50], byteorder='little')
+        self.pixelShaderGuid = int.from_bytes(data[0x50:0x58], byteorder='little')
+
+class MaterialHeader:
+     def __init__(self, data:bytes, pages:list[bytes]):
+        # self.reservedVtbl = int.from_bytes(data[0:8], byteorder='little')
+        # self.padding = int.from_bytes(data[8:0x10], byteorder='little')
+        self.guid = int.from_bytes(data[0x10:0x18], byteorder='little')
+        pNameIndex = int.from_bytes(data[0x18:0x1C], byteorder='little')
+        pNameOffset = int.from_bytes(data[0x1C:0x20], byteorder='little')
+        self.name = getStringFromPage(pages[pNameIndex], pNameOffset)
+        self.pSurfacePropIndex = int.from_bytes(data[0x20:0x24], byteorder='little')
+        self.pSurfacePropOffset = int.from_bytes(data[0x24:0x28], byteorder='little')
+        self.pSurfaceProp2Index = int.from_bytes(data[0x28:0x2C], byteorder='little')
+        self.pSurfaceProp2Offset = int.from_bytes(data[0x2C:0x30], byteorder='little')
+        self.guidRef1 = int.from_bytes(data[0x30:0x38], byteorder='little')
+        self.guidRef2 = int.from_bytes(data[0x38:0x40], byteorder='little')
+        self.guidRef3 = int.from_bytes(data[0x40:0x48], byteorder='little')
+        self.guidRef4 = int.from_bytes(data[0x48:0x50], byteorder='little')
+        # 40 bytes unknownMaterialSectionV12
+        self.shadersetGuid = int.from_bytes(data[0x90:0x98], byteorder='little')
+        self.pTextureHandlesIndex = int.from_bytes(data[0x98:0x9C], byteorder='little')
+        self.pTextureHandlesOffset = int.from_bytes(data[0x9C:0xA0], byteorder='little')
+        self.pStreamingTextureHandlesIndex = int.from_bytes(data[0xA0:0xA4], byteorder='little')
+        self.pStreamingTextureHandlesOffset = int.from_bytes(data[0xA4:0xA8], byteorder='little')
+        self.streamingTextureCount = int.from_bytes(data[0xA8:0xAA], byteorder='little')
+        # self.unknown = int.from_bytes(data[0xAA:0xAE], byteorder='little')
+        self.flags = int.from_bytes(data[0xAE:0xB0], byteorder='little')
+        # self.unknown = int.from_bytes(data[0xB0:0xB2], byteorder='little')
+        # self.reserved = int.from_bytes(data[0xB2:0xBA], byteorder='little')
+        # self.unknown = int.from_bytes(data[0xBA:0xBE], byteorder='little')
+        # self.unknown = int.from_bytes(data[0xBE:0xC2], byteorder='little')
+        self.flags2 = int.from_bytes(data[0xC2:0xCA], byteorder='little')
+        self.width = int.from_bytes(data[0xCA:0xCC], byteorder='little')
+        self.height = int.from_bytes(data[0xCC:0xCE], byteorder='little')
+        # self.unknown = int.from_bytes(data[0xCE:0xD0], byteorder='little')
+
+class TextureHeader:
+     def __init__(self, data:bytes, pages:list[bytes]):
+        self.guid = int.from_bytes(data[0:8], byteorder='little')
+        pNameIndex = int.from_bytes(data[0x8:0xC], byteorder='little')
+        pNameOffset = int.from_bytes(data[0xC:0x10], byteorder='little')
+        self.name = getStringFromPage(pages[pNameIndex], pNameOffset)
+        self.width = int.from_bytes(data[0x10:0x12], byteorder='little')
+        self.height = int.from_bytes(data[0x12:0x14], byteorder='little')
+        self.depth = int.from_bytes(data[0x14:0x16], byteorder='little')
+        self.format = int.from_bytes(data[0x16:0x18], byteorder='little')
+        self.dataSize = int.from_bytes(data[0x18:0x1C], byteorder='little')
+        self.compressionType = int.from_bytes(data[0x1C:0x1D], byteorder='little')
+        self.optStreamedMipCount = int.from_bytes(data[0x1D:0x1E], byteorder='little')
+        self.arraySize = int.from_bytes(data[0x1E:0x1F], byteorder='little')
+        self.layerCount = int.from_bytes(data[0x1F:0x20], byteorder='little')
+        self.mipFlags = int.from_bytes(data[0x20:0x21], byteorder='little')
+        self.permanentMipCount = int.from_bytes(data[0x21:0x22], byteorder='little')
+        self.streamedMipCount = int.from_bytes(data[0x22:0x23], byteorder='little')
+        # self.unknown = int.from_bytes(data[0x22:0x30], byteorder='little')
+        self.numPixels = int.from_bytes(data[0x30:0x38], byteorder='little')
+
+class UIMGHeader:
+     def __init__(self, data:bytes, pages:list[bytes]):
+        self.widthRatio = struct.unpack('f',data[0:4])
+        self.heightRatio = struct.unpack('f',data[4:8])
+        self.width = int.from_bytes(data[0x8:0xA], byteorder='little')
+        self.height = int.from_bytes(data[0xA:0xC], byteorder='little')
+        self.textureOffsetsCount = int.from_bytes(data[0xC:0xE], byteorder='little')
+        self.textureCount = int.from_bytes(data[0xE:0x10], byteorder='little')
+        self.textureOffsetsIndex = int.from_bytes(data[0x10:0x14], byteorder='little')
+        self.textureOffsetsOffset = int.from_bytes(data[0x14:0x18], byteorder='little')
+        self.textureOffsetsDimsIndex = int.from_bytes(data[0x18:0x1C], byteorder='little')
+        self.textureOffsetsDimsOffset = int.from_bytes(data[0x1C:0x20], byteorder='little')
+        # self.unknown = int.from_bytes(data[0x20:0x24], byteorder='little')
+        # self.unknown = int.from_bytes(data[0x24:0x28], byteorder='little')
+        self.textureHashesIndex = int.from_bytes(data[0x28:0x2C], byteorder='little')
+        self.textureHashesOffset = int.from_bytes(data[0x2C:0x30], byteorder='little')
+        self.textureNamesIndex = int.from_bytes(data[0x30:0x34], byteorder='little')
+        self.textureNamesOffset = int.from_bytes(data[0x34:0x38], byteorder='little')
+        self.textureGuid = int.from_bytes(data[0x38:0x40], byteorder='little')
+
+class Texture:
+    def __init__(self, assetEntry:AssetEntry, header:TextureHeader, pages:list[bytes]):
+        self.assetEntry = assetEntry
+        self.header = header
+        dataPageIndex = assetEntry.RawDataPageIndex
+        dataStart = assetEntry.RawDataPageOffset
+        dataBytes = pages[dataPageIndex][dataStart:dataStart+header.dataSize]
+        self.dds = DDS(dataBytes, header.height, header.width)
+        if(header.name.endswith('leaves_ground_spread_01_col')):
+            f = open("test.dds", "wb")
+            f.write(self.dds.data)
+            f.close()
+            
+        print()
+
+class Material:
+    def __init__(self, assetEntry:AssetEntry, header:MaterialHeader, pages:list[bytes]):
+        self.assetEntry = assetEntry
+        self.header = header
+        dataPageIndex = header.pTextureHandlesIndex
+        dataStart = header.pTextureHandlesOffset
+        dataBytes = pages[dataPageIndex][dataStart:]
+        textureCount = int((header.pStreamingTextureHandlesOffset-header.pTextureHandlesOffset) / 8)
+        # Streamed textures are ignored
+        self.texturesGuid = []
+        self.textures: list[Texture] = []
+        for i in range(0,textureCount):
+            textureGuid = int.from_bytes(dataBytes[(i*8):(i*8)+8], byteorder='little')
+            self.texturesGuid.append(textureGuid)
+        self.surfaceProp = getStringFromPage(dataBytes, (textureCount * 8)*2)
+        
+
+    def populateTextures(self, texturesByHash: dict[str, Texture]):
+        self.textures.clear()
+
+        for tex in self.texturesGuid:
+            if(tex == 0): 
+                self.textures.append(None)
+                continue
+            if((tex in texturesByHash) == False): 
+                self.textures.append(None)
+                print("Texture not found : " + str(hex(tex)))
+                continue
+            texture = texturesByHash[tex]
+            self.textures.append(texture)
+
+class Shader:
+    def __init__(self, assetEntry:AssetEntry, header:ShaderHeader):
+        self.assetEntry = assetEntry
+        self.header = header
+
+class ShaderSet:
+    def __init__(self, assetEntry:AssetEntry, header:ShaderSetHeader):
+        self.assetEntry = assetEntry
+        self.header = header
+
+class UIMG:
+    def __init__(self, assetEntry:AssetEntry, header:UIMGHeader):
+        self.assetEntry = assetEntry
+        self.header = header
+
+
+class Rpak:
+    def __init__(self, file):
+        with open(file, mode='rb') as file:
+                fileContent = file.read()
+                counts = struct.unpack('IIIIII',fileContent[0x40:0x58])
+
+                self.header = {
+                    'rpak_version' :int.from_bytes(fileContent[4:6], byteorder='little'),
+                    'flags':fileContent[6:7].decode(),
+                    'isCompressed': struct.unpack('?',fileContent[7:8]), 
+                    'lenStarpakPaths' : int.from_bytes(fileContent[0x38:0x3A], byteorder='little'),
+                    'virtualSegmentCount' : int.from_bytes(fileContent[0x3A:0x3C], byteorder='little'),
+                    'pageInfoCount' : int.from_bytes(fileContent[0x3C:0x3E], byteorder='little'),
+                    # todo : patchIndex 
+                    'descriptorCount' : counts[0],
+                    'assetEntryCount' : counts[1],
+                    'guidDescriptorCount' : counts[2],
+                    'fileRelationCount' : counts[3],
+                    'externalAssetCount' : counts[4],
+                    'externalAssetSize' : counts[5],
+                }
+                
+                pointer = 0x58
+                self.starpakstr = fileContent[pointer:pointer+self.header['lenStarpakPaths']].decode()
+                pointer += 0x20
+                self.VirtualSegment = fileContent[pointer:pointer+self.header['virtualSegmentCount'] * virtualSegmentSize]
+                pointer += self.header['virtualSegmentCount'] * virtualSegmentSize
+
+                rpakPageInfoBytes = [fileContent[pointer:pointer+self.header['pageInfoCount'] * pageInfoSize][i:i+pageInfoSize] for i in range(0,self.header['pageInfoCount'] * pageInfoSize, pageInfoSize)]
+                self.PageInfo: list[PageInfo] = []
+                rpakPageInfoOffset = 0
+                for pageInfoBytes in rpakPageInfoBytes:
+                    info = PageInfo(pageInfoBytes, rpakPageInfoOffset)
+                    rpakPageInfoOffset += info.dataSize
+                    self.PageInfo.append(info)
+                
+                pointer += self.header['pageInfoCount'] * pageInfoSize
+
+                rpakDescriptorBytes = [fileContent[pointer:pointer+self.header['descriptorCount'] * descriptorSize][i:i+descriptorSize] for i in range(0,self.header['descriptorCount'] * descriptorSize, descriptorSize)]
+                self.Descriptor = list(map(Descriptor, rpakDescriptorBytes))
+                pointer += self.header['descriptorCount'] * descriptorSize
+
+                rpakAssetsBytes = [fileContent[pointer:pointer+self.header['assetEntryCount'] * assetEntrySize][i:i+assetEntrySize] for i in range(0,self.header['assetEntryCount'] * assetEntrySize, assetEntrySize)]
+                self.AssetEntries = list(map(AssetEntry, rpakAssetsBytes))
+                pointer += self.header['assetEntryCount'] * assetEntrySize
+
+                # todo : guid descriptor
+                pointer += self.header['guidDescriptorCount'] * guidDescriptorSize
+                # todo : file relation
+                pointer += self.header['fileRelationCount'] * fileRelationSize
+
+                pointer += self.header['externalAssetCount'] * externalAssetSize
+                pointer += self.header['externalAssetSize']
+
+                self.Pages: list[bytes] = []
+                for info in self.PageInfo:
+                     self.Pages.append(fileContent[info.offset+pointer:info.offset+pointer+info.dataSize])
+                
+                self.Assets = {
+                     'shaders':[],
+                     'shaderSets':[],
+                     'materials':[],
+                     'textures':[],
+                     'texturesByHash':{},
+                     'uimgs':[]
+                }
+                length = self.AssetEntries.__len__()
+                for index, assetEntry in enumerate(self.AssetEntries):
+                    pageIndex = assetEntry.SubHeaderPageIndex
+                    headerStart = assetEntry.SubHeaderPageOffset
+                    headerEnd = assetEntry.SubHeaderPageOffset + assetEntry.SubHeaderSize
+                    headerBytes = self.Pages[pageIndex][headerStart:headerEnd]
+                    if(assetEntry.Magic == 'shdr'):
+                        header = ShaderHeader(headerBytes, self.Pages)
+                        asset = Shader(assetEntry, header)
+                        assetType = 'shaders'
+                    elif(assetEntry.Magic == 'shds'):
+                        header = ShaderSetHeader(headerBytes, self.Pages)
+                        asset = ShaderSet(assetEntry, header)
+                        assetType = 'shaderSets'
+                    elif(assetEntry.Magic == 'matl'):
+                        header = MaterialHeader(headerBytes, self.Pages)
+                        asset = Material(assetEntry, header, self.Pages)
+                        assetType = 'materials'
+                    elif(assetEntry.Magic == 'txtr'):
+                        header = TextureHeader(headerBytes, self.Pages)
+                        asset = Texture(assetEntry, header, self.Pages)
+                        assetType = 'textures'
+                        self.Assets['texturesByHash'][header.guid] = asset
+                    elif(assetEntry.Magic == 'uimg'):
+                        header = UIMGHeader(headerBytes, self.Pages)
+                        asset = UIMG(assetEntry, header)
+                        assetType = 'uimgs'
+                    else:
+                        continue
+                    self.Assets[assetType].append(asset)
+                    percent_complete(index, length, title="Importing Assets : " + file.name)
+                percent_complete(length, length, title="Importing Assets : " + file.name)
+                print()
+                length = self.Assets['materials'].__len__()
+                # for index, material in enumerate(self.Assets['materials']):
+                #     material.populateTextures(self.Assets['textures'])
+                #     percent_complete(index, length, title="Linking textures")

rpaksLoader.py

@@ -0,0 +1,29 @@
+from rpak import Rpak
+from progressDisplay import percent_complete
+import os
+from itertools import chain
+
+
+path = "."
+
+rpaks: list[Rpak] = []
+for file in os.listdir(path):
+    if file.endswith(".rpak"):
+        rpaks.append(Rpak(file))
+
+textures = map(lambda x : x.Assets['textures'], rpaks)
+textures = list(chain.from_iterable(textures))
+textureGuids = map(lambda x : x.assetEntry.nameHash, textures)
+textureDict = dict(zip(textureGuids, textures))
+materials = map(lambda x : x.Assets['materials'], rpaks)
+materials = list(chain.from_iterable(materials))
+
+
+length = len(materials)
+for index, material in enumerate(materials):
+    material.populateTextures(textureDict)
+    percent_complete(index, length, title="Linking textures")
+
+
+test = list(filter(lambda x : x.header.name.endswith('00_gloss_flat_gls') == True, textures))
+test

texFormat.py

@@ -0,0 +1,57 @@
+class DDS:
+    def __init__(self, data:bytes, height:int, width:int) -> None:
+        self.data = bytearray(b'\x44\x44\x53\x20') # DDS magic
+        # Header
+        # Size
+        self.data.extend((124).to_bytes(4, "little")) 
+        #flags
+        self.data.extend(b'\x07\x10\x0A\x00') 
+        #height
+        self.data.extend((height).to_bytes(4, "little")) 
+        #width
+        self.data.extend((width).to_bytes(4, "little")) 
+        #pitchOrLinearSize
+        self.data.extend((height*width).to_bytes(4, "little")) 
+        #depth
+        self.data.extend((1).to_bytes(4, "little")) 
+        #mipMapCount
+        self.data.extend((1).to_bytes(4, "little")) 
+        #reserved
+        self.data.extend((0).to_bytes(0x2C, "little")) 
+
+        #dds pixel format
+        #size
+        self.data.extend((32).to_bytes(4, "little")) 
+        #flags
+        self.data.extend((4).to_bytes(4, "little")) 
+        #fourCC
+        self.data.extend((b'\x44\x58\x31\x30')) 
+        #RGBBitCount
+        self.data.extend((0).to_bytes(4, "little")) 
+        #masks
+        self.data.extend((0).to_bytes(4, "little")) 
+        self.data.extend((0).to_bytes(4, "little")) 
+        self.data.extend((0).to_bytes(4, "little")) 
+        self.data.extend((0).to_bytes(4, "little")) 
+
+        #caps
+        self.data.extend((0x1000).to_bytes(4, "little")) 
+        self.data.extend((0).to_bytes(4, "little")) 
+        self.data.extend((0).to_bytes(4, "little")) 
+        self.data.extend((0).to_bytes(4, "little")) 
+        #reserved
+        self.data.extend((0).to_bytes(4, "little")) 
+
+        #dds Header DXT10
+        #DXGI Format
+        self.data.extend((99).to_bytes(4, "little")) 
+        #resourceDimension
+        self.data.extend((3).to_bytes(4, "little")) 
+        #miscFlag
+        self.data.extend((0).to_bytes(4, "little")) 
+        #arraySize
+        self.data.extend((1).to_bytes(4, "little")) 
+        #miscFlag2
+        self.data.extend((0).to_bytes(4, "little")) 
+        self.data.extend(data)
+        pass