commit cc2b8afa33b2c202ee71170206e64a768e6c9c0a Author: Legonzaur <34353603+Legonzaur@users.noreply.github.com> Date: Mon Jul 17 15:43:07 2023 +0200 Add files via upload diff --git a/progressDisplay.py b/progressDisplay.py new file mode 100644 index 0000000..bc74e7a --- /dev/null +++ b/progressDisplay.py @@ -0,0 +1,40 @@ +# credits to WinEunuuchs2Unix +# https://stackoverflow.com/a/70586588 + +def percent_complete(step, total_steps, bar_width=60, title="", print_perc=True): + import sys + + # UTF-8 left blocks: 1, 1/8, 1/4, 3/8, 1/2, 5/8, 3/4, 7/8 + utf_8s = ["█", "▏", "▎", "▍", "▌", "▋", "▊", "█"] + perc = 100 * float(step) / float(total_steps) + max_ticks = bar_width * 8 + num_ticks = int(round(perc / 100 * max_ticks)) + full_ticks = num_ticks / 8 # Number of full blocks + part_ticks = num_ticks % 8 # Size of partial block (array index) + + disp = bar = "" # Blank out variables + bar += utf_8s[0] * int(full_ticks) # Add full blocks into Progress Bar + + # If part_ticks is zero, then no partial block, else append part char + if part_ticks > 0: + bar += utf_8s[part_ticks] + + # Pad Progress Bar with fill character + bar += "▒" * int((max_ticks/8 - float(num_ticks)/8.0)) + + if len(title) > 0: + disp = title + ": " # Optional title to progress display + + # Print progress bar in green: https://stackoverflow.com/a/21786287/6929343 + disp += "\x1b[0;32m" # Color Green + disp += bar # Progress bar to progress display + disp += "\x1b[0m" # Color Reset + if print_perc: + # If requested, append percentage complete to progress display + if perc > 100.0: + perc = 100.0 # Fix "100.04 %" rounding error + disp += " {:6.2f}".format(perc) + " %" + + # Output to terminal repetitively over the same line using '\r'. + sys.stdout.write("\r" + disp) + sys.stdout.flush() \ No newline at end of file diff --git a/rpak.py b/rpak.py new file mode 100644 index 0000000..197ac05 --- /dev/null +++ b/rpak.py @@ -0,0 +1,336 @@ +import struct +from enum import Enum +from progressDisplay import percent_complete +from texFormat import DDS + +virtualSegmentSize = 0x10 +pageInfoSize = 0xC +descriptorSize = 0x8 +assetEntrySize = 0x48 +guidDescriptorSize = 0x8 +fileRelationSize = 0x4 +externalAssetSize = 0x4 + +def getStringFromPage(bytes:bytes, offset:int): + endOfString = False + string = '' + while(endOfString == False): + data = bytes[offset:offset+1] + if(data == b'\x00'): endOfString = True + string += data.decode() + offset += 1 + return string[:-1] + +class ShaderTypes(Enum): + PIXEL = 0 + VERTEX = 1 + GEOMETRY = 2 + HARDWARE = 3 + DOMAIN = 4 + COMPUTE = 5 + +# Rpak +class Descriptor: + def __init__(self, data:bytes): + self.index = int.from_bytes(data[0:4], byteorder='little') + self.offset = int.from_bytes(data[4:8], byteorder='little') + +class PageInfo: + def __init__(self, data:bytes, offset:int = 0): + self.segmentIndex = int.from_bytes(data[0x0:0x4], byteorder='little') + self.align = int.from_bytes(data[0x4:0x8], byteorder='little') + self.dataSize = int.from_bytes(data[0x8:0xC], byteorder='little') + self.offset = offset + self.bytes = data + +class AssetEntry: + def __init__(self, data:bytes): + self.nameHash = int.from_bytes(data[0:8], byteorder='little') + self.padding = int.from_bytes(data[8:0x10], byteorder='little') + self.SubHeaderPageIndex = int.from_bytes(data[0x10:0x14], byteorder='little') + self.SubHeaderPageOffset = int.from_bytes(data[0x14:0x18], byteorder='little', signed=True) + self.RawDataPageIndex = int.from_bytes(data[0x18:0x1C], byteorder='little', signed=True) + self.RawDataPageOffset = int.from_bytes(data[0x1C:0x20], byteorder='little', signed=True) + self.StarpakOffset = int.from_bytes(data[0x20:0x28], byteorder='little', signed=True) + self.HighestPageNum = int.from_bytes(data[0x28:0x2A], byteorder='little') + # self.Un2 = int.from_bytes(data[0x2A:0x2C], byteorder='little') + self.RelationsStartIndex = int.from_bytes(data[0x2C:0x30], byteorder='little') + self.UsesStartIndex = int.from_bytes(data[0x30:0x34], byteorder='little') + self.RelationsCount = int.from_bytes(data[0x34:0x38], byteorder='little') + self.UsesCount = int.from_bytes(data[0x38:0x3A], byteorder='little') + # self.unk = int.from_bytes(data[0x3A:0x3C], byteorder='little') + self.SubHeaderSize = int.from_bytes(data[0x3C:0x40], byteorder='little') + self.Version = int.from_bytes(data[0x40:0x44], byteorder='little') + self.Magic = data[0x44:0x48].decode() +# Pages +# Assets +class ShaderHeader: + def __init__(self, data:bytes, pages: list[bytes]): + pNameIndex = int.from_bytes(data[0:4], byteorder='little') + pNameOffset = int.from_bytes(data[4:8], byteorder='little') + self.name = getStringFromPage(pages[pNameIndex], pNameOffset) + self.ShaderType = ShaderTypes(int.from_bytes(data[8:9], byteorder='little')) + self.padd = struct.unpack('ccc',data[0x9:0xC]) + self.dataCount = int.from_bytes(data[0xC:0x10], byteorder='little') + # 16 bytes unknown data + +class ShaderSetHeader: + def __init__(self, data:bytes, pages: list[bytes]): + # self.reservedVtbl = int.from_bytes(data[0:8], byteorder='little') + pNameIndex = int.from_bytes(data[8:0xC], byteorder='little') + pNameOffset = int.from_bytes(data[0xC:0x10], byteorder='little') + self.name = getStringFromPage(pages[pNameIndex], pNameOffset) + # 8 bytes unknown + # self.count1 = int.from_bytes(data[0x18:0x1A], byteorder='little') + self.textureInputCount = int.from_bytes(data[0x1A:0x1C], byteorder='little') + # self.count3 = int.from_bytes(data[0x1C:0x1E], byteorder='little') + # self.byte1 = int.from_bytes(data[0x1E:0x1F], byteorder='little') + # self.byte2 = int.from_bytes(data[0x1F:0x20], byteorder='little') + # 8 bytes unknown + # self.unk1 = int.from_bytes(data[0x28:0x30], byteorder='little') + # self.unk2 = int.from_bytes(data[0x30:0x38], byteorder='little') + # self.unk3 = int.from_bytes(data[0x38:0x40], byteorder='little') + # self.unk4 = int.from_bytes(data[0x40:0x48], byteorder='little') + self.vertexShaderGuid = int.from_bytes(data[0x48:0x50], byteorder='little') + self.pixelShaderGuid = int.from_bytes(data[0x50:0x58], byteorder='little') + +class MaterialHeader: + def __init__(self, data:bytes, pages:list[bytes]): + # self.reservedVtbl = int.from_bytes(data[0:8], byteorder='little') + # self.padding = int.from_bytes(data[8:0x10], byteorder='little') + self.guid = int.from_bytes(data[0x10:0x18], byteorder='little') + pNameIndex = int.from_bytes(data[0x18:0x1C], byteorder='little') + pNameOffset = int.from_bytes(data[0x1C:0x20], byteorder='little') + self.name = getStringFromPage(pages[pNameIndex], pNameOffset) + self.pSurfacePropIndex = int.from_bytes(data[0x20:0x24], byteorder='little') + self.pSurfacePropOffset = int.from_bytes(data[0x24:0x28], byteorder='little') + self.pSurfaceProp2Index = int.from_bytes(data[0x28:0x2C], byteorder='little') + self.pSurfaceProp2Offset = int.from_bytes(data[0x2C:0x30], byteorder='little') + self.guidRef1 = int.from_bytes(data[0x30:0x38], byteorder='little') + self.guidRef2 = int.from_bytes(data[0x38:0x40], byteorder='little') + self.guidRef3 = int.from_bytes(data[0x40:0x48], byteorder='little') + self.guidRef4 = int.from_bytes(data[0x48:0x50], byteorder='little') + # 40 bytes unknownMaterialSectionV12 + self.shadersetGuid = int.from_bytes(data[0x90:0x98], byteorder='little') + self.pTextureHandlesIndex = int.from_bytes(data[0x98:0x9C], byteorder='little') + self.pTextureHandlesOffset = int.from_bytes(data[0x9C:0xA0], byteorder='little') + self.pStreamingTextureHandlesIndex = int.from_bytes(data[0xA0:0xA4], byteorder='little') + self.pStreamingTextureHandlesOffset = int.from_bytes(data[0xA4:0xA8], byteorder='little') + self.streamingTextureCount = int.from_bytes(data[0xA8:0xAA], byteorder='little') + # self.unknown = int.from_bytes(data[0xAA:0xAE], byteorder='little') + self.flags = int.from_bytes(data[0xAE:0xB0], byteorder='little') + # self.unknown = int.from_bytes(data[0xB0:0xB2], byteorder='little') + # self.reserved = int.from_bytes(data[0xB2:0xBA], byteorder='little') + # self.unknown = int.from_bytes(data[0xBA:0xBE], byteorder='little') + # self.unknown = int.from_bytes(data[0xBE:0xC2], byteorder='little') + self.flags2 = int.from_bytes(data[0xC2:0xCA], byteorder='little') + self.width = int.from_bytes(data[0xCA:0xCC], byteorder='little') + self.height = int.from_bytes(data[0xCC:0xCE], byteorder='little') + # self.unknown = int.from_bytes(data[0xCE:0xD0], byteorder='little') + +class TextureHeader: + def __init__(self, data:bytes, pages:list[bytes]): + self.guid = int.from_bytes(data[0:8], byteorder='little') + pNameIndex = int.from_bytes(data[0x8:0xC], byteorder='little') + pNameOffset = int.from_bytes(data[0xC:0x10], byteorder='little') + self.name = getStringFromPage(pages[pNameIndex], pNameOffset) + self.width = int.from_bytes(data[0x10:0x12], byteorder='little') + self.height = int.from_bytes(data[0x12:0x14], byteorder='little') + self.depth = int.from_bytes(data[0x14:0x16], byteorder='little') + self.format = int.from_bytes(data[0x16:0x18], byteorder='little') + self.dataSize = int.from_bytes(data[0x18:0x1C], byteorder='little') + self.compressionType = int.from_bytes(data[0x1C:0x1D], byteorder='little') + self.optStreamedMipCount = int.from_bytes(data[0x1D:0x1E], byteorder='little') + self.arraySize = int.from_bytes(data[0x1E:0x1F], byteorder='little') + self.layerCount = int.from_bytes(data[0x1F:0x20], byteorder='little') + self.mipFlags = int.from_bytes(data[0x20:0x21], byteorder='little') + self.permanentMipCount = int.from_bytes(data[0x21:0x22], byteorder='little') + self.streamedMipCount = int.from_bytes(data[0x22:0x23], byteorder='little') + # self.unknown = int.from_bytes(data[0x22:0x30], byteorder='little') + self.numPixels = int.from_bytes(data[0x30:0x38], byteorder='little') + +class UIMGHeader: + def __init__(self, data:bytes, pages:list[bytes]): + self.widthRatio = struct.unpack('f',data[0:4]) + self.heightRatio = struct.unpack('f',data[4:8]) + self.width = int.from_bytes(data[0x8:0xA], byteorder='little') + self.height = int.from_bytes(data[0xA:0xC], byteorder='little') + self.textureOffsetsCount = int.from_bytes(data[0xC:0xE], byteorder='little') + self.textureCount = int.from_bytes(data[0xE:0x10], byteorder='little') + self.textureOffsetsIndex = int.from_bytes(data[0x10:0x14], byteorder='little') + self.textureOffsetsOffset = int.from_bytes(data[0x14:0x18], byteorder='little') + self.textureOffsetsDimsIndex = int.from_bytes(data[0x18:0x1C], byteorder='little') + self.textureOffsetsDimsOffset = int.from_bytes(data[0x1C:0x20], byteorder='little') + # self.unknown = int.from_bytes(data[0x20:0x24], byteorder='little') + # self.unknown = int.from_bytes(data[0x24:0x28], byteorder='little') + self.textureHashesIndex = int.from_bytes(data[0x28:0x2C], byteorder='little') + self.textureHashesOffset = int.from_bytes(data[0x2C:0x30], byteorder='little') + self.textureNamesIndex = int.from_bytes(data[0x30:0x34], byteorder='little') + self.textureNamesOffset = int.from_bytes(data[0x34:0x38], byteorder='little') + self.textureGuid = int.from_bytes(data[0x38:0x40], byteorder='little') + +class Texture: + def __init__(self, assetEntry:AssetEntry, header:TextureHeader, pages:list[bytes]): + self.assetEntry = assetEntry + self.header = header + dataPageIndex = assetEntry.RawDataPageIndex + dataStart = assetEntry.RawDataPageOffset + dataBytes = pages[dataPageIndex][dataStart:dataStart+header.dataSize] + self.dds = DDS(dataBytes, header.height, header.width) + if(header.name.endswith('leaves_ground_spread_01_col')): + f = open("test.dds", "wb") + f.write(self.dds.data) + f.close() + + print() + +class Material: + def __init__(self, assetEntry:AssetEntry, header:MaterialHeader, pages:list[bytes]): + self.assetEntry = assetEntry + self.header = header + dataPageIndex = header.pTextureHandlesIndex + dataStart = header.pTextureHandlesOffset + dataBytes = pages[dataPageIndex][dataStart:] + textureCount = int((header.pStreamingTextureHandlesOffset-header.pTextureHandlesOffset) / 8) + # Streamed textures are ignored + self.texturesGuid = [] + self.textures: list[Texture] = [] + for i in range(0,textureCount): + textureGuid = int.from_bytes(dataBytes[(i*8):(i*8)+8], byteorder='little') + self.texturesGuid.append(textureGuid) + self.surfaceProp = getStringFromPage(dataBytes, (textureCount * 8)*2) + + + def populateTextures(self, texturesByHash: dict[str, Texture]): + self.textures.clear() + + for tex in self.texturesGuid: + if(tex == 0): + self.textures.append(None) + continue + if((tex in texturesByHash) == False): + self.textures.append(None) + print("Texture not found : " + str(hex(tex))) + continue + texture = texturesByHash[tex] + self.textures.append(texture) + +class Shader: + def __init__(self, assetEntry:AssetEntry, header:ShaderHeader): + self.assetEntry = assetEntry + self.header = header + +class ShaderSet: + def __init__(self, assetEntry:AssetEntry, header:ShaderSetHeader): + self.assetEntry = assetEntry + self.header = header + +class UIMG: + def __init__(self, assetEntry:AssetEntry, header:UIMGHeader): + self.assetEntry = assetEntry + self.header = header + + +class Rpak: + def __init__(self, file): + with open(file, mode='rb') as file: + fileContent = file.read() + counts = struct.unpack('IIIIII',fileContent[0x40:0x58]) + + self.header = { + 'rpak_version' :int.from_bytes(fileContent[4:6], byteorder='little'), + 'flags':fileContent[6:7].decode(), + 'isCompressed': struct.unpack('?',fileContent[7:8]), + 'lenStarpakPaths' : int.from_bytes(fileContent[0x38:0x3A], byteorder='little'), + 'virtualSegmentCount' : int.from_bytes(fileContent[0x3A:0x3C], byteorder='little'), + 'pageInfoCount' : int.from_bytes(fileContent[0x3C:0x3E], byteorder='little'), + # todo : patchIndex + 'descriptorCount' : counts[0], + 'assetEntryCount' : counts[1], + 'guidDescriptorCount' : counts[2], + 'fileRelationCount' : counts[3], + 'externalAssetCount' : counts[4], + 'externalAssetSize' : counts[5], + } + + pointer = 0x58 + self.starpakstr = fileContent[pointer:pointer+self.header['lenStarpakPaths']].decode() + pointer += 0x20 + self.VirtualSegment = fileContent[pointer:pointer+self.header['virtualSegmentCount'] * virtualSegmentSize] + pointer += self.header['virtualSegmentCount'] * virtualSegmentSize + + rpakPageInfoBytes = [fileContent[pointer:pointer+self.header['pageInfoCount'] * pageInfoSize][i:i+pageInfoSize] for i in range(0,self.header['pageInfoCount'] * pageInfoSize, pageInfoSize)] + self.PageInfo: list[PageInfo] = [] + rpakPageInfoOffset = 0 + for pageInfoBytes in rpakPageInfoBytes: + info = PageInfo(pageInfoBytes, rpakPageInfoOffset) + rpakPageInfoOffset += info.dataSize + self.PageInfo.append(info) + + pointer += self.header['pageInfoCount'] * pageInfoSize + + rpakDescriptorBytes = [fileContent[pointer:pointer+self.header['descriptorCount'] * descriptorSize][i:i+descriptorSize] for i in range(0,self.header['descriptorCount'] * descriptorSize, descriptorSize)] + self.Descriptor = list(map(Descriptor, rpakDescriptorBytes)) + pointer += self.header['descriptorCount'] * descriptorSize + + rpakAssetsBytes = [fileContent[pointer:pointer+self.header['assetEntryCount'] * assetEntrySize][i:i+assetEntrySize] for i in range(0,self.header['assetEntryCount'] * assetEntrySize, assetEntrySize)] + self.AssetEntries = list(map(AssetEntry, rpakAssetsBytes)) + pointer += self.header['assetEntryCount'] * assetEntrySize + + # todo : guid descriptor + pointer += self.header['guidDescriptorCount'] * guidDescriptorSize + # todo : file relation + pointer += self.header['fileRelationCount'] * fileRelationSize + + pointer += self.header['externalAssetCount'] * externalAssetSize + pointer += self.header['externalAssetSize'] + + self.Pages: list[bytes] = [] + for info in self.PageInfo: + self.Pages.append(fileContent[info.offset+pointer:info.offset+pointer+info.dataSize]) + + self.Assets = { + 'shaders':[], + 'shaderSets':[], + 'materials':[], + 'textures':[], + 'texturesByHash':{}, + 'uimgs':[] + } + length = self.AssetEntries.__len__() + for index, assetEntry in enumerate(self.AssetEntries): + pageIndex = assetEntry.SubHeaderPageIndex + headerStart = assetEntry.SubHeaderPageOffset + headerEnd = assetEntry.SubHeaderPageOffset + assetEntry.SubHeaderSize + headerBytes = self.Pages[pageIndex][headerStart:headerEnd] + if(assetEntry.Magic == 'shdr'): + header = ShaderHeader(headerBytes, self.Pages) + asset = Shader(assetEntry, header) + assetType = 'shaders' + elif(assetEntry.Magic == 'shds'): + header = ShaderSetHeader(headerBytes, self.Pages) + asset = ShaderSet(assetEntry, header) + assetType = 'shaderSets' + elif(assetEntry.Magic == 'matl'): + header = MaterialHeader(headerBytes, self.Pages) + asset = Material(assetEntry, header, self.Pages) + assetType = 'materials' + elif(assetEntry.Magic == 'txtr'): + header = TextureHeader(headerBytes, self.Pages) + asset = Texture(assetEntry, header, self.Pages) + assetType = 'textures' + self.Assets['texturesByHash'][header.guid] = asset + elif(assetEntry.Magic == 'uimg'): + header = UIMGHeader(headerBytes, self.Pages) + asset = UIMG(assetEntry, header) + assetType = 'uimgs' + else: + continue + self.Assets[assetType].append(asset) + percent_complete(index, length, title="Importing Assets : " + file.name) + percent_complete(length, length, title="Importing Assets : " + file.name) + print() + length = self.Assets['materials'].__len__() + # for index, material in enumerate(self.Assets['materials']): + # material.populateTextures(self.Assets['textures']) + # percent_complete(index, length, title="Linking textures") \ No newline at end of file diff --git a/rpaksLoader.py b/rpaksLoader.py new file mode 100644 index 0000000..4bb8e3d --- /dev/null +++ b/rpaksLoader.py @@ -0,0 +1,29 @@ +from rpak import Rpak +from progressDisplay import percent_complete +import os +from itertools import chain + + +path = "." + +rpaks: list[Rpak] = [] +for file in os.listdir(path): + if file.endswith(".rpak"): + rpaks.append(Rpak(file)) + +textures = map(lambda x : x.Assets['textures'], rpaks) +textures = list(chain.from_iterable(textures)) +textureGuids = map(lambda x : x.assetEntry.nameHash, textures) +textureDict = dict(zip(textureGuids, textures)) +materials = map(lambda x : x.Assets['materials'], rpaks) +materials = list(chain.from_iterable(materials)) + + +length = len(materials) +for index, material in enumerate(materials): + material.populateTextures(textureDict) + percent_complete(index, length, title="Linking textures") + + +test = list(filter(lambda x : x.header.name.endswith('00_gloss_flat_gls') == True, textures)) +test \ No newline at end of file diff --git a/texFormat.py b/texFormat.py new file mode 100644 index 0000000..081d33f --- /dev/null +++ b/texFormat.py @@ -0,0 +1,57 @@ +class DDS: + def __init__(self, data:bytes, height:int, width:int) -> None: + self.data = bytearray(b'\x44\x44\x53\x20') # DDS magic + # Header + # Size + self.data.extend((124).to_bytes(4, "little")) + #flags + self.data.extend(b'\x07\x10\x0A\x00') + #height + self.data.extend((height).to_bytes(4, "little")) + #width + self.data.extend((width).to_bytes(4, "little")) + #pitchOrLinearSize + self.data.extend((height*width).to_bytes(4, "little")) + #depth + self.data.extend((1).to_bytes(4, "little")) + #mipMapCount + self.data.extend((1).to_bytes(4, "little")) + #reserved + self.data.extend((0).to_bytes(0x2C, "little")) + + #dds pixel format + #size + self.data.extend((32).to_bytes(4, "little")) + #flags + self.data.extend((4).to_bytes(4, "little")) + #fourCC + self.data.extend((b'\x44\x58\x31\x30')) + #RGBBitCount + self.data.extend((0).to_bytes(4, "little")) + #masks + self.data.extend((0).to_bytes(4, "little")) + self.data.extend((0).to_bytes(4, "little")) + self.data.extend((0).to_bytes(4, "little")) + self.data.extend((0).to_bytes(4, "little")) + + #caps + self.data.extend((0x1000).to_bytes(4, "little")) + self.data.extend((0).to_bytes(4, "little")) + self.data.extend((0).to_bytes(4, "little")) + self.data.extend((0).to_bytes(4, "little")) + #reserved + self.data.extend((0).to_bytes(4, "little")) + + #dds Header DXT10 + #DXGI Format + self.data.extend((99).to_bytes(4, "little")) + #resourceDimension + self.data.extend((3).to_bytes(4, "little")) + #miscFlag + self.data.extend((0).to_bytes(4, "little")) + #arraySize + self.data.extend((1).to_bytes(4, "little")) + #miscFlag2 + self.data.extend((0).to_bytes(4, "little")) + self.data.extend(data) + pass \ No newline at end of file