From c069300785fcd7b9a3cf0a3f51e244143db31cdf Mon Sep 17 00:00:00 2001 From: Roderick Kennedy Date: Mon, 18 Nov 2024 23:32:13 +0000 Subject: [PATCH] Sfx compiler - fix checks for unchanged shaders - correct sfxo is now generated. --- Applications/Sfx/Compiler.cpp | 194 +++++++++++++++++++++------------ Applications/Sfx/Main.cpp | 11 +- Applications/Sfx/Sfx.cpp | 7 ++ DirectX11/CreateEffectDX1x.cpp | 2 +- DirectX12/CMakeLists.txt | 1 + Vulkan/CMakeLists.txt | 1 + Vulkan/RenderPlatform.cpp | 10 ++ Vulkan/Texture.cpp | 26 +++-- 8 files changed, 167 insertions(+), 85 deletions(-) diff --git a/Applications/Sfx/Compiler.cpp b/Applications/Sfx/Compiler.cpp index 2a1cfb4b..353cdfa3 100644 --- a/Applications/Sfx/Compiler.cpp +++ b/Applications/Sfx/Compiler.cpp @@ -438,12 +438,10 @@ void ReplaceRegexes(string &src, const std::map &replace) src = std::regex_replace(src, param_re, i.second); } } +static std::map useCompilerPath; -wstring BuildCompileCommand(std::shared_ptr shaderInstance,const SfxConfig &sfxConfig,const SfxOptions &sfxOptions,wstring targetDir,wstring outputFile, - wstring tempFilename,ShaderType t, PixelOutputFormat pixelOutputFormat) +extern void CalcCompilerPathToUse(const SfxConfig &sfxConfig,const SfxOptions &sfxOptions) { - if (sfxConfig.compiler.empty()) - return L""; std::string usePath=""; std::string compiler_exe = sfxConfig.compiler; size_t space_pos=compiler_exe.find(" "); @@ -458,15 +456,31 @@ wstring BuildCompileCommand(std::shared_ptr shaderInstance,const } for(const auto &p:sfxConfig.compilerPaths) { - std::string t=p+"/"s+compiler_exe; - //std::cout << "Checking " << t.c_str() << std::endl; - if(fs::exists(t)) + std::filesystem::path pth(p); + pth=pth.append(compiler_exe); + pth=pth.lexically_normal(); + if(sfxOptions.verbose) + std::cout << "Checking: " << pth.generic_string().c_str() << std::endl; + if(fs::exists(pth)) { - usePath=p; + if(sfxOptions.verbose) + std::cout << "Using: " << pth.generic_string().c_str() << std::endl; + useCompilerPath[sfxConfig.api]=usePath=p; break; } } -// std::cout<< "Using path " << usePath.c_str() << std::endl; +} +wstring BuildCompileCommand(std::shared_ptr shaderInstance,const SfxConfig &sfxConfig,const SfxOptions &sfxOptions,wstring targetDir,wstring outputFile, + wstring generatedSourceFilename,ShaderType t, PixelOutputFormat pixelOutputFormat) +{ + if (sfxConfig.compiler.empty()) + return L""; + + if(useCompilerPath.find(sfxConfig.api)==useCompilerPath.end()) + { + CalcCompilerPathToUse(sfxConfig,sfxOptions); + } +// std::cout<< "Using path " << useCompilerPath.c_str() << std::endl; wstring command; string stageName = "NO_STAGES_IN_JSON"; @@ -481,8 +495,8 @@ wstring BuildCompileCommand(std::shared_ptr shaderInstance,const return L""; } std::string currentCompiler = FillInVariable(sfxConfig.compiler,"stage",stageName); - if(usePath.length()) - currentCompiler=usePath+"/"s+currentCompiler; + if(useCompilerPath[sfxConfig.api].length()) + currentCompiler=useCompilerPath[sfxConfig.api]+"/"s+currentCompiler; command += Utf8ToWString(currentCompiler) ; // Add additional options (requested from the XX.json) @@ -520,7 +534,7 @@ wstring BuildCompileCommand(std::shared_ptr shaderInstance,const command += Utf8ToWString(std::regex_replace(sfxConfig.entryPointOption, std::regex("\\{name\\}"), shaderInstance->entryPoint)) + L" "; } string filename_root=WStringToString(outputFile); - dot_pos = filename_root.rfind("."); + size_t dot_pos = filename_root.rfind("."); if(dot_pos shaderInstance,const } }*/ command += L"\""; - command += tempFilename.c_str(); + command += generatedSourceFilename.c_str(); command += L"\""; return command; @@ -644,6 +658,25 @@ bool RewriteOutput(const SfxConfig &sfxConfig return has_errors; } +bool IsShaderUnchanged(std::string sourceFilename,std::string outputFilename,const std::string &src) +{ + if(!std::filesystem::exists(outputFilename)) + return false; + if(!std::filesystem::exists(sourceFilename)) + return false; + std::filesystem::file_time_type binary_time = std::filesystem::last_write_time(outputFilename); + std::filesystem::file_time_type source_time = std::filesystem::last_write_time(sourceFilename); + if(binary_time<=source_time) + return false; + std::ostringstream sstr; + ifstream ifs(sourceFilename.c_str());; + sstr << ifs.rdbuf(); + if(src!=sstr.str()) + return false; + return true; +} + + int Compile(std::shared_ptr shaderInstance ,const string &sourceFile ,string targetFile @@ -785,11 +818,11 @@ int Compile(std::shared_ptr shaderInstance std::string rootSrc((std::istreambuf_iterator(rootSrcFile)), (std::istreambuf_iterator())); preamble += rootSrc; } - wstring tempFilename ; + wstring generatedSourceFilename ; if(sfxOptions.intermediateDirectory.length()) - tempFilename+= StringToWString(sfxOptions.intermediateDirectory+ "/"); + generatedSourceFilename+= StringToWString(sfxOptions.intermediateDirectory+ "/"); - tempFilename+=targetFilename+wstring(L".")+Utf8ToWString(sfxConfig.sourceExtension); + generatedSourceFilename+=targetFilename+wstring(L".")+Utf8ToWString(sfxConfig.sourceExtension); char buffer[_MAX_PATH]; string wd=""; @@ -799,7 +832,7 @@ int Compile(std::shared_ptr shaderInstance if(getcwd(buffer,_MAX_PATH)) #endif wd=string(buffer)+"/"; - string tempf=WStringToUtf8(tempFilename); + string tempf=WStringToUtf8(generatedSourceFilename); if(tempf.find(":")>=tempf.length()) tempf=wd+"/"+tempf; find_and_replace(tempf,"\\","/"); @@ -844,18 +877,28 @@ int Compile(std::shared_ptr shaderInstance targetDir+=L"/"; mkpath(targetDir); mkpath(StringToWString(sfxOptions.intermediateDirectory)+L"/"); + wstring outputFile = ((sfxOptions.wrapOutput ? StringToWString(sfxOptions.intermediateDirectory) : targetDir) + L"/") + (targetFilename + L".") + Utf8ToWString(sfxConfig.outputExtension); + // Now: if the source we want to save is the same as the file that's there, + // AND the date of the target file is AFTER the date of the source, that means + // that this shader hasn't changed, even if the sfx file has. + + // Therefore we don't compile it. + bool unchanged=IsShaderUnchanged(WStringToUtf8(generatedSourceFilename),WStringToUtf8(outputFile),src); + + if(!unchanged) + { #ifdef _MSC_VER - ofstream ofs(tempFilename.c_str()); + ofstream ofs(generatedSourceFilename.c_str()); #else - ofstream ofs(WStringToUtf8(tempFilename).c_str()); + ofstream ofs(WStringToUtf8(generatedSourceFilename).c_str()); #endif - ofs.write(strSrc, strlen(strSrc)); - ofs.close(); - + ofs.write(strSrc, strlen(strSrc)); + ofs.close(); + } #ifdef _MSC_VER - // Nowe delete the corresponding sdb's - wstring sdbFile=tempFilename.substr(0,tempFilename.length()-5); + // Now delete the corresponding sdb's + wstring sdbFile=generatedSourceFilename.substr(0,generatedSourceFilename.length()-5); WIN32_FIND_DATAW fd; HANDLE hFind = FindFirstFileW((sdbFile+L"*.sdb").c_str(), &fd); if (hFind != INVALID_HANDLE_VALUE) @@ -867,7 +910,6 @@ int Compile(std::shared_ptr shaderInstance FindClose(hFind); } #endif - wstring outputFile = ((sfxOptions.wrapOutput ? StringToWString(sfxOptions.intermediateDirectory) : targetDir) + L"/") + (targetFilename + L".") + Utf8ToWString(sfxConfig.outputExtension); // Add the output filename int slash = (int)outputFile.rfind(L"/"); int backslash = (int)outputFile.rfind(L"\\"); @@ -884,55 +926,60 @@ int Compile(std::shared_ptr shaderInstance } else shaderInstance->sbFilenames[0] = sbf; - // Get the compile command - wstring compile_command = BuildCompileCommand - ( - shaderInstance, - sfxConfig, - sfxOptions, - targetDir, - outputFile, - tempFilename, - t, - pixelOutputFormat - ); ostringstream log; - // If no compiler provided, we can return now (perhaps we are only interested in - // the shader source) - if (compile_command.empty()) + // Get the compile command + wstring compile_command; + + if(!unchanged) { - if(sfxOptions.verbose) - std::cout< shaderInstance int repetitions=0; while(!created_output) { - res=RunDOSCommand(compile_command.c_str(),wd,log,sfxConfig,cc); - if (res) + bool write_log=false; + if(!unchanged) { - bool write_log=false; + res=RunDOSCommand(compile_command.c_str(),wd,log,sfxConfig,cc); const string &log_str=log.str(); if (sfxOptions.verbose) { @@ -970,6 +1017,13 @@ int Compile(std::shared_ptr shaderInstance std::cerr << log.str() << std::endl; res = 0; } + } + else + { + res=true; + } + if (res) + { if (sfxOptions.wrapOutput) { //concatenate diff --git a/Applications/Sfx/Main.cpp b/Applications/Sfx/Main.cpp index bffbbe91..b3593fa0 100644 --- a/Applications/Sfx/Main.cpp +++ b/Applications/Sfx/Main.cpp @@ -204,6 +204,13 @@ int main(int argc, char** argv) //std::cerr< 0) diff --git a/Applications/Sfx/Sfx.cpp b/Applications/Sfx/Sfx.cpp index f80c50b1..ddb44d15 100644 --- a/Applications/Sfx/Sfx.cpp +++ b/Applications/Sfx/Sfx.cpp @@ -545,6 +545,13 @@ bool sfxParseEffectFromFile(int effect, const char *file, const std::vector0) { string outf=outputfile; + std::filesystem::path outputPath(outf); + try{ + std::filesystem::create_directories(outputPath); + }catch(...) + { + std::cerr<<"Could not create path "<Release(); missing=(newestFileTime==0&&hr!=S_OK); - return newestFileTime; + return (double)newestFileTime; } #define D3D10_SHADER_ENABLE_STRICTNESS (1 << 11) diff --git a/DirectX12/CMakeLists.txt b/DirectX12/CMakeLists.txt index a25be148..5a0ccb2a 100644 --- a/DirectX12/CMakeLists.txt +++ b/DirectX12/CMakeLists.txt @@ -65,6 +65,7 @@ set(SOURCES ${SOURCES} "${SIMUL_PLATFORM_DIR}/External/D3D12MemoryAllocator/src/ add_static_library( SimulDirectX12 SOURCES ${SOURCES} ${HEADERS} ${SHADER_DEFS} DEFINITIONS ${DX12_DEFS} PROPERTIES FOLDER ${SIMUL_PLATFORM_FOLDER_PREFIX} INCLUDES "${SIMUL_PLATFORM_DIR}/External/DirectX/DirectXTex/DirectXTex" PUBLICINCLUDES "${SIMUL_PLATFORM_DIR}/DirectX12") +add_dependencies(SimulDirectX12${STATIC_LINK_SUFFIX} SimulCrossPlatform${STATIC_LINK_SUFFIX}) # For D3D12MA if(SIMUL_SOURCE_BUILD) diff --git a/Vulkan/CMakeLists.txt b/Vulkan/CMakeLists.txt index c56a4a74..be2a1616 100644 --- a/Vulkan/CMakeLists.txt +++ b/Vulkan/CMakeLists.txt @@ -39,6 +39,7 @@ set(HEADERS ${HEADERS} "${SIMUL_PLATFORM_DIR}/External/VulkanMemoryAllocator/inc if(SIMUL_SOURCE_BUILD) add_static_library( SimulVulkan SOURCES ${SOURCES} ${HEADERS} ${SHADER_DEFS} ${SHADER_INCLUDES} DEFINITIONS ${VK_DEFS} FOLDER ${SIMUL_PLATFORM_FOLDER_PREFIX}) + add_dependencies(SimulVulkan${STATIC_LINK_SUFFIX} SimulCrossPlatform${STATIC_LINK_SUFFIX}) if(PLATFORM_WINDOWS) target_compile_definitions(SimulVulkan${STATIC_LINK_SUFFIX} PRIVATE VK_USE_PLATFORM_WIN32_KHR ) endif() diff --git a/Vulkan/RenderPlatform.cpp b/Vulkan/RenderPlatform.cpp index b39b82f4..8d0a16b7 100644 --- a/Vulkan/RenderPlatform.cpp +++ b/Vulkan/RenderPlatform.cpp @@ -1406,7 +1406,13 @@ vk::Format RenderPlatform::ToVulkanFormat(crossplatform::PixelFormat p,crossplat case RGB_11_11_10_FLOAT: return vk::Format::eB10G11R11UfloatPack32; case RGBA_16_FLOAT: + switch (c) + { + case crossplatform::CompressionFormat::BC6H: + return vk::Format::eBc6HUfloatBlock; + default: return vk::Format::eR16G16B16A16Sfloat; + }; case RGBA_32_FLOAT: return vk::Format::eR32G32B32A32Sfloat; case RGBA_32_UINT: @@ -1432,6 +1438,8 @@ vk::Format RenderPlatform::ToVulkanFormat(crossplatform::PixelFormat p,crossplat return vk::Format::eBc1RgbaUnormBlock; case crossplatform::CompressionFormat::BC3: return vk::Format::eBc3UnormBlock; + case crossplatform::CompressionFormat::BC7_M6_OPAQUE_ONLY: + return vk::Format::eBc7UnormBlock; default: return vk::Format::eR8G8B8A8Unorm; }; @@ -1444,6 +1452,8 @@ vk::Format RenderPlatform::ToVulkanFormat(crossplatform::PixelFormat p,crossplat return vk::Format::eBc1RgbaUnormBlock; case crossplatform::CompressionFormat::BC3: return vk::Format::eBc3UnormBlock; + case crossplatform::CompressionFormat::BC7_M6_OPAQUE_ONLY: + return vk::Format::eBc7UnormBlock; default: return vk::Format::eB8G8R8A8Unorm; }; diff --git a/Vulkan/Texture.cpp b/Vulkan/Texture.cpp index f2254693..55b1b040 100644 --- a/Vulkan/Texture.cpp +++ b/Vulkan/Texture.cpp @@ -352,9 +352,9 @@ vk::ImageView *Texture::CreateVulkanImageView(crossplatform::TextureView texture // TODO: Should we override aspect if the texture is a depth stencil? - AJR vk::ImageAspectFlagBits aspect = depthStencil ? vk::ImageAspectFlagBits::eDepth : vk::ImageAspectFlagBits(textureView.elements.subresourceRange.aspectMask); const uint8_t &startMip = textureView.elements.subresourceRange.baseMipLevel; - const uint8_t &numMips = textureView.elements.subresourceRange.mipLevelCount == uint8_t(0xFF) ? mips - startMip : textureView.elements.subresourceRange.mipLevelCount; + const uint8_t numMips = textureView.elements.subresourceRange.mipLevelCount == uint8_t(0xFF) ? mips - startMip : textureView.elements.subresourceRange.mipLevelCount; const uint8_t &startLayer = textureView.elements.subresourceRange.baseArrayLayer; - const uint8_t &numLayers = textureView.elements.subresourceRange.arrayLayerCount == uint8_t(0xFF) ? NumFaces() - startLayer : textureView.elements.subresourceRange.arrayLayerCount; + const uint8_t numLayers = textureView.elements.subresourceRange.arrayLayerCount == uint8_t(0xFF) ? NumFaces() - startLayer : textureView.elements.subresourceRange.arrayLayerCount; crossplatform::ShaderResourceType type = textureView.elements.type; if(type==crossplatform::ShaderResourceType::UNKNOWN) @@ -560,8 +560,8 @@ bool Texture::ensureTexture2DSizeAndFormat(crossplatform::RenderPlatform* r, int int mip_width=width; int mip_length=length; ClearLoadedTextures(); - ResizeLoadedTextures(mips, 1); - for (uint32_t i = 0; i < uint32_t(mips); i++) + ResizeLoadedTextures(data->size(), 1); + for (uint32_t i = 0; i < uint32_t(mLoadedTextures.size()); i++) { uint32_t n = CalculateSubresourceIndex(i, 0, 0, mips, 1); LoadedTexture& loadedTexture=mLoadedTextures[i][0]; @@ -891,6 +891,8 @@ void Texture::LoadTextureData(LoadedTexture <,const char* path) void Texture::SetTextureData(LoadedTexture <,const void *data,int x,int y,int z,int n,crossplatform::PixelFormat f,crossplatform::CompressionFormat cf) { + if(!data) + return; lt.data=( unsigned char*)data; lt.x=x; lt.y=y; @@ -902,6 +904,7 @@ void Texture::SetTextureData(LoadedTexture <,const void *data,int x,int y,int size_t SysMemPitch =x*bytesPerTexel; size_t bufferSize = SysMemPitch*y; size_t numRowsToCopy = y; + size_t block_size_bytes=bytesPerTexel; switch (cf) { case crossplatform::CompressionFormat::BC1: @@ -909,21 +912,25 @@ void Texture::SetTextureData(LoadedTexture <,const void *data,int x,int y,int case crossplatform::CompressionFormat::BC5: case crossplatform::CompressionFormat::ETC1: case crossplatform::CompressionFormat::ETC2: + block_size_bytes=(bytesPerTexel==4)?16:8; + break; + case crossplatform::CompressionFormat::BC6H: + block_size_bytes=16; + break; + default: + break; + }; + if(cf!=crossplatform::CompressionFormat::UNCOMPRESSED) { // The memory pitch of a line for uncompressed formats, becomes the memory pitch // of a row of blocks in the case of the compressed. size_t block_width =std::max(1,(x+3)/4); size_t block_height =std::max(1,(y+3)/4); - size_t block_size_bytes=(bytesPerTexel==4)?16:8; SysMemPitch = block_size_bytes*block_width; // buffer must be at least one block in size. bufferSize = std::max(block_size_bytes,SysMemPitch*block_height); numRowsToCopy =block_height; } - break; - default: - break; - }; //int texelBytes=vulkan::RenderPlatform::FormatTexelBytes(f); vk::Device *vulkanDevice = ((vulkan::RenderPlatform *)renderPlatform)->AsVulkanDevice(); vulkan::RenderPlatform *vkRenderPlatform=(vulkan::RenderPlatform *)renderPlatform; @@ -940,7 +947,6 @@ void Texture::SetTextureData(LoadedTexture <,const void *data,int x,int y,int vmaMapMemory(lt.allocationInfo.allocator, lt.allocationInfo.allocation, &mapped_data); SIMUL_ASSERT(mapped_data !=nullptr); - //memcpy(data, lt.data, lt.x * lt.y*4); uint8_t *target_data =(uint8_t*)mapped_data; uint8_t *cPtr =(uint8_t*)lt.data; size_t totalBytes=0;