seedlingattempt/godot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Support 64-bit sizes in Compression

Browse Source
This commit is contained in:
Aaron Franke
2025-05-08 14:13:30 -07:00
parent e45cc68092
commit 5777a88b76
13 changed files with 58 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
core/io/compression.cpp
View File

@ -47,12 +47,13 @@ static ZSTD_DCtx *current_zstd_d_ctx = nullptr;
static bool current_zstd_long_distance_matching; static bool current_zstd_long_distance_matching;
static int current_zstd_window_log_size; static int current_zstd_window_log_size;
int Compression::compress(uint8_t *p_dst, const uint8_t *p_src, int p_src_size, Mode p_mode) { int64_t Compression::compress(uint8_t *p_dst, const uint8_t *p_src, int64_t p_src_size, Mode p_mode) {
switch (p_mode) { switch (p_mode) {
case MODE_BROTLI: { case MODE_BROTLI: {
ERR_FAIL_V_MSG(-1, "Only brotli decompression is supported."); ERR_FAIL_V_MSG(-1, "Only brotli decompression is supported.");
} break; } break;
case MODE_FASTLZ: { case MODE_FASTLZ: {
ERR_FAIL_COND_V_MSG(p_src_size > INT32_MAX, -1, "Cannot compress a FastLZ/LZ77 file 2 GiB or larger. LZ77 supports larger files, but FastLZ's implementation uses C++ `int` so is limited to 2 GiB. Consider using Zstd instead.");
if (p_src_size < 16) { if (p_src_size < 16) {
uint8_t src[16]; uint8_t src[16];
memset(&src[p_src_size], 0, 16 - p_src_size); memset(&src[p_src_size], 0, 16 - p_src_size);
@ -65,6 +66,7 @@ int Compression::compress(uint8_t *p_dst, const uint8_t *p_src, int p_src_size,
} break; } break;
case MODE_DEFLATE: case MODE_DEFLATE:
case MODE_GZIP: { case MODE_GZIP: {
ERR_FAIL_COND_V_MSG(p_src_size > INT32_MAX, -1, "Cannot compress a Deflate or GZip file 2 GiB or larger. Deflate and GZip are both limited to 4 GiB, and ZLib's implementation uses C++ `int` so is limited to 2 GiB. Consider using Zstd instead.");
int window_bits = p_mode == MODE_DEFLATE ? 15 : 15 + 16; int window_bits = p_mode == MODE_DEFLATE ? 15 : 15 + 16;
z_stream strm; z_stream strm;
@ -95,22 +97,23 @@ int Compression::compress(uint8_t *p_dst, const uint8_t *p_src, int p_src_size,
ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1); ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1);
ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, zstd_window_log_size); ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, zstd_window_log_size);
} }
int max_dst_size = get_max_compressed_buffer_size(p_src_size, MODE_ZSTD); const int64_t max_dst_size = get_max_compressed_buffer_size(p_src_size, MODE_ZSTD);
int ret = ZSTD_compressCCtx(cctx, p_dst, max_dst_size, p_src, p_src_size, zstd_level); const size_t ret = ZSTD_compressCCtx(cctx, p_dst, max_dst_size, p_src, p_src_size, zstd_level);
ZSTD_freeCCtx(cctx); ZSTD_freeCCtx(cctx);
return ret; return (int64_t)ret;
} break; } break;
} }
ERR_FAIL_V(-1); ERR_FAIL_V(-1);
} }
int Compression::get_max_compressed_buffer_size(int p_src_size, Mode p_mode) { int64_t Compression::get_max_compressed_buffer_size(int64_t p_src_size, Mode p_mode) {
switch (p_mode) { switch (p_mode) {
case MODE_BROTLI: { case MODE_BROTLI: {
ERR_FAIL_V_MSG(-1, "Only brotli decompression is supported."); ERR_FAIL_V_MSG(-1, "Only brotli decompression is supported.");
} break; } break;
case MODE_FASTLZ: { case MODE_FASTLZ: {
ERR_FAIL_COND_V_MSG(p_src_size > INT32_MAX, -1, "Cannot compress a FastLZ/LZ77 file 2 GiB or larger. LZ77 supports larger files, but FastLZ's implementation uses C++ `int` so is limited to 2 GiB. Consider using Zstd instead.");
int ss = p_src_size + p_src_size * 6 / 100; int ss = p_src_size + p_src_size * 6 / 100;
if (ss < 66) { if (ss < 66) {
ss = 66; ss = 66;
@ -120,6 +123,7 @@ int Compression::get_max_compressed_buffer_size(int p_src_size, Mode p_mode) {
} break; } break;
case MODE_DEFLATE: case MODE_DEFLATE:
case MODE_GZIP: { case MODE_GZIP: {
ERR_FAIL_COND_V_MSG(p_src_size > INT32_MAX, -1, "Cannot compress a Deflate or GZip file 2 GiB or larger. Deflate and GZip are both limited to 4 GiB, and ZLib's implementation uses C++ `int` so is limited to 2 GiB. Consider using Zstd instead.");
int window_bits = p_mode == MODE_DEFLATE ? 15 : 15 + 16; int window_bits = p_mode == MODE_DEFLATE ? 15 : 15 + 16;
z_stream strm; z_stream strm;
@ -142,7 +146,7 @@ int Compression::get_max_compressed_buffer_size(int p_src_size, Mode p_mode) {
ERR_FAIL_V(-1); ERR_FAIL_V(-1);
} }
int Compression::decompress(uint8_t *p_dst, int p_dst_max_size, const uint8_t *p_src, int p_src_size, Mode p_mode) { int64_t Compression::decompress(uint8_t *p_dst, int64_t p_dst_max_size, const uint8_t *p_src, int64_t p_src_size, Mode p_mode) {
switch (p_mode) { switch (p_mode) {
case MODE_BROTLI: { case MODE_BROTLI: {
#ifdef BROTLI_ENABLED #ifdef BROTLI_ENABLED
@ -155,6 +159,7 @@ int Compression::decompress(uint8_t *p_dst, int p_dst_max_size, const uint8_t *p
#endif #endif
} break; } break;
case MODE_FASTLZ: { case MODE_FASTLZ: {
ERR_FAIL_COND_V_MSG(p_dst_max_size > INT32_MAX, -1, "Cannot decompress a FastLZ/LZ77 file 2 GiB or larger. LZ77 supports larger files, but FastLZ's implementation uses C++ `int` so is limited to 2 GiB. Consider using Zstd instead.");
int ret_size = 0; int ret_size = 0;
if (p_dst_max_size < 16) { if (p_dst_max_size < 16) {
@ -169,6 +174,7 @@ int Compression::decompress(uint8_t *p_dst, int p_dst_max_size, const uint8_t *p
} break; } break;
case MODE_DEFLATE: case MODE_DEFLATE:
case MODE_GZIP: { case MODE_GZIP: {
ERR_FAIL_COND_V_MSG(p_dst_max_size > INT32_MAX, -1, "Cannot decompress a Deflate or GZip file 2 GiB or larger. Deflate and GZip are both limited to 4 GiB, and ZLib's implementation uses C++ `int` so is limited to 2 GiB. Consider using Zstd instead.");
int window_bits = p_mode == MODE_DEFLATE ? 15 : 15 + 16; int window_bits = p_mode == MODE_DEFLATE ? 15 : 15 + 16;
z_stream strm; z_stream strm;
@ -207,8 +213,8 @@ int Compression::decompress(uint8_t *p_dst, int p_dst_max_size, const uint8_t *p
current_zstd_window_log_size = zstd_window_log_size; current_zstd_window_log_size = zstd_window_log_size;
} }
int ret = ZSTD_decompressDCtx(current_zstd_d_ctx, p_dst, p_dst_max_size, p_src, p_src_size); size_t ret = ZSTD_decompressDCtx(current_zstd_d_ctx, p_dst, p_dst_max_size, p_src, p_src_size);
return ret; return (int64_t)ret;
} break; } break;
} }
@ -220,7 +226,7 @@ int Compression::decompress(uint8_t *p_dst, int p_dst_max_size, const uint8_t *p
This is required for compressed data whose final uncompressed size is unknown, as is the case for HTTP response bodies. This is required for compressed data whose final uncompressed size is unknown, as is the case for HTTP response bodies.
This is much slower however than using Compression::decompress because it may result in multiple full copies of the output buffer. This is much slower however than using Compression::decompress because it may result in multiple full copies of the output buffer.
*/ */
int Compression::decompress_dynamic(Vector<uint8_t> *p_dst_vect, int p_max_dst_size, const uint8_t *p_src, int p_src_size, Mode p_mode) { int Compression::decompress_dynamic(Vector<uint8_t> *p_dst_vect, int64_t p_max_dst_size, const uint8_t *p_src, int64_t p_src_size, Mode p_mode) {
uint8_t *dst = nullptr; uint8_t *dst = nullptr;
int out_mark = 0; int out_mark = 0;

8
core/io/compression.h
View File

@ -52,8 +52,8 @@ public:
MODE_BROTLI MODE_BROTLI
}; };
static int compress(uint8_t *p_dst, const uint8_t *p_src, int p_src_size, Mode p_mode = MODE_ZSTD); static int64_t compress(uint8_t *p_dst, const uint8_t *p_src, int64_t p_src_size, Mode p_mode = MODE_ZSTD);
static int get_max_compressed_buffer_size(int p_src_size, Mode p_mode = MODE_ZSTD); static int64_t get_max_compressed_buffer_size(int64_t p_src_size, Mode p_mode = MODE_ZSTD);
static int decompress(uint8_t *p_dst, int p_dst_max_size, const uint8_t *p_src, int p_src_size, Mode p_mode = MODE_ZSTD); static int64_t decompress(uint8_t *p_dst, int64_t p_dst_max_size, const uint8_t *p_src, int64_t p_src_size, Mode p_mode = MODE_ZSTD);
static int decompress_dynamic(Vector<uint8_t> *p_dst_vect, int p_max_dst_size, const uint8_t *p_src, int p_src_size, Mode p_mode); static int decompress_dynamic(Vector<uint8_t> *p_dst_vect, int64_t p_max_dst_size, const uint8_t *p_src, int64_t p_src_size, Mode p_mode);
}; };

13
core/io/file_access_compressed.cpp
View File

@ -68,7 +68,7 @@ Error FileAccessCompressed::open_after_magic(Ref<FileAccess> p_base) {
read_block_count = bc; read_block_count = bc;
read_block_size = read_blocks.size() == 1 ? read_total : block_size; read_block_size = read_blocks.size() == 1 ? read_total : block_size;
int ret = Compression::decompress(buffer.ptrw(), read_block_size, comp_buffer.ptr(), read_blocks[0].csize, cmode); const int64_t ret = Compression::decompress(buffer.ptrw(), read_block_size, comp_buffer.ptr(), read_blocks[0].csize, cmode);
read_block = 0; read_block = 0;
read_pos = 0; read_pos = 0;
@ -137,10 +137,11 @@ void FileAccessCompressed::_close() {
Vector<uint8_t> cblock; Vector<uint8_t> cblock;
cblock.resize(Compression::get_max_compressed_buffer_size(bl, cmode)); cblock.resize(Compression::get_max_compressed_buffer_size(bl, cmode));
int s = Compression::compress(cblock.ptrw(), bp, bl, cmode); const int64_t compressed_size = Compression::compress(cblock.ptrw(), bp, bl, cmode);
ERR_FAIL_COND_MSG(compressed_size < 0, "FileAccessCompressed: Error compressing data.");
f->store_buffer(cblock.ptr(), s); f->store_buffer(cblock.ptr(), (uint64_t)compressed_size);
block_sizes.push_back(s); block_sizes.push_back(compressed_size);
} }
f->seek(16); //ok write block sizes f->seek(16); //ok write block sizes
@ -200,7 +201,7 @@ void FileAccessCompressed::seek(uint64_t p_position) {
read_block = block_idx; read_block = block_idx;
f->seek(read_blocks[read_block].offset); f->seek(read_blocks[read_block].offset);
f->get_buffer(comp_buffer.ptrw(), read_blocks[read_block].csize); f->get_buffer(comp_buffer.ptrw(), read_blocks[read_block].csize);
int ret = Compression::decompress(buffer.ptrw(), read_blocks.size() == 1 ? read_total : block_size, comp_buffer.ptr(), read_blocks[read_block].csize, cmode); const int64_t ret = Compression::decompress(buffer.ptrw(), read_blocks.size() == 1 ? read_total : block_size, comp_buffer.ptr(), read_blocks[read_block].csize, cmode);
ERR_FAIL_COND_MSG(ret == -1, "Compressed file is corrupt."); ERR_FAIL_COND_MSG(ret == -1, "Compressed file is corrupt.");
read_block_size = read_block == read_block_count - 1 ? read_total % block_size : block_size; read_block_size = read_block == read_block_count - 1 ? read_total % block_size : block_size;
} }
@ -288,7 +289,7 @@ uint64_t FileAccessCompressed::get_buffer(uint8_t *p_dst, uint64_t p_length) con
// Read the next block of compressed data. // Read the next block of compressed data.
f->get_buffer(comp_buffer.ptrw(), read_blocks[read_block].csize); f->get_buffer(comp_buffer.ptrw(), read_blocks[read_block].csize);
int ret = Compression::decompress(buffer.ptrw(), read_blocks.size() == 1 ? read_total : block_size, comp_buffer.ptr(), read_blocks[read_block].csize, cmode); const int64_t ret = Compression::decompress(buffer.ptrw(), read_blocks.size() == 1 ? read_total : block_size, comp_buffer.ptr(), read_blocks[read_block].csize, cmode);
ERR_FAIL_COND_V_MSG(ret == -1, -1, "Compressed file is corrupt."); ERR_FAIL_COND_V_MSG(ret == -1, -1, "Compressed file is corrupt.");
read_block_size = read_block == read_block_count - 1 ? read_total % block_size : block_size; read_block_size = read_block == read_block_count - 1 ? read_total % block_size : block_size;
read_pos = 0; read_pos = 0;

4
core/io/remote_filesystem_client.cpp
View File

@ -215,7 +215,7 @@ Error RemoteFilesystemClient::_synchronize_with_server(const String &p_host, int
Vector<uint8_t> file_cache_buffer; Vector<uint8_t> file_cache_buffer;
if (file_cache.size()) { if (file_cache.size()) {
StringBuilder sbuild; StringBuilder sbuild;
for (int i = 0; i < file_cache.size(); i++) { for (int64_t i = 0; i < file_cache.size(); i++) {
sbuild.append(file_cache[i].path); sbuild.append(file_cache[i].path);
sbuild.append("::"); sbuild.append("::");
sbuild.append(itos(file_cache[i].server_modified_time)); sbuild.append(itos(file_cache[i].server_modified_time));
@ -224,7 +224,7 @@ Error RemoteFilesystemClient::_synchronize_with_server(const String &p_host, int
String s = sbuild.as_string(); String s = sbuild.as_string();
CharString cs = s.utf8(); CharString cs = s.utf8();
file_cache_buffer.resize(Compression::get_max_compressed_buffer_size(cs.length(), Compression::MODE_ZSTD)); file_cache_buffer.resize(Compression::get_max_compressed_buffer_size(cs.length(), Compression::MODE_ZSTD));
int res_len = Compression::compress(file_cache_buffer.ptrw(), (const uint8_t *)cs.ptr(), cs.length(), Compression::MODE_ZSTD); const int64_t res_len = Compression::compress(file_cache_buffer.ptrw(), (const uint8_t *)cs.ptr(), cs.length(), Compression::MODE_ZSTD);
file_cache_buffer.resize(res_len); file_cache_buffer.resize(res_len);
tcp_client->put_32(cs.length()); // Size of buffer uncompressed tcp_client->put_32(cs.length()); // Size of buffer uncompressed

4
core/variant/variant_call.cpp
View File

@ -799,7 +799,7 @@ struct _VariantCall {
if (p_instance->size() > 0) { if (p_instance->size() > 0) {
Compression::Mode mode = (Compression::Mode)(p_mode); Compression::Mode mode = (Compression::Mode)(p_mode);
compressed.resize(Compression::get_max_compressed_buffer_size(p_instance->size(), mode)); compressed.resize(Compression::get_max_compressed_buffer_size(p_instance->size(), mode));
int result = Compression::compress(compressed.ptrw(), p_instance->ptr(), p_instance->size(), mode); int64_t result = Compression::compress(compressed.ptrw(), p_instance->ptr(), p_instance->size(), mode);
result = result >= 0 ? result : 0; result = result >= 0 ? result : 0;
compressed.resize(result); compressed.resize(result);
@ -822,7 +822,7 @@ struct _VariantCall {
} }
decompressed.resize(buffer_size); decompressed.resize(buffer_size);
int result = Compression::decompress(decompressed.ptrw(), buffer_size, p_instance->ptr(), p_instance->size(), mode); int64_t result = Compression::decompress(decompressed.ptrw(), buffer_size, p_instance->ptr(), p_instance->size(), mode);
result = result >= 0 ? result : 0; result = result >= 0 ? result : 0;
decompressed.resize(result); decompressed.resize(result);

3
editor/debugger/editor_file_server.cpp
View File

@ -173,7 +173,8 @@ void EditorFileServer::poll() {
ERR_FAIL_COND(err != OK); ERR_FAIL_COND(err != OK);
// Decompress the text with all the files // Decompress the text with all the files
Compression::decompress(file_buffer_decompressed.ptr(), file_buffer_decompressed.size(), file_buffer.ptr(), file_buffer.size(), Compression::MODE_ZSTD); const int64_t decompressed_size = Compression::decompress(file_buffer_decompressed.ptr(), file_buffer_decompressed.size(), file_buffer.ptr(), file_buffer.size(), Compression::MODE_ZSTD);
ERR_FAIL_COND_MSG(decompressed_size != file_buffer_decompressed.size(), "Error decompressing file buffer. Decompressed size did not match the expected size.");
String files_text = String::utf8((const char *)file_buffer_decompressed.ptr(), file_buffer_decompressed.size()); String files_text = String::utf8((const char *)file_buffer_decompressed.ptr(), file_buffer_decompressed.size());
Vector<String> files = files_text.split("\n"); Vector<String> files = files_text.split("\n");

6
editor/doc_tools.cpp
View File

@ -1798,10 +1798,10 @@ Error DocTools::save_classes(const String &p_default_path, const HashMap<String,
return OK; return OK;
} }
Error DocTools::load_compressed(const uint8_t *p_data, int p_compressed_size, int p_uncompressed_size) { Error DocTools::load_compressed(const uint8_t *p_data, int64_t p_compressed_size, int64_t p_uncompressed_size) {
Vector<uint8_t> data; Vector<uint8_t> data;
data.resize(p_uncompressed_size); data.resize(p_uncompressed_size);
int ret = Compression::decompress(data.ptrw(), p_uncompressed_size, p_data, p_compressed_size, Compression::MODE_DEFLATE); const int64_t ret = Compression::decompress(data.ptrw(), p_uncompressed_size, p_data, p_compressed_size, Compression::MODE_DEFLATE);
ERR_FAIL_COND_V_MSG(ret == -1, ERR_FILE_CORRUPT, "Compressed file is corrupt."); ERR_FAIL_COND_V_MSG(ret == -1, ERR_FILE_CORRUPT, "Compressed file is corrupt.");
class_list.clear(); class_list.clear();
@ -1816,7 +1816,7 @@ Error DocTools::load_compressed(const uint8_t *p_data, int p_compressed_size, in
return OK; return OK;
} }
Error DocTools::load_xml(const uint8_t *p_data, int p_size) { Error DocTools::load_xml(const uint8_t *p_data, int64_t p_size) {
Ref<XMLParser> parser = memnew(XMLParser); Ref<XMLParser> parser = memnew(XMLParser);
Error err = parser->_open_buffer(p_data, p_size); Error err = parser->_open_buffer(p_data, p_size);
if (err) { if (err) {

4
editor/doc_tools.h
View File

@ -55,6 +55,6 @@ public:
Error save_classes(const String &p_default_path, const HashMap<String, String> &p_class_path, bool p_use_relative_schema = true); Error save_classes(const String &p_default_path, const HashMap<String, String> &p_class_path, bool p_use_relative_schema = true);
Error _load(Ref<XMLParser> parser); Error _load(Ref<XMLParser> parser);
Error load_compressed(const uint8_t *p_data, int p_compressed_size, int p_uncompressed_size); Error load_compressed(const uint8_t *p_data, int64_t p_compressed_size, int64_t p_uncompressed_size);
Error load_xml(const uint8_t *p_data, int p_size); Error load_xml(const uint8_t *p_data, int64_t p_size);
}; };

10
editor/editor_translation.cpp
View File

@ -61,7 +61,7 @@ void load_editor_translations(const String &p_locale) {
if (etl->lang == p_locale) { if (etl->lang == p_locale) {
Vector<uint8_t> data; Vector<uint8_t> data;
data.resize(etl->uncomp_size); data.resize(etl->uncomp_size);
int ret = Compression::decompress(data.ptrw(), etl->uncomp_size, etl->data, etl->comp_size, Compression::MODE_DEFLATE); const int64_t ret = Compression::decompress(data.ptrw(), etl->uncomp_size, etl->data, etl->comp_size, Compression::MODE_DEFLATE);
ERR_FAIL_COND_MSG(ret == -1, "Compressed file is corrupt."); ERR_FAIL_COND_MSG(ret == -1, "Compressed file is corrupt.");
Ref<FileAccessMemory> fa; Ref<FileAccessMemory> fa;
@ -89,7 +89,7 @@ void load_property_translations(const String &p_locale) {
if (etl->lang == p_locale) { if (etl->lang == p_locale) {
Vector<uint8_t> data; Vector<uint8_t> data;
data.resize(etl->uncomp_size); data.resize(etl->uncomp_size);
int ret = Compression::decompress(data.ptrw(), etl->uncomp_size, etl->data, etl->comp_size, Compression::MODE_DEFLATE); const int64_t ret = Compression::decompress(data.ptrw(), etl->uncomp_size, etl->data, etl->comp_size, Compression::MODE_DEFLATE);
ERR_FAIL_COND_MSG(ret == -1, "Compressed file is corrupt."); ERR_FAIL_COND_MSG(ret == -1, "Compressed file is corrupt.");
Ref<FileAccessMemory> fa; Ref<FileAccessMemory> fa;
@ -117,7 +117,7 @@ void load_doc_translations(const String &p_locale) {
if (dtl->lang == p_locale) { if (dtl->lang == p_locale) {
Vector<uint8_t> data; Vector<uint8_t> data;
data.resize(dtl->uncomp_size); data.resize(dtl->uncomp_size);
int ret = Compression::decompress(data.ptrw(), dtl->uncomp_size, dtl->data, dtl->comp_size, Compression::MODE_DEFLATE); const int64_t ret = Compression::decompress(data.ptrw(), dtl->uncomp_size, dtl->data, dtl->comp_size, Compression::MODE_DEFLATE);
ERR_FAIL_COND_MSG(ret == -1, "Compressed file is corrupt."); ERR_FAIL_COND_MSG(ret == -1, "Compressed file is corrupt.");
Ref<FileAccessMemory> fa; Ref<FileAccessMemory> fa;
@ -145,7 +145,7 @@ void load_extractable_translations(const String &p_locale) {
if (etl->lang == p_locale) { if (etl->lang == p_locale) {
Vector<uint8_t> data; Vector<uint8_t> data;
data.resize(etl->uncomp_size); data.resize(etl->uncomp_size);
int ret = Compression::decompress(data.ptrw(), etl->uncomp_size, etl->data, etl->comp_size, Compression::MODE_DEFLATE); const int64_t ret = Compression::decompress(data.ptrw(), etl->uncomp_size, etl->data, etl->comp_size, Compression::MODE_DEFLATE);
ERR_FAIL_COND_MSG(ret == -1, "Compressed file is corrupt."); ERR_FAIL_COND_MSG(ret == -1, "Compressed file is corrupt.");
Ref<FileAccessMemory> fa; Ref<FileAccessMemory> fa;
@ -177,7 +177,7 @@ Vector<Vector<String>> get_extractable_message_list() {
Vector<uint8_t> data; Vector<uint8_t> data;
data.resize(etl->uncomp_size); data.resize(etl->uncomp_size);
int ret = Compression::decompress(data.ptrw(), etl->uncomp_size, etl->data, etl->comp_size, Compression::MODE_DEFLATE); const int64_t ret = Compression::decompress(data.ptrw(), etl->uncomp_size, etl->data, etl->comp_size, Compression::MODE_DEFLATE);
ERR_FAIL_COND_V_MSG(ret == -1, list, "Compressed file is corrupt."); ERR_FAIL_COND_V_MSG(ret == -1, list, "Compressed file is corrupt.");
Ref<FileAccessMemory> fa; Ref<FileAccessMemory> fa;

17
modules/enet/enet_connection.cpp
View File

@ -422,11 +422,11 @@ size_t ENetConnection::Compressor::enet_compress(void *context, const ENetBuffer
compressor->src_mem.resize(inLimit); compressor->src_mem.resize(inLimit);
} }
int total = inLimit; size_t total = inLimit;
int ofs = 0; size_t ofs = 0;
while (total) { while (total) {
for (size_t i = 0; i < inBufferCount; i++) { for (size_t i = 0; i < inBufferCount; i++) {
int to_copy = MIN(total, int(inBuffers[i].dataLength)); const size_t to_copy = MIN(total, inBuffers[i].dataLength);
memcpy(&compressor->src_mem.write[ofs], inBuffers[i].data, to_copy); memcpy(&compressor->src_mem.write[ofs], inBuffers[i].data, to_copy);
ofs += to_copy; ofs += to_copy;
total -= to_copy; total -= to_copy;
@ -450,28 +450,29 @@ size_t ENetConnection::Compressor::enet_compress(void *context, const ENetBuffer
} }
} }
int req_size = Compression::get_max_compressed_buffer_size(ofs, mode); const int64_t req_size = Compression::get_max_compressed_buffer_size(ofs, mode);
if (compressor->dst_mem.size() < req_size) { if (compressor->dst_mem.size() < req_size) {
compressor->dst_mem.resize(req_size); compressor->dst_mem.resize(req_size);
} }
int ret = Compression::compress(compressor->dst_mem.ptrw(), compressor->src_mem.ptr(), ofs, mode); const int64_t ret = Compression::compress(compressor->dst_mem.ptrw(), compressor->src_mem.ptr(), ofs, mode);
if (ret < 0) { if (ret < 0) {
return 0; return 0;
} }
if (ret > int(outLimit)) { const size_t ret_size = size_t(ret);
if (ret_size > outLimit) {
return 0; // Do not bother return 0; // Do not bother
} }
memcpy(outData, compressor->dst_mem.ptr(), ret); memcpy(outData, compressor->dst_mem.ptr(), ret_size);
return ret; return ret;
} }
size_t ENetConnection::Compressor::enet_decompress(void *context, const enet_uint8 *inData, size_t inLimit, enet_uint8 *outData, size_t outLimit) { size_t ENetConnection::Compressor::enet_decompress(void *context, const enet_uint8 *inData, size_t inLimit, enet_uint8 *outData, size_t outLimit) {
Compressor *compressor = (Compressor *)(context); Compressor *compressor = (Compressor *)(context);
int ret = -1; int64_t ret = -1;
switch (compressor->mode) { switch (compressor->mode) {
case COMPRESS_FASTLZ: { case COMPRESS_FASTLZ: {
ret = Compression::decompress(outData, outLimit, inData, inLimit, Compression::MODE_FASTLZ); ret = Compression::decompress(outData, outLimit, inData, inLimit, Compression::MODE_FASTLZ);

6
modules/gdscript/gdscript_tokenizer_buffer.cpp
View File

@ -152,7 +152,7 @@ Error GDScriptTokenizerBuffer::set_code_buffer(const Vector<uint8_t> &p_buffer)
contents = p_buffer.slice(12); contents = p_buffer.slice(12);
} else { } else {
contents.resize(decompressed_size); contents.resize(decompressed_size);
int result = Compression::decompress(contents.ptrw(), contents.size(), &buf[12], p_buffer.size() - 12, Compression::MODE_ZSTD); const int64_t result = Compression::decompress(contents.ptrw(), contents.size(), &buf[12], p_buffer.size() - 12, Compression::MODE_ZSTD);
ERR_FAIL_COND_V_MSG(result != decompressed_size, ERR_INVALID_DATA, "Error decompressing GDScript tokenizer buffer."); ERR_FAIL_COND_V_MSG(result != decompressed_size, ERR_INVALID_DATA, "Error decompressing GDScript tokenizer buffer.");
} }
@ -371,10 +371,10 @@ Vector<uint8_t> GDScriptTokenizerBuffer::parse_code_string(const String &p_code,
case COMPRESS_ZSTD: { case COMPRESS_ZSTD: {
encode_uint32(contents.size(), &buf.write[8]); encode_uint32(contents.size(), &buf.write[8]);
Vector<uint8_t> compressed; Vector<uint8_t> compressed;
int max_size = Compression::get_max_compressed_buffer_size(contents.size(), Compression::MODE_ZSTD); const int64_t max_size = Compression::get_max_compressed_buffer_size(contents.size(), Compression::MODE_ZSTD);
compressed.resize(max_size); compressed.resize(max_size);
int compressed_size = Compression::compress(compressed.ptrw(), contents.ptr(), contents.size(), Compression::MODE_ZSTD); const int64_t compressed_size = Compression::compress(compressed.ptrw(), contents.ptr(), contents.size(), Compression::MODE_ZSTD);
ERR_FAIL_COND_V_MSG(compressed_size < 0, Vector<uint8_t>(), "Error compressing GDScript tokenizer buffer."); ERR_FAIL_COND_V_MSG(compressed_size < 0, Vector<uint8_t>(), "Error compressing GDScript tokenizer buffer.");
compressed.resize(compressed_size); compressed.resize(compressed_size);

3
modules/mbedtls/crypto_mbedtls.cpp
View File

@ -371,7 +371,8 @@ void CryptoMbedTLS::load_default_certificates(const String &p_path) {
// Use builtin certs if there are no system certs. // Use builtin certs if there are no system certs.
PackedByteArray certs; PackedByteArray certs;
certs.resize(_certs_uncompressed_size + 1); certs.resize(_certs_uncompressed_size + 1);
Compression::decompress(certs.ptrw(), _certs_uncompressed_size, _certs_compressed, _certs_compressed_size, Compression::MODE_DEFLATE); const int64_t decompressed_size = Compression::decompress(certs.ptrw(), _certs_uncompressed_size, _certs_compressed, _certs_compressed_size, Compression::MODE_DEFLATE);
ERR_FAIL_COND_MSG(decompressed_size != _certs_uncompressed_size, "Error decompressing builtin CA certificates. Decompressed size did not match expected size.");
certs.write[_certs_uncompressed_size] = 0; // Make sure it ends with string terminator certs.write[_certs_uncompressed_size] = 0; // Make sure it ends with string terminator
default_certs->load_from_memory(certs.ptr(), certs.size()); default_certs->load_from_memory(certs.ptr(), certs.size());
print_verbose("Loaded builtin CA certificates"); print_verbose("Loaded builtin CA certificates");

4
servers/rendering/rendering_shader_container.cpp
View File

@ -422,10 +422,10 @@ bool RenderingShaderContainer::compress_code(const uint8_t *p_decompressed_bytes
*r_compressed_flags = 0; *r_compressed_flags = 0;
PackedByteArray zstd_bytes; PackedByteArray zstd_bytes;
int zstd_max_bytes = Compression::get_max_compressed_buffer_size(p_decompressed_size, Compression::MODE_ZSTD); const int64_t zstd_max_bytes = Compression::get_max_compressed_buffer_size(p_decompressed_size, Compression::MODE_ZSTD);
zstd_bytes.resize(zstd_max_bytes); zstd_bytes.resize(zstd_max_bytes);
int zstd_size = Compression::compress(zstd_bytes.ptrw(), p_decompressed_bytes, p_decompressed_size, Compression::MODE_ZSTD); const int64_t zstd_size = Compression::compress(zstd_bytes.ptrw(), p_decompressed_bytes, p_decompressed_size, Compression::MODE_ZSTD);
if (zstd_size > 0 && (uint32_t)(zstd_size) < p_decompressed_size) { if (zstd_size > 0 && (uint32_t)(zstd_size) < p_decompressed_size) {
// Only choose Zstd if it results in actual compression. // Only choose Zstd if it results in actual compression.
memcpy(p_compressed_bytes, zstd_bytes.ptr(), zstd_size); memcpy(p_compressed_bytes, zstd_bytes.ptr(), zstd_size);