seedlingattempt/godot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add support for delta encoding to patch PCKs

Browse Source
This commit is contained in:
Mikael Hermansson
2025-10-24 16:10:45 +02:00
parent 9dd6c4dbac
commit 0cc88f34da
17 changed files with 851 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
editor/export/editor_export.cpp
View File

@ -83,7 +83,13 @@ void EditorExport::_save() {
config->set_value(section, "include_filter", preset->get_include_filter());
config->set_value(section, "exclude_filter", preset->get_exclude_filter());
config->set_value(section, "export_path", preset->get_export_path());
config->set_value(section, "patches", preset->get_patches());
config->set_value(section, "patch_delta_encoding", preset->is_patch_delta_encoding_enabled());
config->set_value(section, "patch_delta_compression_level_zstd", preset->get_patch_delta_zstd_level());
config->set_value(section, "patch_delta_min_reduction", preset->get_patch_delta_min_reduction());
config->set_value(section, "patch_delta_include_filters", preset->get_patch_delta_include_filter());
config->set_value(section, "patch_delta_exclude_filters", preset->get_patch_delta_exclude_filter());
config->set_value(section, "encryption_include_filters", preset->get_enc_in_filter());
config->set_value(section, "encryption_exclude_filters", preset->get_enc_ex_filter());
@ -332,6 +338,22 @@ void EditorExport::load_config() {
preset->set_script_export_mode(config->get_value(section, "script_export_mode", EditorExportPreset::MODE_SCRIPT_BINARY_TOKENS_COMPRESSED));
preset->set_patches(config->get_value(section, "patches", Vector<String>()));
if (config->has_section_key(section, "patch_delta_encoding")) {
preset->set_patch_delta_encoding_enabled(config->get_value(section, "patch_delta_encoding"));
}
if (config->has_section_key(section, "patch_delta_compression_level_zstd")) {
preset->set_patch_delta_zstd_level(config->get_value(section, "patch_delta_compression_level_zstd"));
}
if (config->has_section_key(section, "patch_delta_min_reduction")) {
preset->set_patch_delta_min_reduction(config->get_value(section, "patch_delta_min_reduction"));
}
if (config->has_section_key(section, "patch_delta_include_filters")) {
preset->set_patch_delta_include_filter(config->get_value(section, "patch_delta_include_filters"));
}
if (config->has_section_key(section, "patch_delta_exclude_filters")) {
preset->set_patch_delta_exclude_filter(config->get_value(section, "patch_delta_exclude_filters"));
}
if (config->has_section_key(section, "seed")) {
preset->set_seed(config->get_value(section, "seed"));
}

152
editor/export/editor_export_platform.cpp
View File

@ -35,6 +35,7 @@
#include "core/config/project_settings.h"
#include "core/crypto/crypto_core.h"
#include "core/extension/gdextension.h"
#include "core/io/delta_encoding.h"
#include "core/io/dir_access.h"
#include "core/io/file_access_encrypted.h"
#include "core/io/file_access_pack.h" // PACK_HEADER_MAGIC, PACK_FORMAT_VERSION
@ -66,12 +67,12 @@ class EditorExportSaveProxy {
public:
bool has_saved(const String &p_path) const { return saved_paths.has(p_path); }
Error save_file(void *p_userdata, const String &p_path, const Vector<uint8_t> &p_data, int p_file, int p_total, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed) {
Error save_file(const Ref<EditorExportPreset> &p_preset, void *p_userdata, const String &p_path, const Vector<uint8_t> &p_data, int p_file, int p_total, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed, bool p_delta) {
if (tracking_saves) {
saved_paths.insert(p_path.simplify_path().trim_prefix("res://"));
}
return save_func(p_userdata, p_path, p_data, p_file, p_total, p_enc_in_filters, p_enc_ex_filters, p_key, p_seed);
return save_func(p_preset, p_userdata, p_path, p_data, p_file, p_total, p_enc_in_filters, p_enc_ex_filters, p_key, p_seed, p_delta);
}
EditorExportSaveProxy(EditorExportPlatform::EditorExportSaveFunction p_save_func, bool p_track_saves) :
@ -218,26 +219,6 @@ bool EditorExportPlatform::fill_log_messages(RichTextLabel *p_log, Error p_err)
return has_messages;
}
bool EditorExportPlatform::_check_hash(const uint8_t *p_hash, const Vector<uint8_t> &p_data) {
if (p_hash == nullptr) {
return false;
}
unsigned char hash[16];
Error err = CryptoCore::md5(p_data.ptr(), p_data.size(), hash);
if (err != OK) {
return false;
}
for (int i = 0; i < 16; i++) {
if (p_hash[i] != hash[i]) {
return false;
}
}
return true;
}
Error EditorExportPlatform::_load_patches(const Vector<String> &p_patches) {
Error err = OK;
if (!p_patches.is_empty()) {
@ -310,7 +291,7 @@ Error EditorExportPlatform::_encrypt_and_store_data(Ref<FileAccess> p_fd, const
return OK;
}
Error EditorExportPlatform::_save_pack_file(void *p_userdata, const String &p_path, const Vector<uint8_t> &p_data, int p_file, int p_total, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed) {
Error EditorExportPlatform::_save_pack_file(const Ref<EditorExportPreset> &p_preset, void *p_userdata, const String &p_path, const Vector<uint8_t> &p_data, int p_file, int p_total, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed, bool p_delta) {
ERR_FAIL_COND_V_MSG(p_total < 1, ERR_PARAMETER_RANGE_ERROR, "Must select at least one file to export.");
PackData *pd = (PackData *)p_userdata;
@ -328,6 +309,7 @@ Error EditorExportPlatform::_save_pack_file(void *p_userdata, const String &p_pa
sd.path_utf8 = simplified_path.trim_prefix("res://").utf8();
sd.ofs = (pd->use_sparse_pck) ? 0 : pd->f->get_position();
sd.size = p_data.size();
sd.delta = p_delta;
Error err = _encrypt_and_store_data(ftmp, simplified_path, p_data, p_enc_in_filters, p_enc_ex_filters, p_key, p_seed, sd.encrypted);
if (err != OK) {
return err;
@ -363,15 +345,67 @@ Error EditorExportPlatform::_save_pack_file(void *p_userdata, const String &p_pa
return OK;
}
Error EditorExportPlatform::_save_pack_patch_file(void *p_userdata, const String &p_path, const Vector<uint8_t> &p_data, int p_file, int p_total, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed) {
if (_check_hash(PackedData::get_singleton()->get_file_hash(p_path), p_data)) {
return OK;
Error EditorExportPlatform::_save_pack_patch_file(const Ref<EditorExportPreset> &p_preset, void *p_userdata, const String &p_path, const Vector<uint8_t> &p_data, int p_file, int p_total, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed, bool p_delta) {
Ref<FileAccess> old_file = PackedData::get_singleton()->try_open_path(p_path);
if (old_file.is_null()) {
return _save_pack_file(p_preset, p_userdata, p_path, p_data, p_file, p_total, p_enc_in_filters, p_enc_ex_filters, p_key, p_seed, false);
}
return _save_pack_file(p_userdata, p_path, p_data, p_file, p_total, p_enc_in_filters, p_enc_ex_filters, p_key, p_seed);
Vector<uint8_t> old_data = old_file->get_buffer(old_file->get_length());
// We can't rely on the MD5 as stored in the PCKs, since delta patches could have made it stale.
if (p_data == old_data) {
return OK; // Do nothing if the file hasn't changed.
}
if (!p_preset->is_patch_delta_encoding_enabled()) {
return _save_pack_file(p_preset, p_userdata, p_path, p_data, p_file, p_total, p_enc_in_filters, p_enc_ex_filters, p_key, p_seed, false);
}
bool delta = false;
for (const String &filter : p_preset->get_patch_delta_include_filter().split(",", false)) {
String filter_stripped = filter.strip_edges();
if (p_path.matchn(filter_stripped) || p_path.trim_prefix("res://").matchn(filter_stripped)) {
delta = true;
break;
}
}
for (const String &filter : p_preset->get_patch_delta_exclude_filter().split(",", false)) {
String filter_stripped = filter.strip_edges();
if (p_path.matchn(filter_stripped) || p_path.trim_prefix("res://").matchn(filter_stripped)) {
delta = false;
break;
}
}
Vector<uint8_t> patch_data = p_data;
if (delta) {
Error err = DeltaEncoding::encode_delta(old_data, p_data, patch_data, p_preset->get_patch_delta_zstd_level());
if (err != OK) {
return err;
}
int64_t reduction_bytes = MAX(0, p_data.size() - patch_data.size());
double reduction_ratio = reduction_bytes / (double)p_data.size();
if (reduction_ratio >= p_preset->get_patch_delta_min_reduction()) {
print_verbose(vformat("Used delta encoding for patch of \"%s\", resulting in a patch of %d bytes, which reduced the size by %.1f%% (%d bytes) compared to the actual file.", p_path, patch_data.size(), reduction_ratio * 100, reduction_bytes));
} else {
print_verbose(vformat("Skipped delta encoding for patch of \"%s\", as it resulted in a patch of %d bytes, which only reduced the size by %.1f%% (%d bytes) compared to the actual file.", p_path, patch_data.size(), reduction_ratio * 100, reduction_bytes));
patch_data = p_data;
delta = false;
}
} else {
print_verbose(vformat("Skipped delta encoding for patch of \"%s\", due to include/exclude filters.", p_path));
}
return _save_pack_file(p_preset, p_userdata, p_path, patch_data, p_file, p_total, p_enc_in_filters, p_enc_ex_filters, p_key, p_seed, delta);
}
Error EditorExportPlatform::_save_zip_file(void *p_userdata, const String &p_path, const Vector<uint8_t> &p_data, int p_file, int p_total, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed) {
Error EditorExportPlatform::_save_zip_file(const Ref<EditorExportPreset> &p_preset, void *p_userdata, const String &p_path, const Vector<uint8_t> &p_data, int p_file, int p_total, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed, bool p_delta) {
ERR_FAIL_COND_V_MSG(p_total < 1, ERR_PARAMETER_RANGE_ERROR, "Must select at least one file to export.");
const String path = simplify_path(p_path).replace_first("res://", "");
@ -403,12 +437,18 @@ Error EditorExportPlatform::_save_zip_file(void *p_userdata, const String &p_pat
return OK;
}
Error EditorExportPlatform::_save_zip_patch_file(void *p_userdata, const String &p_path, const Vector<uint8_t> &p_data, int p_file, int p_total, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed) {
if (_check_hash(PackedData::get_singleton()->get_file_hash(p_path), p_data)) {
return OK;
Error EditorExportPlatform::_save_zip_patch_file(const Ref<EditorExportPreset> &p_preset, void *p_userdata, const String &p_path, const Vector<uint8_t> &p_data, int p_file, int p_total, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed, bool p_delta) {
Ref<FileAccess> old_file = PackedData::get_singleton()->try_open_path(p_path);
if (old_file.is_valid()) {
Vector<uint8_t> old_data = old_file->get_buffer(old_file->get_length());
// We can't rely on the MD5 as stored in the PCKs, since delta patches could have made it stale.
if (p_data == old_data) {
return OK; // Do nothing if the file hasn't changed.
}
}
return _save_zip_file(p_userdata, p_path, p_data, p_file, p_total, p_enc_in_filters, p_enc_ex_filters, p_key, p_seed);
return _save_zip_file(p_preset, p_userdata, p_path, p_data, p_file, p_total, p_enc_in_filters, p_enc_ex_filters, p_key, p_seed, p_delta);
}
Ref<Texture2D> EditorExportPlatform::get_option_icon(int p_index) const {
@ -1036,7 +1076,7 @@ Vector<String> EditorExportPlatform::get_forced_export_files(const Ref<EditorExp
return files;
}
Error EditorExportPlatform::_script_save_file(void *p_userdata, const String &p_path, const Vector<uint8_t> &p_data, int p_file, int p_total, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed) {
Error EditorExportPlatform::_script_save_file(const Ref<EditorExportPreset> &p_preset, void *p_userdata, const String &p_path, const Vector<uint8_t> &p_data, int p_file, int p_total, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed, bool p_delta) {
Callable cb = ((ScriptCallbackData *)p_userdata)->file_cb;
ERR_FAIL_COND_V(!cb.is_valid(), FAILED);
@ -1060,7 +1100,7 @@ Error EditorExportPlatform::_script_save_file(void *p_userdata, const String &p_
return (Error)ret.operator int();
}
Error EditorExportPlatform::_script_add_shared_object(void *p_userdata, const SharedObject &p_so) {
Error EditorExportPlatform::_script_add_shared_object(const Ref<EditorExportPreset> &p_preset, void *p_userdata, const SharedObject &p_so) {
Callable cb = ((ScriptCallbackData *)p_userdata)->so_cb;
if (!cb.is_valid()) {
return OK; // Optional.
@ -1216,14 +1256,14 @@ Error EditorExportPlatform::export_project_files(const Ref<EditorExportPreset> &
for (int i = 0; i < export_plugins.size(); i++) {
if (p_so_func) {
for (int j = 0; j < export_plugins[i]->shared_objects.size(); j++) {
err = p_so_func(p_udata, export_plugins[i]->shared_objects[j]);
err = p_so_func(p_preset, p_udata, export_plugins[i]->shared_objects[j]);
if (err != OK) {
return err;
}
}
}
for (int j = 0; j < export_plugins[i]->extra_files.size(); j++) {
err = save_proxy.save_file(p_udata, export_plugins[i]->extra_files[j].path, export_plugins[i]->extra_files[j].data, 0, paths.size(), enc_in_filters, enc_ex_filters, key, seed);
err = save_proxy.save_file(p_preset, p_udata, export_plugins[i]->extra_files[j].path, export_plugins[i]->extra_files[j].data, 0, paths.size(), enc_in_filters, enc_ex_filters, key, seed, false);
if (err != OK) {
return err;
}
@ -1315,9 +1355,8 @@ Error EditorExportPlatform::export_project_files(const Ref<EditorExportPreset> &
// for continue statements without accidentally skipping an increment.
int idx = total > 0 ? -1 : 0;
for (const String &E : paths) {
for (const String &path : paths) {
idx++;
String path = E;
String type = ResourceLoader::get_resource_type(path);
bool has_import_file = FileAccess::exists(path + ".import");
@ -1347,7 +1386,7 @@ Error EditorExportPlatform::export_project_files(const Ref<EditorExportPreset> &
}
if (p_so_func) {
for (int j = 0; j < export_plugins[i]->shared_objects.size(); j++) {
err = p_so_func(p_udata, export_plugins[i]->shared_objects[j]);
err = p_so_func(p_preset, p_udata, export_plugins[i]->shared_objects[j]);
if (err != OK) {
return err;
}
@ -1355,7 +1394,7 @@ Error EditorExportPlatform::export_project_files(const Ref<EditorExportPreset> &
}
for (int j = 0; j < export_plugins[i]->extra_files.size(); j++) {
err = save_proxy.save_file(p_udata, export_plugins[i]->extra_files[j].path, export_plugins[i]->extra_files[j].data, idx, total, enc_in_filters, enc_ex_filters, key, seed);
err = save_proxy.save_file(p_preset, p_udata, export_plugins[i]->extra_files[j].path, export_plugins[i]->extra_files[j].data, idx, total, enc_in_filters, enc_ex_filters, key, seed, false);
if (err != OK) {
return err;
}
@ -1385,7 +1424,7 @@ Error EditorExportPlatform::export_project_files(const Ref<EditorExportPreset> &
if (importer_type == "keep") {
// Just keep file as-is.
Vector<uint8_t> array = FileAccess::get_file_as_bytes(path);
err = save_proxy.save_file(p_udata, path, array, idx, total, enc_in_filters, enc_ex_filters, key, seed);
err = save_proxy.save_file(p_preset, p_udata, path, array, idx, total, enc_in_filters, enc_ex_filters, key, seed, false);
if (err != OK) {
return err;
@ -1427,13 +1466,13 @@ Error EditorExportPlatform::export_project_files(const Ref<EditorExportPreset> &
sarr.resize(cs.size());
memcpy(sarr.ptrw(), cs.ptr(), sarr.size());
err = save_proxy.save_file(p_udata, path + ".import", sarr, idx, total, enc_in_filters, enc_ex_filters, key, seed);
err = save_proxy.save_file(p_preset, p_udata, path + ".import", sarr, idx, total, enc_in_filters, enc_ex_filters, key, seed, false);
if (err != OK) {
return err;
}
// Now actual remapped file:
sarr = FileAccess::get_file_as_bytes(export_path);
err = save_proxy.save_file(p_udata, export_path, sarr, idx, total, enc_in_filters, enc_ex_filters, key, seed);
err = save_proxy.save_file(p_preset, p_udata, export_path, sarr, idx, total, enc_in_filters, enc_ex_filters, key, seed, false);
if (err != OK) {
return err;
}
@ -1461,14 +1500,14 @@ Error EditorExportPlatform::export_project_files(const Ref<EditorExportPreset> &
if (remap == "path") {
String remapped_path = config->get_value("remap", remap);
Vector<uint8_t> array = FileAccess::get_file_as_bytes(remapped_path);
err = save_proxy.save_file(p_udata, remapped_path, array, idx, total, enc_in_filters, enc_ex_filters, key, seed);
err = save_proxy.save_file(p_preset, p_udata, remapped_path, array, idx, total, enc_in_filters, enc_ex_filters, key, seed, false);
} else if (remap.begins_with("path.")) {
String feature = remap.get_slicec('.', 1);
if (remap_features.has(feature)) {
String remapped_path = config->get_value("remap", remap);
Vector<uint8_t> array = FileAccess::get_file_as_bytes(remapped_path);
err = save_proxy.save_file(p_udata, remapped_path, array, idx, total, enc_in_filters, enc_ex_filters, key, seed);
err = save_proxy.save_file(p_preset, p_udata, remapped_path, array, idx, total, enc_in_filters, enc_ex_filters, key, seed, false);
} else {
// Remove paths if feature not enabled.
config->erase_section_key("remap", remap);
@ -1494,7 +1533,7 @@ Error EditorExportPlatform::export_project_files(const Ref<EditorExportPreset> &
sarr.resize(cs.size());
memcpy(sarr.ptrw(), cs.ptr(), sarr.size());
err = save_proxy.save_file(p_udata, path + ".import", sarr, idx, total, enc_in_filters, enc_ex_filters, key, seed);
err = save_proxy.save_file(p_preset, p_udata, path + ".import", sarr, idx, total, enc_in_filters, enc_ex_filters, key, seed, false);
if (err != OK) {
return err;
@ -1515,7 +1554,7 @@ Error EditorExportPlatform::export_project_files(const Ref<EditorExportPreset> &
}
Vector<uint8_t> array = FileAccess::get_file_as_bytes(export_path);
err = save_proxy.save_file(p_udata, export_path, array, idx, total, enc_in_filters, enc_ex_filters, key, seed);
err = save_proxy.save_file(p_preset, p_udata, export_path, array, idx, total, enc_in_filters, enc_ex_filters, key, seed, false);
if (err != OK) {
return err;
}
@ -1584,7 +1623,7 @@ Error EditorExportPlatform::export_project_files(const Ref<EditorExportPreset> &
new_file.write[j] = utf8[j];
}
err = save_proxy.save_file(p_udata, from + ".remap", new_file, idx, total, enc_in_filters, enc_ex_filters, key, seed);
err = save_proxy.save_file(p_preset, p_udata, from + ".remap", new_file, idx, total, enc_in_filters, enc_ex_filters, key, seed, false);
if (err != OK) {
return err;
}
@ -1602,7 +1641,7 @@ Error EditorExportPlatform::export_project_files(const Ref<EditorExportPreset> &
} else {
array = FileAccess::get_file_as_bytes(forced_export[i]);
}
err = save_proxy.save_file(p_udata, forced_export[i], array, idx, total, enc_in_filters, enc_ex_filters, key, seed);
err = save_proxy.save_file(p_preset, p_udata, forced_export[i], array, idx, total, enc_in_filters, enc_ex_filters, key, seed, false);
if (err != OK) {
return err;
}
@ -1611,7 +1650,7 @@ Error EditorExportPlatform::export_project_files(const Ref<EditorExportPreset> &
Dictionary int_export = get_internal_export_files(p_preset, p_debug);
for (const KeyValue<Variant, Variant> &int_export_kv : int_export) {
const PackedByteArray &array = int_export_kv.value;
err = save_proxy.save_file(p_udata, int_export_kv.key, array, idx, total, enc_in_filters, enc_ex_filters, key, seed);
err = save_proxy.save_file(p_preset, p_udata, int_export_kv.key, array, idx, total, enc_in_filters, enc_ex_filters, key, seed, false);
if (err != OK) {
return err;
}
@ -1624,7 +1663,7 @@ Error EditorExportPlatform::export_project_files(const Ref<EditorExportPreset> &
Vector<uint8_t> data = FileAccess::get_file_as_bytes(engine_cfb);
DirAccess::remove_file_or_error(engine_cfb);
err = save_proxy.save_file(p_udata, "res://" + config_file, data, idx, total, enc_in_filters, enc_ex_filters, key, seed);
err = save_proxy.save_file(p_preset, p_udata, "res://" + config_file, data, idx, total, enc_in_filters, enc_ex_filters, key, seed, false);
if (err != OK) {
return err;
}
@ -1633,7 +1672,7 @@ Error EditorExportPlatform::export_project_files(const Ref<EditorExportPreset> &
HashSet<String> currently_loaded_paths = PackedData::get_singleton()->get_file_paths();
for (const String &path : currently_loaded_paths) {
if (!save_proxy.has_saved(path)) {
err = p_remove_func(p_udata, path);
err = p_remove_func(p_preset, p_udata, path);
if (err != OK) {
return err;
}
@ -1660,7 +1699,7 @@ Vector<uint8_t> EditorExportPlatform::_filter_extension_list_config_file(const S
return data;
}
Error EditorExportPlatform::_pack_add_shared_object(void *p_userdata, const SharedObject &p_so) {
Error EditorExportPlatform::_pack_add_shared_object(const Ref<EditorExportPreset> &p_preset, void *p_userdata, const SharedObject &p_so) {
PackData *pack_data = (PackData *)p_userdata;
if (pack_data->so_files) {
pack_data->so_files->push_back(p_so);
@ -1669,7 +1708,7 @@ Error EditorExportPlatform::_pack_add_shared_object(void *p_userdata, const Shar
return OK;
}
Error EditorExportPlatform::_remove_pack_file(void *p_userdata, const String &p_path) {
Error EditorExportPlatform::_remove_pack_file(const Ref<EditorExportPreset> &p_preset, void *p_userdata, const String &p_path) {
PackData *pd = (PackData *)p_userdata;
SavedData sd;
@ -1691,7 +1730,7 @@ Error EditorExportPlatform::_remove_pack_file(void *p_userdata, const String &p_
return OK;
}
Error EditorExportPlatform::_zip_add_shared_object(void *p_userdata, const SharedObject &p_so) {
Error EditorExportPlatform::_zip_add_shared_object(const Ref<EditorExportPreset> &p_preset, void *p_userdata, const SharedObject &p_so) {
ZipData *zip_data = (ZipData *)p_userdata;
if (zip_data->so_files) {
zip_data->so_files->push_back(p_so);
@ -1996,6 +2035,9 @@ bool EditorExportPlatform::_encrypt_and_store_directory(Ref<FileAccess> p_fd, Pa
if (p_pack_data.file_ofs[i].removal) {
flags |= PACK_FILE_REMOVAL;
}
if (p_pack_data.file_ofs[i].delta) {
flags |= PACK_FILE_DELTA;
}
fhead->store_32(flags);
}

27
editor/export/editor_export_platform.h
View File

@ -53,9 +53,9 @@ protected:
static void _bind_methods();
public:
typedef Error (*EditorExportSaveFunction)(void *p_userdata, const String &p_path, const Vector<uint8_t> &p_data, int p_file, int p_total, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed);
typedef Error (*EditorExportRemoveFunction)(void *p_userdata, const String &p_path);
typedef Error (*EditorExportSaveSharedObject)(void *p_userdata, const SharedObject &p_so);
typedef Error (*EditorExportSaveFunction)(const Ref<EditorExportPreset> &p_preset, void *p_userdata, const String &p_path, const Vector<uint8_t> &p_data, int p_file, int p_total, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed, bool p_delta);
typedef Error (*EditorExportRemoveFunction)(const Ref<EditorExportPreset> &p_preset, void *p_userdata, const String &p_path);
typedef Error (*EditorExportSaveSharedObject)(const Ref<EditorExportPreset> &p_preset, void *p_userdata, const SharedObject &p_so);
enum DebugFlags {
DEBUG_FLAG_DUMB_CLIENT = 1,
@ -83,6 +83,7 @@ public:
uint64_t size = 0;
bool encrypted = false;
bool removal = false;
bool delta = false;
Vector<uint8_t> md5;
CharString path_utf8;
@ -119,25 +120,23 @@ private:
void _export_find_customized_resources(const Ref<EditorExportPreset> &p_preset, EditorFileSystemDirectory *p_dir, EditorExportPreset::FileExportMode p_mode, HashSet<String> &p_paths);
void _export_find_dependencies(const String &p_path, HashSet<String> &p_paths);
static bool _check_hash(const uint8_t *p_hash, const Vector<uint8_t> &p_data);
static Error _save_pack_file(const Ref<EditorExportPreset> &p_preset, void *p_userdata, const String &p_path, const Vector<uint8_t> &p_data, int p_file, int p_total, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed, bool p_delta);
static Error _save_pack_patch_file(const Ref<EditorExportPreset> &p_preset, void *p_userdata, const String &p_path, const Vector<uint8_t> &p_data, int p_file, int p_total, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed, bool p_delta);
static Error _pack_add_shared_object(const Ref<EditorExportPreset> &p_preset, void *p_userdata, const SharedObject &p_so);
static Error _save_pack_file(void *p_userdata, const String &p_path, const Vector<uint8_t> &p_data, int p_file, int p_total, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed);
static Error _save_pack_patch_file(void *p_userdata, const String &p_path, const Vector<uint8_t> &p_data, int p_file, int p_total, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed);
static Error _pack_add_shared_object(void *p_userdata, const SharedObject &p_so);
static Error _remove_pack_file(const Ref<EditorExportPreset> &p_preset, void *p_userdata, const String &p_path);
static Error _remove_pack_file(void *p_userdata, const String &p_path);
static Error _save_zip_file(void *p_userdata, const String &p_path, const Vector<uint8_t> &p_data, int p_file, int p_total, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed);
static Error _save_zip_patch_file(void *p_userdata, const String &p_path, const Vector<uint8_t> &p_data, int p_file, int p_total, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed);
static Error _zip_add_shared_object(void *p_userdata, const SharedObject &p_so);
static Error _save_zip_file(const Ref<EditorExportPreset> &p_preset, void *p_userdata, const String &p_path, const Vector<uint8_t> &p_data, int p_file, int p_total, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed, bool p_delta);
static Error _save_zip_patch_file(const Ref<EditorExportPreset> &p_preset, void *p_userdata, const String &p_path, const Vector<uint8_t> &p_data, int p_file, int p_total, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed, bool p_delta);
static Error _zip_add_shared_object(const Ref<EditorExportPreset> &p_preset, void *p_userdata, const SharedObject &p_so);
struct ScriptCallbackData {
Callable file_cb;
Callable so_cb;
};
static Error _script_save_file(void *p_userdata, const String &p_path, const Vector<uint8_t> &p_data, int p_file, int p_total, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed);
static Error _script_add_shared_object(void *p_userdata, const SharedObject &p_so);
static Error _script_save_file(const Ref<EditorExportPreset> &p_preset, void *p_userdata, const String &p_path, const Vector<uint8_t> &p_data, int p_file, int p_total, const Vector<String> &p_enc_in_filters, const Vector<String> &p_enc_ex_filters, const Vector<uint8_t> &p_key, uint64_t p_seed, bool p_delta);
static Error _script_add_shared_object(const Ref<EditorExportPreset> &p_preset, void *p_userdata, const SharedObject &p_so);
void _edit_files_with_filter(Ref<DirAccess> &da, const Vector<String> &p_filters, HashSet<String> &r_list, bool exclude);
void _edit_filter_list(HashSet<String> &r_list, const String &p_filter, bool exclude);

48
editor/export/editor_export_preset.cpp
View File

@ -28,10 +28,11 @@
/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */
/**************************************************************************/
#include "editor_export.h"
#include "editor_export_preset.h"
#include "core/config/project_settings.h"
#include "core/io/dir_access.h"
#include "editor/export/editor_export.h"
#include "editor/settings/editor_settings.h"
bool EditorExportPreset::_set(const StringName &p_name, const Variant &p_value) {
@ -440,6 +441,51 @@ Vector<String> EditorExportPreset::get_patches() const {
return patches;
}
void EditorExportPreset::set_patch_delta_encoding_enabled(bool p_enable) {
patch_delta_encoding_enabled = p_enable;
EditorExport::singleton->save_presets();
}
bool EditorExportPreset::is_patch_delta_encoding_enabled() const {
return patch_delta_encoding_enabled;
}
void EditorExportPreset::set_patch_delta_zstd_level(int p_level) {
patch_delta_zstd_level = p_level;
EditorExport::singleton->save_presets();
}
int EditorExportPreset::get_patch_delta_zstd_level() const {
return patch_delta_zstd_level;
}
void EditorExportPreset::set_patch_delta_min_reduction(double p_ratio) {
patch_delta_min_reduction = p_ratio;
EditorExport::singleton->save_presets();
}
double EditorExportPreset::get_patch_delta_min_reduction() const {
return patch_delta_min_reduction;
}
void EditorExportPreset::set_patch_delta_include_filter(const String &p_filter) {
patch_delta_include_filter = p_filter;
EditorExport::singleton->save_presets();
}
String EditorExportPreset::get_patch_delta_include_filter() const {
return patch_delta_include_filter;
}
void EditorExportPreset::set_patch_delta_exclude_filter(const String &p_filter) {
patch_delta_exclude_filter = p_filter;
EditorExport::singleton->save_presets();
}
String EditorExportPreset::get_patch_delta_exclude_filter() const {
return patch_delta_exclude_filter;
}
void EditorExportPreset::set_custom_features(const String &p_custom_features) {
custom_features = p_custom_features;
EditorExport::singleton->save_presets();

22
editor/export/editor_export_preset.h
View File

@ -73,6 +73,11 @@ private:
bool dedicated_server = false;
Vector<String> patches;
bool patch_delta_encoding_enabled = false;
int patch_delta_zstd_level = 19;
double patch_delta_min_reduction = 0.1;
String patch_delta_include_filter = "*";
String patch_delta_exclude_filter;
friend class EditorExport;
friend class EditorExportPlatform;
@ -148,11 +153,28 @@ public:
void add_patch(const String &p_path, int p_at_pos = -1);
void set_patch(int p_index, const String &p_path);
String get_patch(int p_index);
void remove_patch(int p_index);
void set_patches(const Vector<String> &p_patches);
Vector<String> get_patches() const;
void set_patch_delta_encoding_enabled(bool p_enable);
bool is_patch_delta_encoding_enabled() const;
void set_patch_delta_zstd_level(int p_level);
int get_patch_delta_zstd_level() const;
void set_patch_delta_min_reduction(double p_ratio);
double get_patch_delta_min_reduction() const;
void set_patch_delta_include_filter(const String &p_filter);
String get_patch_delta_include_filter() const;
void set_patch_delta_exclude_filter(const String &p_filter);
String get_patch_delta_exclude_filter() const;
void set_custom_features(const String &p_custom_features);
String get_custom_features() const;

135
editor/export/project_export.cpp
View File

@ -51,11 +51,14 @@
#include "scene/gui/option_button.h"
#include "scene/gui/popup_menu.h"
#include "scene/gui/rich_text_label.h"
#include "scene/gui/spin_box.h"
#include "scene/gui/split_container.h"
#include "scene/gui/tab_container.h"
#include "scene/gui/texture_rect.h"
#include "scene/gui/tree.h"
#include <zstd.h>
void ProjectExportTextureFormatError::_on_fix_texture_format_pressed() {
export_dialog->hide();
ProjectSettingsEditor *project_settings = EditorNode::get_singleton()->get_project_settings();
@ -301,6 +304,19 @@ void ProjectExportDialog::_edit_preset(int p_index) {
exclude_filters->set_text(current->get_exclude_filter());
server_strip_message->set_visible(current->get_export_filter() == EditorExportPreset::EXPORT_CUSTOMIZED);
bool patch_delta_encoding_enabled = current->is_patch_delta_encoding_enabled();
patch_delta_encoding->set_pressed(patch_delta_encoding_enabled);
patch_delta_zstd_level->set_editable(patch_delta_encoding_enabled);
patch_delta_zstd_level->set_value(current->get_patch_delta_zstd_level());
patch_delta_min_reduction->set_editable(patch_delta_encoding_enabled);
patch_delta_min_reduction->set_value(current->get_patch_delta_min_reduction() * 100);
patch_delta_include_filter->set_editable(patch_delta_encoding_enabled);
patch_delta_exclude_filter->set_editable(patch_delta_encoding_enabled);
if (!updating_patch_delta_filters) {
patch_delta_include_filter->set_text(current->get_patch_delta_include_filter());
patch_delta_exclude_filter->set_text(current->get_patch_delta_exclude_filter());
}
patches->clear();
TreeItem *patch_root = patches->create_item();
Vector<String> patch_list = current->get_patches();
@ -745,6 +761,11 @@ void ProjectExportDialog::_duplicate_preset() {
preset->set_include_filter(current->get_include_filter());
preset->set_exclude_filter(current->get_exclude_filter());
preset->set_patches(current->get_patches());
preset->set_patch_delta_encoding_enabled(current->is_patch_delta_encoding_enabled());
preset->set_patch_delta_zstd_level(current->get_patch_delta_zstd_level());
preset->set_patch_delta_min_reduction(current->get_patch_delta_min_reduction());
preset->set_patch_delta_include_filter(current->get_patch_delta_include_filter());
preset->set_patch_delta_exclude_filter(current->get_patch_delta_exclude_filter());
preset->set_custom_features(current->get_custom_features());
preset->set_enc_in_filter(current->get_enc_in_filter());
preset->set_enc_ex_filter(current->get_enc_ex_filter());
@ -1201,6 +1222,75 @@ void ProjectExportDialog::_set_file_export_mode(int p_id) {
_propagate_file_export_mode(include_files->get_root(), EditorExportPreset::MODE_FILE_NOT_CUSTOMIZED);
}
void ProjectExportDialog::_patch_delta_encoding_changed(bool p_pressed) {
if (updating) {
return;
}
Ref<EditorExportPreset> current = get_current_preset();
ERR_FAIL_COND(current.is_null());
current->set_patch_delta_encoding_enabled(p_pressed);
_update_current_preset();
}
void ProjectExportDialog::_patch_delta_include_filter_changed(const String &p_filter) {
if (updating) {
return;
}
Ref<EditorExportPreset> current = get_current_preset();
ERR_FAIL_COND(current.is_null());
current->set_patch_delta_include_filter(patch_delta_include_filter->get_text());
updating_patch_delta_filters = true;
_update_current_preset();
updating_patch_delta_filters = false;
}
void ProjectExportDialog::_patch_delta_exclude_filter_changed(const String &p_filter) {
if (updating) {
return;
}
Ref<EditorExportPreset> current = get_current_preset();
ERR_FAIL_COND(current.is_null());
current->set_patch_delta_exclude_filter(patch_delta_exclude_filter->get_text());
updating_patch_delta_filters = true;
_update_current_preset();
updating_patch_delta_filters = false;
}
void ProjectExportDialog::_patch_delta_zstd_level_changed(double p_value) {
if (updating) {
return;
}
Ref<EditorExportPreset> current = get_current_preset();
ERR_FAIL_COND(current.is_null());
current->set_patch_delta_zstd_level((int)p_value);
_update_current_preset();
}
void ProjectExportDialog::_patch_delta_min_reduction_changed(double p_value) {
if (updating) {
return;
}
Ref<EditorExportPreset> current = get_current_preset();
ERR_FAIL_COND(current.is_null());
current->set_patch_delta_min_reduction(p_value / 100.0);
_update_current_preset();
}
void ProjectExportDialog::_patch_tree_button_clicked(Object *p_item, int p_column, int p_id, int p_mouse_button_index) {
TreeItem *ti = Object::cast_to<TreeItem>(p_item);
@ -1657,11 +1747,52 @@ ProjectExportDialog::ProjectExportDialog() {
exclude_filters);
exclude_filters->connect(SceneStringName(text_changed), callable_mp(this, &ProjectExportDialog::_filter_changed));
// Patch packages.
// Patching.
VBoxContainer *patch_vb = memnew(VBoxContainer);
sections->add_child(patch_vb);
patch_vb->set_name(TTR("Patches"));
patch_vb->set_name(TTRC("Patching"));
patch_delta_encoding = memnew(CheckButton);
patch_delta_encoding->connect(SceneStringName(toggled), callable_mp(this, &ProjectExportDialog::_patch_delta_encoding_changed));
patch_delta_encoding->set_text(TTRC("Enable Delta Encoding"));
patch_delta_encoding->set_tooltip_text(TTRC("If checked, any change to a file already present in the base packs will be exported as the difference between the old file and the new file.\n"
"Enabling this comes at the cost of longer export times as well as longer load times for patched resources."));
patch_vb->add_child(patch_delta_encoding);
patch_delta_zstd_level = memnew(SpinBox);
patch_delta_zstd_level->set_min(ZSTD_minCLevel());
patch_delta_zstd_level->set_max(ZSTD_maxCLevel());
patch_delta_zstd_level->set_step(1);
patch_delta_zstd_level->set_tooltip_text(
vformat(TTR("The Zstandard compression level to use when generating delta-encoded patches.\n"
"Higher positive levels will reduce patch sizes, at the cost of longer export time, but do not affect the time it takes to apply patches.\n"
"Negative levels will reduce the time it takes to apply patches, at the cost of worse compression.\n"
"Levels above 19 require more memory both during export and when applying patches, usually for very little benefit.\n"
"Level 0 will cause Zstandard to use its default compression level, which is currently level %d."),
ZSTD_CLEVEL_DEFAULT));
patch_delta_zstd_level->connect(SceneStringName(value_changed), callable_mp(this, &ProjectExportDialog::_patch_delta_zstd_level_changed));
patch_vb->add_margin_child(TTRC("Delta Encoding Compression Level"), patch_delta_zstd_level);
patch_delta_min_reduction = memnew(SpinBox);
patch_delta_min_reduction->set_min(0.0);
patch_delta_min_reduction->set_max(100.0);
patch_delta_min_reduction->set_step(1.0);
patch_delta_min_reduction->set_suffix("%");
patch_delta_min_reduction->set_tooltip_text(TTRC("How much smaller, when compared to the new file, a delta-encoded patch needs to be for it to be exported.\n"
"If the patch is not at least this much smaller, the new file will be exported as-is."));
patch_delta_min_reduction->connect(SceneStringName(value_changed), callable_mp(this, &ProjectExportDialog::_patch_delta_min_reduction_changed));
patch_vb->add_margin_child(TTRC("Delta Encoding Minimum Size Reduction"), patch_delta_min_reduction);
patch_delta_include_filter = memnew(LineEdit);
patch_delta_include_filter->set_accessibility_name(TTRC("Delta Encoding Include Filters"));
patch_delta_include_filter->connect(SceneStringName(text_changed), callable_mp(this, &ProjectExportDialog::_patch_delta_include_filter_changed));
patch_vb->add_margin_child(TTRC("Filters to include files/folders from being delta-encoded\n(comma-separated, e.g: *.gdc, scripts/*)"), patch_delta_include_filter);
patch_delta_exclude_filter = memnew(LineEdit);
patch_delta_exclude_filter->set_accessibility_name(TTRC("Delta Encoding Exclude Filters"));
patch_delta_exclude_filter->connect(SceneStringName(text_changed), callable_mp(this, &ProjectExportDialog::_patch_delta_exclude_filter_changed));
patch_vb->add_margin_child(TTRC("Filters to exclude files/folders from being delta-encoded\n(comma-separated, e.g: *.ctex, textures/*)"), patch_delta_exclude_filter);
patches = memnew(Tree);
patches->set_v_size_flags(Control::SIZE_EXPAND_FILL);

12
editor/export/project_export.h
View File

@ -46,6 +46,7 @@ class OptionButton;
class PopupMenu;
class ProjectExportDialog;
class RichTextLabel;
class SpinBox;
class TabContainer;
class Tree;
class TreeItem;
@ -107,6 +108,11 @@ class ProjectExportDialog : public ConfirmationDialog {
RBSet<String> feature_set;
CheckButton *patch_delta_encoding = nullptr;
SpinBox *patch_delta_zstd_level = nullptr;
SpinBox *patch_delta_min_reduction = nullptr;
LineEdit *patch_delta_include_filter = nullptr;
LineEdit *patch_delta_exclude_filter = nullptr;
Tree *patches = nullptr;
int patch_index = -1;
EditorFileDialog *patch_dialog = nullptr;
@ -158,6 +164,12 @@ class ProjectExportDialog : public ConfirmationDialog {
void _tree_popup_edited(bool p_arrow_clicked);
void _set_file_export_mode(int p_id);
bool updating_patch_delta_filters = false;
void _patch_delta_encoding_changed(bool p_pressed);
void _patch_delta_include_filter_changed(const String &p_filter);
void _patch_delta_exclude_filter_changed(const String &p_filter);
void _patch_delta_zstd_level_changed(double p_value);
void _patch_delta_min_reduction_changed(double p_value);
void _patch_tree_button_clicked(Object *p_item, int p_column, int p_id, int p_mouse_button_index);
void _patch_tree_item_edited();
void _patch_file_selected(const String &p_path);