C#: Move marshaling logic and generated glue to C#

We will be progressively moving most code to C#.
The plan is to only use Mono's embedding APIs to set things at launch.
This will make it much easier to later support CoreCLR too which
doesn't have rich embedding APIs.

Additionally the code in C# is more maintainable and makes it easier
to implement new features, e.g.: runtime codegen which we could use to
avoid using reflection for marshaling everytime a field, property or
method is accessed.

SOME NOTES ON INTEROP

We make the same assumptions as GDNative about the size of the Godot
structures we use. We take it a bit further by also assuming the layout
of fields in some cases, which is riskier but let's us squeeze out some
performance by avoiding unnecessary managed to native calls.

Code that deals with native structs is less safe than before as there's
no RAII and copy constructors in C#. It's like using the GDNative C API
directly. One has to take special care to free values they own.
Perhaps we could use roslyn analyzers to check this, but I don't know
any that uses attributes to determine what's owned or borrowed.

As to why we maily use pointers for native structs instead of ref/out:
- AFAIK (and confirmed with a benchmark) ref/out are pinned
  during P/Invoke calls and that has a cost.
- Native struct fields can't be ref/out in the first place.
- A `using` local can't be passed as ref/out, only `in`. Calling a
  method or property on an `in` value makes a silent copy, so we want
  to avoid `in`.

REGARDING THE BUILD SYSTEM

There's no longer a `mono_glue=yes/no` SCons options. We no longer
need to build with `mono_glue=no`, generate the glue and then build
again with `mono_glue=yes`. We build only once and generate the glue
(which is in C# now).
However, SCons no longer builds the C# projects for us. Instead one
must run `build_assemblies.py`, e.g.:
```sh
%godot_src_root%/modules/mono/build_scripts/build_assemblies.py \
        --godot-output-dir=%godot_src_root%/bin \
        --godot-target=release_debug`
```
We could turn this into a custom build target, but I don't know how
to do that with SCons (it's possible with Meson).

OTHER NOTES

Most of the moved code doesn't follow the C# naming convention and
still has the word Mono in the names despite no longer dealing with
Mono's embedding APIs. This is just temporary while transitioning,
to make it easier to understand what was moved where.
This commit is contained in:
Ignacio Roldán Etcheverry
2021-05-03 15:21:06 +02:00
parent 0c5f254956
commit 124fbf95f8
71 changed files with 5715 additions and 5608 deletions

View File

@ -1,80 +0,0 @@
# Build the Godot API solution
import os
from SCons.Script import Dir
def build_api_solution(source, target, env):
# source and target elements are of type SCons.Node.FS.File, hence why we convert them to str
module_dir = env["module_dir"]
solution_path = os.path.join(module_dir, "glue/GodotSharp/GodotSharp.sln")
build_config = env["solution_build_config"]
extra_msbuild_args = ["/p:NoWarn=1591"] # Ignore missing documentation warnings
from .solution_builder import build_solution
build_solution(env, solution_path, build_config, extra_msbuild_args=extra_msbuild_args)
# Copy targets
core_src_dir = os.path.abspath(os.path.join(solution_path, os.pardir, "GodotSharp", "bin", build_config))
editor_src_dir = os.path.abspath(os.path.join(solution_path, os.pardir, "GodotSharpEditor", "bin", build_config))
dst_dir = os.path.abspath(os.path.join(str(target[0]), os.pardir))
if not os.path.isdir(dst_dir):
assert not os.path.isfile(dst_dir)
os.makedirs(dst_dir)
def copy_target(target_path):
from shutil import copy
filename = os.path.basename(target_path)
src_path = os.path.join(core_src_dir, filename)
if not os.path.isfile(src_path):
src_path = os.path.join(editor_src_dir, filename)
copy(src_path, target_path)
for scons_target in target:
copy_target(str(scons_target))
def build(env_mono):
assert env_mono["tools"]
target_filenames = [
"GodotSharp.dll",
"GodotSharp.pdb",
"GodotSharp.xml",
"GodotSharpEditor.dll",
"GodotSharpEditor.pdb",
"GodotSharpEditor.xml",
]
depend_cmd = []
for build_config in ["Debug", "Release"]:
output_dir = Dir("#bin").abspath
editor_api_dir = os.path.join(output_dir, "GodotSharp", "Api", build_config)
targets = [os.path.join(editor_api_dir, filename) for filename in target_filenames]
cmd = env_mono.CommandNoCache(
targets, depend_cmd, build_api_solution, module_dir=os.getcwd(), solution_build_config=build_config
)
env_mono.AlwaysBuild(cmd)
# Make the Release build of the API solution depend on the Debug build.
# We do this in order to prevent SCons from building them in parallel,
# which can freak out MSBuild. In many cases, one of the builds would
# hang indefinitely requiring a key to be pressed for it to continue.
depend_cmd = cmd
return depend_cmd

View File

@ -0,0 +1,306 @@
#!/usr/bin/python3
import os
import os.path
import shlex
import subprocess
from dataclasses import dataclass
def find_dotnet_cli():
if os.name == "nt":
for hint_dir in os.environ["PATH"].split(os.pathsep):
hint_dir = hint_dir.strip('"')
hint_path = os.path.join(hint_dir, "dotnet")
if os.path.isfile(hint_path) and os.access(hint_path, os.X_OK):
return hint_path
if os.path.isfile(hint_path + ".exe") and os.access(hint_path + ".exe", os.X_OK):
return hint_path + ".exe"
else:
for hint_dir in os.environ["PATH"].split(os.pathsep):
hint_dir = hint_dir.strip('"')
hint_path = os.path.join(hint_dir, "dotnet")
if os.path.isfile(hint_path) and os.access(hint_path, os.X_OK):
return hint_path
def find_msbuild_standalone_windows():
msbuild_tools_path = find_msbuild_tools_path_reg()
if msbuild_tools_path:
return os.path.join(msbuild_tools_path, "MSBuild.exe")
return None
def find_msbuild_mono_windows(mono_prefix):
assert mono_prefix is not None
mono_bin_dir = os.path.join(mono_prefix, "bin")
msbuild_mono = os.path.join(mono_bin_dir, "msbuild.bat")
if os.path.isfile(msbuild_mono):
return msbuild_mono
return None
def find_msbuild_mono_unix():
import sys
hint_dirs = []
if sys.platform == "darwin":
hint_dirs[:0] = [
"/Library/Frameworks/Mono.framework/Versions/Current/bin",
"/usr/local/var/homebrew/linked/mono/bin",
]
for hint_dir in hint_dirs:
hint_path = os.path.join(hint_dir, "msbuild")
if os.path.isfile(hint_path):
return hint_path
elif os.path.isfile(hint_path + ".exe"):
return hint_path + ".exe"
for hint_dir in os.environ["PATH"].split(os.pathsep):
hint_dir = hint_dir.strip('"')
hint_path = os.path.join(hint_dir, "msbuild")
if os.path.isfile(hint_path) and os.access(hint_path, os.X_OK):
return hint_path
if os.path.isfile(hint_path + ".exe") and os.access(hint_path + ".exe", os.X_OK):
return hint_path + ".exe"
return None
def find_msbuild_tools_path_reg():
import subprocess
program_files = os.getenv("PROGRAMFILES(X86)")
if not program_files:
program_files = os.getenv("PROGRAMFILES")
vswhere = os.path.join(program_files, "Microsoft Visual Studio", "Installer", "vswhere.exe")
vswhere_args = ["-latest", "-products", "*", "-requires", "Microsoft.Component.MSBuild"]
try:
lines = subprocess.check_output([vswhere] + vswhere_args).splitlines()
for line in lines:
parts = line.decode("utf-8").split(":", 1)
if len(parts) < 2 or parts[0] != "installationPath":
continue
val = parts[1].strip()
if not val:
raise ValueError("Value of `installationPath` entry is empty")
# Since VS2019, the directory is simply named "Current"
msbuild_dir = os.path.join(val, "MSBuild", "Current", "Bin")
if os.path.isdir(msbuild_dir):
return msbuild_dir
# Directory name "15.0" is used in VS 2017
return os.path.join(val, "MSBuild", "15.0", "Bin")
raise ValueError("Cannot find `installationPath` entry")
except ValueError as e:
print("Error reading output from vswhere: " + str(e))
except OSError:
pass # Fine, vswhere not found
except (subprocess.CalledProcessError, OSError):
pass
@dataclass
class ToolsLocation:
dotnet_cli: str = ""
msbuild_standalone: str = ""
msbuild_mono: str = ""
mono_bin_dir: str = ""
def find_any_msbuild_tool(mono_prefix):
# Preference order: dotnet CLI > Standalone MSBuild > Mono's MSBuild
# Find dotnet CLI
dotnet_cli = find_dotnet_cli()
if dotnet_cli:
return ToolsLocation(dotnet_cli=dotnet_cli)
# Find standalone MSBuild
if os.name == "nt":
msbuild_standalone = find_msbuild_standalone_windows()
if msbuild_standalone:
return ToolsLocation(msbuild_standalone=msbuild_standalone)
if mono_prefix:
# Find Mono's MSBuild
if os.name == "nt":
msbuild_mono = find_msbuild_mono_windows(mono_prefix)
if msbuild_mono:
return ToolsLocation(msbuild_mono=msbuild_mono)
else:
msbuild_mono = find_msbuild_mono_unix()
if msbuild_mono:
return ToolsLocation(msbuild_mono=msbuild_mono)
return None
def run_msbuild(tools: ToolsLocation, sln: str, msbuild_args: [str] = None):
if msbuild_args is None:
msbuild_args = []
using_msbuild_mono = False
# Preference order: dotnet CLI > Standalone MSBuild > Mono's MSBuild
if tools.dotnet_cli:
args = [tools.dotnet_cli, "msbuild"]
elif tools.msbuild_standalone:
args = [tools.msbuild_standalone]
elif tools.msbuild_mono:
args = [tools.msbuild_mono]
using_msbuild_mono = True
else:
raise RuntimeError("Path to MSBuild or dotnet CLI not provided.")
args += [sln]
if len(msbuild_args) > 0:
args += msbuild_args
print("Running MSBuild: ", " ".join(shlex.quote(arg) for arg in args), flush=True)
msbuild_env = os.environ.copy()
# Needed when running from Developer Command Prompt for VS
if "PLATFORM" in msbuild_env:
del msbuild_env["PLATFORM"]
if using_msbuild_mono:
# The (Csc/Vbc/Fsc)ToolExe environment variables are required when
# building with Mono's MSBuild. They must point to the batch files
# in Mono's bin directory to make sure they are executed with Mono.
msbuild_env.update(
{
"CscToolExe": os.path.join(tools.mono_bin_dir, "csc.bat"),
"VbcToolExe": os.path.join(tools.mono_bin_dir, "vbc.bat"),
"FscToolExe": os.path.join(tools.mono_bin_dir, "fsharpc.bat"),
}
)
return subprocess.call(args, env=msbuild_env)
def build_godot_api(msbuild_tool, module_dir, output_dir):
target_filenames = [
"GodotSharp.dll",
"GodotSharp.pdb",
"GodotSharp.xml",
"GodotSharpEditor.dll",
"GodotSharpEditor.pdb",
"GodotSharpEditor.xml",
]
for build_config in ["Debug", "Release"]:
editor_api_dir = os.path.join(output_dir, "GodotSharp", "Api", build_config)
targets = [os.path.join(editor_api_dir, filename) for filename in target_filenames]
sln = os.path.join(module_dir, "glue/GodotSharp/GodotSharp.sln")
exit_code = run_msbuild(
msbuild_tool,
sln=sln,
msbuild_args=["/restore", "/t:Build", "/p:Configuration=" + build_config, "/p:NoWarn=1591"],
)
if exit_code != 0:
return exit_code
# Copy targets
core_src_dir = os.path.abspath(os.path.join(sln, os.pardir, "GodotSharp", "bin", build_config))
editor_src_dir = os.path.abspath(os.path.join(sln, os.pardir, "GodotSharpEditor", "bin", build_config))
if not os.path.isdir(editor_api_dir):
assert not os.path.isfile(editor_api_dir)
os.makedirs(editor_api_dir)
def copy_target(target_path):
from shutil import copy
filename = os.path.basename(target_path)
src_path = os.path.join(core_src_dir, filename)
if not os.path.isfile(src_path):
src_path = os.path.join(editor_src_dir, filename)
print(f"Copying assembly to {target_path}...")
copy(src_path, target_path)
for scons_target in targets:
copy_target(scons_target)
return 0
def build_all(msbuild_tool, module_dir, output_dir, dev_debug, godot_platform):
# Godot API
exit_code = build_godot_api(msbuild_tool, module_dir, output_dir)
if exit_code != 0:
return exit_code
# GodotTools
sln = os.path.join(module_dir, "editor/GodotTools/GodotTools.sln")
args = ["/restore", "/t:Build", "/p:Configuration=" + ("Debug" if dev_debug else "Release")] + (
["/p:GodotPlatform=" + godot_platform] if godot_platform else []
)
exit_code = run_msbuild(msbuild_tool, sln=sln, msbuild_args=args)
if exit_code != 0:
return exit_code
# Godot.NET.Sdk
sln = os.path.join(module_dir, "editor/Godot.NET.Sdk/Godot.NET.Sdk.sln")
exit_code = run_msbuild(msbuild_tool, sln=sln, msbuild_args=["/restore", "/t:Build", "/p:Configuration=Release"])
if exit_code != 0:
return exit_code
return 0
def main():
import argparse
import sys
parser = argparse.ArgumentParser(description="Builds all Godot .NET solutions")
parser.add_argument("--godot-output-dir", type=str, required=True)
parser.add_argument(
"--dev-debug",
action="store_true",
default=False,
help="Build GodotTools and Godot.NET.Sdk with 'Configuration=Debug'",
)
parser.add_argument("--godot-platform", type=str, default="")
parser.add_argument("--mono-prefix", type=str, default="")
args = parser.parse_args()
this_script_dir = os.path.dirname(os.path.realpath(__file__))
module_dir = os.path.abspath(os.path.join(this_script_dir, os.pardir))
output_dir = os.path.abspath(args.godot_output_dir)
msbuild_tool = find_any_msbuild_tool(args.mono_prefix)
if msbuild_tool is None:
print("Unable to find MSBuild")
sys.exit(1)
exit_code = build_all(msbuild_tool, module_dir, output_dir, args.godot_platform, args.dev_debug)
sys.exit(exit_code)
if __name__ == "__main__":
main()

View File

@ -1,20 +0,0 @@
def generate_header(solution_dir, version_header_dst):
import os
latest_mtime = 0
for root, dirs, files in os.walk(solution_dir, topdown=True):
dirs[:] = [d for d in dirs if d not in ["Generated"]] # Ignored generated files
files = [f for f in files if f.endswith(".cs")]
for file in files:
filepath = os.path.join(root, file)
mtime = os.path.getmtime(filepath)
latest_mtime = mtime if mtime > latest_mtime else latest_mtime
glue_version = int(latest_mtime) # The latest modified time will do for now
with open(version_header_dst, "w") as version_header:
version_header.write("/* THIS FILE IS GENERATED DO NOT EDIT */\n")
version_header.write("#ifndef CS_GLUE_VERSION_H\n")
version_header.write("#define CS_GLUE_VERSION_H\n\n")
version_header.write("#define CS_GLUE_VERSION UINT32_C(" + str(glue_version) + ")\n")
version_header.write("\n#endif // CS_GLUE_VERSION_H\n")

View File

@ -1,55 +0,0 @@
# Build Godot.NET.Sdk solution
import os
from SCons.Script import Dir
def build_godot_net_sdk(source, target, env):
# source and target elements are of type SCons.Node.FS.File, hence why we convert them to str
module_dir = env["module_dir"]
solution_path = os.path.join(module_dir, "editor/Godot.NET.Sdk/Godot.NET.Sdk.sln")
build_config = "Release"
from .solution_builder import build_solution
extra_msbuild_args = ["/p:GodotPlatform=" + env["platform"]]
build_solution(env, solution_path, build_config, extra_msbuild_args)
# No need to copy targets. The Godot.NET.Sdk csproj takes care of copying them.
def get_nupkgs_versions(props_file):
import xml.etree.ElementTree as ET
tree = ET.parse(props_file)
root = tree.getroot()
return {
"Godot.NET.Sdk": root.find("./PropertyGroup/PackageVersion_Godot_NET_Sdk").text.strip(),
"Godot.SourceGenerators": root.find("./PropertyGroup/PackageVersion_Godot_SourceGenerators").text.strip(),
}
def build(env_mono):
assert env_mono["tools"]
output_dir = Dir("#bin").abspath
editor_tools_dir = os.path.join(output_dir, "GodotSharp", "Tools")
nupkgs_dir = os.path.join(editor_tools_dir, "nupkgs")
module_dir = os.getcwd()
nupkgs_versions = get_nupkgs_versions(os.path.join(module_dir, "SdkPackageVersions.props"))
target_filenames = [
"Godot.NET.Sdk.%s.nupkg" % nupkgs_versions["Godot.NET.Sdk"],
"Godot.SourceGenerators.%s.nupkg" % nupkgs_versions["Godot.SourceGenerators"],
]
targets = [os.path.join(nupkgs_dir, filename) for filename in target_filenames]
cmd = env_mono.CommandNoCache(targets, [], build_godot_net_sdk, module_dir=module_dir)
env_mono.AlwaysBuild(cmd)

View File

@ -1,38 +0,0 @@
# Build GodotTools solution
import os
from SCons.Script import Dir
def build_godot_tools(source, target, env):
# source and target elements are of type SCons.Node.FS.File, hence why we convert them to str
module_dir = env["module_dir"]
solution_path = os.path.join(module_dir, "editor/GodotTools/GodotTools.sln")
build_config = "Debug" if env["target"] == "debug" else "Release"
from .solution_builder import build_solution
extra_msbuild_args = ["/p:GodotPlatform=" + env["platform"]]
build_solution(env, solution_path, build_config, extra_msbuild_args)
# No need to copy targets. The GodotTools csproj takes care of copying them.
def build(env_mono, api_sln_cmd):
assert env_mono["tools"]
output_dir = Dir("#bin").abspath
editor_tools_dir = os.path.join(output_dir, "GodotSharp", "Tools")
target_filenames = ["GodotTools.dll"]
if env_mono["target"] == "debug":
target_filenames += ["GodotTools.pdb"]
targets = [os.path.join(editor_tools_dir, filename) for filename in target_filenames]
cmd = env_mono.CommandNoCache(targets, api_sln_cmd, build_godot_tools, module_dir=os.getcwd())
env_mono.AlwaysBuild(cmd)

View File

@ -1,145 +0,0 @@
import os
verbose = False
def find_dotnet_cli():
import os.path
if os.name == "nt":
for hint_dir in os.environ["PATH"].split(os.pathsep):
hint_dir = hint_dir.strip('"')
hint_path = os.path.join(hint_dir, "dotnet")
if os.path.isfile(hint_path) and os.access(hint_path, os.X_OK):
return hint_path
if os.path.isfile(hint_path + ".exe") and os.access(hint_path + ".exe", os.X_OK):
return hint_path + ".exe"
else:
for hint_dir in os.environ["PATH"].split(os.pathsep):
hint_dir = hint_dir.strip('"')
hint_path = os.path.join(hint_dir, "dotnet")
if os.path.isfile(hint_path) and os.access(hint_path, os.X_OK):
return hint_path
def find_msbuild_unix():
import os.path
import sys
hint_dirs = []
if sys.platform == "darwin":
hint_dirs[:0] = [
"/Library/Frameworks/Mono.framework/Versions/Current/bin",
"/usr/local/var/homebrew/linked/mono/bin",
]
for hint_dir in hint_dirs:
hint_path = os.path.join(hint_dir, "msbuild")
if os.path.isfile(hint_path):
return hint_path
elif os.path.isfile(hint_path + ".exe"):
return hint_path + ".exe"
for hint_dir in os.environ["PATH"].split(os.pathsep):
hint_dir = hint_dir.strip('"')
hint_path = os.path.join(hint_dir, "msbuild")
if os.path.isfile(hint_path) and os.access(hint_path, os.X_OK):
return hint_path
if os.path.isfile(hint_path + ".exe") and os.access(hint_path + ".exe", os.X_OK):
return hint_path + ".exe"
return None
def find_msbuild_windows(env):
from .mono_reg_utils import find_mono_root_dir, find_msbuild_tools_path_reg
mono_root = env["mono_prefix"] or find_mono_root_dir(env["bits"])
if not mono_root:
raise RuntimeError("Cannot find mono root directory")
mono_bin_dir = os.path.join(mono_root, "bin")
msbuild_mono = os.path.join(mono_bin_dir, "msbuild.bat")
msbuild_tools_path = find_msbuild_tools_path_reg()
if msbuild_tools_path:
return (os.path.join(msbuild_tools_path, "MSBuild.exe"), {})
if os.path.isfile(msbuild_mono):
# The (Csc/Vbc/Fsc)ToolExe environment variables are required when
# building with Mono's MSBuild. They must point to the batch files
# in Mono's bin directory to make sure they are executed with Mono.
mono_msbuild_env = {
"CscToolExe": os.path.join(mono_bin_dir, "csc.bat"),
"VbcToolExe": os.path.join(mono_bin_dir, "vbc.bat"),
"FscToolExe": os.path.join(mono_bin_dir, "fsharpc.bat"),
}
return (msbuild_mono, mono_msbuild_env)
return None
def run_command(command, args, env_override=None, name=None):
def cmd_args_to_str(cmd_args):
return " ".join([arg if not " " in arg else '"%s"' % arg for arg in cmd_args])
args = [command] + args
if name is None:
name = os.path.basename(command)
if verbose:
print("Running '%s': %s" % (name, cmd_args_to_str(args)))
import subprocess
try:
if env_override is None:
subprocess.check_call(args)
else:
subprocess.check_call(args, env=env_override)
except subprocess.CalledProcessError as e:
raise RuntimeError("'%s' exited with error code: %s" % (name, e.returncode))
def build_solution(env, solution_path, build_config, extra_msbuild_args=[]):
global verbose
verbose = env["verbose"]
msbuild_env = os.environ.copy()
# Needed when running from Developer Command Prompt for VS
if "PLATFORM" in msbuild_env:
del msbuild_env["PLATFORM"]
msbuild_args = []
dotnet_cli = find_dotnet_cli()
if dotnet_cli:
msbuild_path = dotnet_cli
msbuild_args += ["msbuild"] # `dotnet msbuild` command
else:
# Find MSBuild
if os.name == "nt":
msbuild_info = find_msbuild_windows(env)
if msbuild_info is None:
raise RuntimeError("Cannot find MSBuild executable")
msbuild_path = msbuild_info[0]
msbuild_env.update(msbuild_info[1])
else:
msbuild_path = find_msbuild_unix()
if msbuild_path is None:
raise RuntimeError("Cannot find MSBuild executable")
print("MSBuild path: " + msbuild_path)
# Build solution
msbuild_args += [solution_path, "/restore", "/t:Build", "/p:Configuration=" + build_config]
msbuild_args += extra_msbuild_args
run_command(msbuild_path, msbuild_args, env_override=msbuild_env, name="msbuild")