Moto/UnrealPrototyping
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Tooling

Browse Source
This commit is contained in:
Rainer Leit
2026-09-25 17:02:24 +03:00
parent cc43ed8dc8
commit 9597629951
2149 changed files with 460234 additions and 1770 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Scripts/Authoring/collect_terrain_assets.py
+358
View File
@@ -0,0 +1,358 @@
"""Builds Content/Terrain/: the project's own ground, copied out of the asset packs so there is one place that
says what this world is made of. Driven entirely by RawContent/Terrain/ground.json; adding a substance is a line
in that file and a rerun.
UnrealEditor-Cmd.exe Salty.uproject -run=pythonscript -script=Scripts/Authoring/collect_terrain_assets.py
... --dry-run # say what would be copied and repointed, change nothing
Idempotent: an asset already at its destination is left alone, so a rerun after adding one line copies one
asset. Nothing is ever deleted from a pack, and nothing is moved: a pack stays exactly as it shipped.
The hard part is that a copy is not ownership. Duplicating a material function gives a function that still
samples the *pack's* textures, because the reference lives inside the graph and duplication does not rewrite
it. So after everything is copied, every copy is walked and repointed at its siblings: a texture sample gets
the copied texture, a function call gets the copied function, an instance gets the copied parent and an
override for every texture parameter that still points into a pack. Then the whole set is checked, and
anything still referencing outside Content/Terrain is reported rather than passed over in silence - that
report is the only honest answer to "can the pack be deleted yet".
"""
import json
import os
import sys
import unreal
HERE = os.path.dirname(os.path.abspath(__file__))
PROJECT_ROOT = os.path.normpath(os.path.join(HERE, "..", ".."))
GROUND_PATH = os.path.join(PROJECT_ROOT, "RawContent", "Terrain", "ground.json")
asset_lib = unreal.EditorAssetLibrary
def load_ground(path):
with open(path, "r", encoding="utf-8") as f:
data = json.load(f)
destination = data.get("destination", "/Game/Terrain").rstrip("/")
entries = []
for entry in data["assets"]:
source = entry["source"]
dest = f"{destination}/{entry.get('dest', source.rsplit('/', 1)[1])}"
entries.append((source, dest, entry.get("parameters") or {}))
return destination, entries
def copy_assets(entries, dry_run):
"""Every source to its destination, skipping what is already there. Returns {source path: dest path} for
the whole set, copied or not, which is the map everything downstream is repointed through."""
mapping = {}
copied, skipped = 0, 0
for source, dest, _ in entries:
mapping[source] = dest
if asset_lib.does_asset_exist(dest):
skipped += 1
continue
if not asset_lib.does_asset_exist(source):
raise RuntimeError(f"{source} not found; is the pack still in Content/?")
if dry_run:
unreal.log(f" would copy {source} -> {dest}")
copied += 1
continue
if not asset_lib.duplicate_asset(source, dest):
raise RuntimeError(f"could not copy {source} to {dest}")
unreal.log(f" copied {source} -> {dest}")
copied += 1
unreal.log(f"{copied} copied, {skipped} already there")
return mapping
def material_expressions(asset):
"""Every expression node in a Material or MaterialFunction.
It has to come from MaterialEditingLibrary: the `Expressions` property is protected on UMaterial and absent
on UMaterialFunction, so `get_editor_property` cannot reach either of them ("Property 'Expressions' ... is
protected and cannot be read"), and a first attempt that went that way found nothing and reported nothing
repointed. And a function needs its own call - `get_material_expressions` takes a Material and refuses a
MaterialFunction outright, which is what left the three layer functions still sampling the pack."""
getter = (unreal.MaterialEditingLibrary.get_material_function_expressions
if isinstance(asset, unreal.MaterialFunction)
else unreal.MaterialEditingLibrary.get_material_expressions)
try:
nodes = getter(asset)
except Exception as error:
unreal.log_warning(f"{asset.get_path_name()}: could not read its expressions ({error})")
return []
return [node for node in (nodes or []) if node is not None]
def repoint_property(node, property_name, mapping, dry_run):
"""One object-valued property of an expression, if it points at something we copied."""
try:
current = node.get_editor_property(property_name)
except Exception:
return None
if current is None:
return None
source = current.get_path_name().split(".")[0]
dest = mapping.get(source)
if dest is None or dest == source:
return None
if dry_run:
return f"{property_name}: {source} -> {dest}"
replacement = unreal.load_asset(dest)
if replacement is None:
raise RuntimeError(f"{dest} was copied but will not load")
node.set_editor_property(property_name, replacement)
return f"{property_name}: {source} -> {dest}"
# Expression property names that hold a reference to another asset. A texture sample and its parameter
# variants keep theirs in `texture`; a function call keeps its function in `material_function`.
REFERENCE_PROPERTIES = ("texture", "material_function")
def repoint_graphs(mapping, dry_run):
"""Every copied Material and MaterialFunction, repointed at the copies of whatever it used to reference."""
changed = 0
for dest in sorted(set(mapping.values())):
asset = unreal.load_asset(dest)
if not isinstance(asset, (unreal.Material, unreal.MaterialFunction)):
continue
nodes = material_expressions(asset)
if not nodes:
unreal.log_warning(f"{dest}: no expressions found, so nothing could be repointed inside it; "
f"check it by hand before trusting the ownership report")
continue
edits = []
for node in nodes:
for name in REFERENCE_PROPERTIES:
edit = repoint_property(node, name, mapping, dry_run)
if edit:
edits.append(edit)
if not edits:
continue
changed += len(edits)
unreal.log(f" {dest}: {len(edits)} reference(s) repointed")
for edit in edits:
unreal.log(f" {edit}")
if dry_run:
continue
if isinstance(asset, unreal.MaterialFunction):
unreal.MaterialEditingLibrary.update_material_function(asset)
else:
unreal.MaterialEditingLibrary.recompile_material(asset)
asset_lib.save_asset(dest)
unreal.log(f"{changed} graph reference(s) repointed")
def repoint_override_array(instance, mapping, dry_run):
"""Texture overrides stored on the instance, read straight off `texture_parameter_values` rather than
through the parent's parameter list, so an orphaned override is reached too. Returns the edits made."""
try:
values = instance.get_editor_property("texture_parameter_values")
except Exception as error:
unreal.log_warning(f"{instance.get_path_name()}: could not read its texture overrides ({error})")
return []
edits, rewritten = [], []
for entry in values or []:
current = entry.get_editor_property("parameter_value")
replacement = mapping.get(current.get_path_name().split(".")[0]) if current else None
if replacement and replacement != current.get_path_name().split(".")[0]:
edits.append(f"override {entry.get_editor_property('parameter_info').get_editor_property('name')}: "
f"{current.get_path_name().split('.')[0]} -> {replacement}")
if not dry_run:
entry.set_editor_property("parameter_value", unreal.load_asset(replacement))
rewritten.append(entry)
if edits and not dry_run:
instance.set_editor_property("texture_parameter_values", rewritten)
return edits
def repoint_instances(mapping, dry_run):
"""Every copied material instance: its parent, and an override for each texture parameter that still points
into a pack. An instance is the one place the engine lets Python set a texture reference directly, so this
is also the belt to the graph repointing's braces."""
changed = 0
for dest in sorted(set(mapping.values())):
asset = unreal.load_asset(dest)
if not isinstance(asset, unreal.MaterialInstanceConstant):
continue
edits = []
edit = repoint_property(asset, "parent", mapping, dry_run)
if edit:
edits.append(edit)
parent = asset.get_editor_property("parent")
if parent is not None:
for name in unreal.MaterialEditingLibrary.get_texture_parameter_names(parent):
current = unreal.MaterialEditingLibrary.get_material_instance_texture_parameter_value(asset, name)
if current is None:
continue
source = current.get_path_name().split(".")[0]
replacement = mapping.get(source)
if replacement is None or replacement == source:
continue
edits.append(f"parameter {name}: {source} -> {replacement}")
if not dry_run:
unreal.MaterialEditingLibrary.set_material_instance_texture_parameter_value(
asset, name, unreal.load_asset(replacement))
# And the override array itself. The loop above can only see parameters the *parent* still exposes, so
# an override left over for one it no longer does is invisible to it - which is how the instance went
# on referencing the pack's T_Rock_Shade_Variation after every parameter it shares with its parent had
# been repointed. Walking the array catches those too.
edits.extend(repoint_override_array(asset, mapping, dry_run))
if not edits:
continue
changed += len(edits)
unreal.log(f" {dest}: {len(edits)} reference(s) repointed")
for edit in edits:
unreal.log(f" {edit}")
if not dry_run:
unreal.MaterialEditingLibrary.update_material_instance(asset)
asset_lib.save_asset(dest)
unreal.log(f"{changed} instance reference(s) repointed")
def apply_parameters(entries, dry_run):
"""The `parameters` block of an entry, written onto the copy. A copy is the project's own asset, so this is
where it stops being a duplicate of the pack and starts being ours: the pack's instance is left exactly as
it shipped and the numbers that differ live in ground.json, next to the reason they differ.
Idempotent by comparison, not by memory: a value already set is not written again, so a rerun after an
unrelated edit reports nothing.
Every write is read back and checked. A misspelled parameter name is not an error to the engine - the
getter answers with a zero colour for a name that does not exist and the setter does nothing at all - so
without the read-back a typo here would look exactly like a successful run and leave the ground the colour
it already was."""
changed = 0
for _, dest, parameters in entries:
if not parameters:
continue
instance = unreal.load_asset(dest)
if not isinstance(instance, unreal.MaterialInstanceConstant):
raise RuntimeError(f"{dest} has parameters in ground.json but is a {type(instance).__name__}, "
f"not a material instance")
for kind, names in (("vector", parameters.get("vector") or {}), ("scalar", parameters.get("scalar") or {})):
for name, value in names.items():
wanted = unreal.LinearColor(*value) if kind == "vector" else float(value)
if read_parameter(instance, kind, name) == quantise(wanted):
continue
before = read_parameter(instance, kind, name)
if not dry_run:
write_parameter(instance, kind, name, wanted)
after = read_parameter(instance, kind, name)
if after != quantise(wanted):
raise RuntimeError(f"{dest}: setting {kind} parameter {name!r} to {quantise(wanted)} "
f"left it at {after}; is that the name the parent exposes?")
unreal.log(f" {dest}: {name} {before} -> {quantise(wanted)}")
changed += 1
if changed and not dry_run:
unreal.MaterialEditingLibrary.update_material_instance(instance)
asset_lib.save_asset(dest, only_if_is_dirty=False)
unreal.log(f"{changed} parameter(s) set from ground.json")
def quantise(value):
"""A value rounded to what a comparison should care about, so a float that survived a round trip through the
asset still equals what was asked for."""
if isinstance(value, unreal.LinearColor):
return tuple(round(getattr(value, c), 4) for c in ("r", "g", "b", "a"))
return round(float(value), 4)
def read_parameter(instance, kind, name):
getter = (unreal.MaterialEditingLibrary.get_material_instance_vector_parameter_value if kind == "vector"
else unreal.MaterialEditingLibrary.get_material_instance_scalar_parameter_value)
return quantise(getter(instance, name))
def write_parameter(instance, kind, name, value):
setter = (unreal.MaterialEditingLibrary.set_material_instance_vector_parameter_value if kind == "vector"
else unreal.MaterialEditingLibrary.set_material_instance_scalar_parameter_value)
setter(instance, name, value)
def refresh_all(mapping):
"""Recompile every copy, functions first, then materials, then instances, whether or not this run changed
anything. A material caches the textures its graph and its called functions reference, and an instance
caches the defaults it inherits; both caches were captured while those still pointed into the pack, and
repointing a function does not rewrite them. Without this pass the asset registry went on reporting five
pack textures under the master and one under the instance after every reference in both had been
repointed, and a rerun could not clear it because a rerun has nothing left to change."""
order = (unreal.MaterialFunction, unreal.Material, unreal.MaterialInstanceConstant)
refreshed = 0
for kind in order:
for dest in sorted(set(mapping.values())):
asset = unreal.load_asset(dest)
if not isinstance(asset, kind):
continue
if kind is unreal.MaterialFunction:
unreal.MaterialEditingLibrary.update_material_function(asset)
elif kind is unreal.Material:
unreal.MaterialEditingLibrary.recompile_material(asset)
else:
unreal.MaterialEditingLibrary.update_material_instance(asset)
# only_if_is_dirty=False: the point of the pass is to rewrite the saved package's import table, and
# a recompile does not always mark the package dirty, so the default would skip the one save that
# matters.
asset_lib.save_asset(dest, only_if_is_dirty=False)
refreshed += 1
unreal.log(f"{refreshed} material asset(s) recompiled and saved so their cached references are rebuilt")
def report_ownership(destination, mapping):
"""What Content/Terrain still needs from outside itself, read from the asset registry rather than from what
this script believes it did. A clean report means the packs could be deleted and the ground would still
render; anything listed is a reference that did not get repointed."""
registry = unreal.AssetRegistryHelpers.get_asset_registry()
# Without this the newly copied assets are not in the registry yet and every one of them comes back with no
# dependencies at all - which an earlier version read as "clean" and reported as standing on its own while
# the master material was still calling the pack's functions. An empty answer is unknown, not clean.
registry.scan_paths_synchronous([destination], force_rescan=True)
options = unreal.AssetRegistryDependencyOptions(include_soft_package_references=True,
include_hard_package_references=True)
outside, unknown = {}, []
for dest in sorted(set(mapping.values())):
dependencies = registry.get_dependencies(unreal.Name(dest), options)
if dependencies is None:
unknown.append(dest)
continue
leaks = sorted({str(name) for name in dependencies
if str(name).startswith("/Game/") and not str(name).startswith(destination + "/")})
if leaks:
outside[dest] = leaks
if unknown:
unreal.log_warning(f"{len(unknown)} asset(s) are not in the asset registry, so nothing can be said "
f"about what they reference: {', '.join(unknown)}")
if not outside:
verdict = "stands on its own" if not unknown else "has no known references outside itself, but see above"
unreal.log(f"{destination} {verdict}")
return
unreal.log_warning(f"{destination}: {len(outside)} asset(s) still reference something outside it")
for dest, leaks in outside.items():
unreal.log_warning(f" {dest} -> {', '.join(leaks)}")
unreal.log_warning("the packs cannot be deleted while those remain")
def main():
# UE passes its own arguments through sys.argv under -run=pythonscript, so only the flags this script knows
# are read out of it rather than handing the lot to argparse.
argv = sys.argv[1:]
dry_run = "--dry-run" in argv
ground = GROUND_PATH
if "--ground" in argv:
ground = argv[argv.index("--ground") + 1]
destination, entries = load_ground(ground)
unreal.log(f"{len(entries)} ground assets from {os.path.relpath(ground, PROJECT_ROOT)} into {destination}"
+ (" (dry run)" if dry_run else ""))
mapping = copy_assets(entries, dry_run)
repoint_graphs(mapping, dry_run)
repoint_instances(mapping, dry_run)
apply_parameters(entries, dry_run)
if not dry_run:
refresh_all(mapping)
unreal.EditorLoadingAndSavingUtils.save_dirty_packages(False, True)
report_ownership(destination, mapping)
main()