"""Builds Content/Terrain/: the project's own ground, copied out of the asset packs so there is one place that says what this world is made of. Driven entirely by RawContent/Terrain/ground.json; adding a substance is a line in that file and a rerun. UnrealEditor-Cmd.exe Salty.uproject -run=pythonscript -script=Scripts/Authoring/collect_terrain_assets.py ... --dry-run # say what would be copied and repointed, change nothing Idempotent: an asset already at its destination is left alone, so a rerun after adding one line copies one asset. Nothing is ever deleted from a pack, and nothing is moved: a pack stays exactly as it shipped. The hard part is that a copy is not ownership. Duplicating a material function gives a function that still samples the *pack's* textures, because the reference lives inside the graph and duplication does not rewrite it. So after everything is copied, every copy is walked and repointed at its siblings: a texture sample gets the copied texture, a function call gets the copied function, an instance gets the copied parent and an override for every texture parameter that still points into a pack. Then the whole set is checked, and anything still referencing outside Content/Terrain is reported rather than passed over in silence - that report is the only honest answer to "can the pack be deleted yet". """ import json import os import sys import unreal HERE = os.path.dirname(os.path.abspath(__file__)) PROJECT_ROOT = os.path.normpath(os.path.join(HERE, "..", "..")) GROUND_PATH = os.path.join(PROJECT_ROOT, "RawContent", "Terrain", "ground.json") asset_lib = unreal.EditorAssetLibrary def load_ground(path): with open(path, "r", encoding="utf-8") as f: data = json.load(f) destination = data.get("destination", "/Game/Terrain").rstrip("/") entries = [] for entry in data["assets"]: source = entry["source"] dest = f"{destination}/{entry.get('dest', source.rsplit('/', 1)[1])}" entries.append((source, dest, entry.get("parameters") or {})) return destination, entries def copy_assets(entries, dry_run): """Every source to its destination, skipping what is already there. Returns {source path: dest path} for the whole set, copied or not, which is the map everything downstream is repointed through.""" mapping = {} copied, skipped = 0, 0 for source, dest, _ in entries: mapping[source] = dest if asset_lib.does_asset_exist(dest): skipped += 1 continue if not asset_lib.does_asset_exist(source): raise RuntimeError(f"{source} not found; is the pack still in Content/?") if dry_run: unreal.log(f" would copy {source} -> {dest}") copied += 1 continue if not asset_lib.duplicate_asset(source, dest): raise RuntimeError(f"could not copy {source} to {dest}") unreal.log(f" copied {source} -> {dest}") copied += 1 unreal.log(f"{copied} copied, {skipped} already there") return mapping def material_expressions(asset): """Every expression node in a Material or MaterialFunction. It has to come from MaterialEditingLibrary: the `Expressions` property is protected on UMaterial and absent on UMaterialFunction, so `get_editor_property` cannot reach either of them ("Property 'Expressions' ... is protected and cannot be read"), and a first attempt that went that way found nothing and reported nothing repointed. And a function needs its own call - `get_material_expressions` takes a Material and refuses a MaterialFunction outright, which is what left the three layer functions still sampling the pack.""" getter = (unreal.MaterialEditingLibrary.get_material_function_expressions if isinstance(asset, unreal.MaterialFunction) else unreal.MaterialEditingLibrary.get_material_expressions) try: nodes = getter(asset) except Exception as error: unreal.log_warning(f"{asset.get_path_name()}: could not read its expressions ({error})") return [] return [node for node in (nodes or []) if node is not None] def repoint_property(node, property_name, mapping, dry_run): """One object-valued property of an expression, if it points at something we copied.""" try: current = node.get_editor_property(property_name) except Exception: return None if current is None: return None source = current.get_path_name().split(".")[0] dest = mapping.get(source) if dest is None or dest == source: return None if dry_run: return f"{property_name}: {source} -> {dest}" replacement = unreal.load_asset(dest) if replacement is None: raise RuntimeError(f"{dest} was copied but will not load") node.set_editor_property(property_name, replacement) return f"{property_name}: {source} -> {dest}" # Expression property names that hold a reference to another asset. A texture sample and its parameter # variants keep theirs in `texture`; a function call keeps its function in `material_function`. REFERENCE_PROPERTIES = ("texture", "material_function") def repoint_graphs(mapping, dry_run): """Every copied Material and MaterialFunction, repointed at the copies of whatever it used to reference.""" changed = 0 for dest in sorted(set(mapping.values())): asset = unreal.load_asset(dest) if not isinstance(asset, (unreal.Material, unreal.MaterialFunction)): continue nodes = material_expressions(asset) if not nodes: unreal.log_warning(f"{dest}: no expressions found, so nothing could be repointed inside it; " f"check it by hand before trusting the ownership report") continue edits = [] for node in nodes: for name in REFERENCE_PROPERTIES: edit = repoint_property(node, name, mapping, dry_run) if edit: edits.append(edit) if not edits: continue changed += len(edits) unreal.log(f" {dest}: {len(edits)} reference(s) repointed") for edit in edits: unreal.log(f" {edit}") if dry_run: continue if isinstance(asset, unreal.MaterialFunction): unreal.MaterialEditingLibrary.update_material_function(asset) else: unreal.MaterialEditingLibrary.recompile_material(asset) asset_lib.save_asset(dest) unreal.log(f"{changed} graph reference(s) repointed") def repoint_override_array(instance, mapping, dry_run): """Texture overrides stored on the instance, read straight off `texture_parameter_values` rather than through the parent's parameter list, so an orphaned override is reached too. Returns the edits made.""" try: values = instance.get_editor_property("texture_parameter_values") except Exception as error: unreal.log_warning(f"{instance.get_path_name()}: could not read its texture overrides ({error})") return [] edits, rewritten = [], [] for entry in values or []: current = entry.get_editor_property("parameter_value") replacement = mapping.get(current.get_path_name().split(".")[0]) if current else None if replacement and replacement != current.get_path_name().split(".")[0]: edits.append(f"override {entry.get_editor_property('parameter_info').get_editor_property('name')}: " f"{current.get_path_name().split('.')[0]} -> {replacement}") if not dry_run: entry.set_editor_property("parameter_value", unreal.load_asset(replacement)) rewritten.append(entry) if edits and not dry_run: instance.set_editor_property("texture_parameter_values", rewritten) return edits def repoint_instances(mapping, dry_run): """Every copied material instance: its parent, and an override for each texture parameter that still points into a pack. An instance is the one place the engine lets Python set a texture reference directly, so this is also the belt to the graph repointing's braces.""" changed = 0 for dest in sorted(set(mapping.values())): asset = unreal.load_asset(dest) if not isinstance(asset, unreal.MaterialInstanceConstant): continue edits = [] edit = repoint_property(asset, "parent", mapping, dry_run) if edit: edits.append(edit) parent = asset.get_editor_property("parent") if parent is not None: for name in unreal.MaterialEditingLibrary.get_texture_parameter_names(parent): current = unreal.MaterialEditingLibrary.get_material_instance_texture_parameter_value(asset, name) if current is None: continue source = current.get_path_name().split(".")[0] replacement = mapping.get(source) if replacement is None or replacement == source: continue edits.append(f"parameter {name}: {source} -> {replacement}") if not dry_run: unreal.MaterialEditingLibrary.set_material_instance_texture_parameter_value( asset, name, unreal.load_asset(replacement)) # And the override array itself. The loop above can only see parameters the *parent* still exposes, so # an override left over for one it no longer does is invisible to it - which is how the instance went # on referencing the pack's T_Rock_Shade_Variation after every parameter it shares with its parent had # been repointed. Walking the array catches those too. edits.extend(repoint_override_array(asset, mapping, dry_run)) if not edits: continue changed += len(edits) unreal.log(f" {dest}: {len(edits)} reference(s) repointed") for edit in edits: unreal.log(f" {edit}") if not dry_run: unreal.MaterialEditingLibrary.update_material_instance(asset) asset_lib.save_asset(dest) unreal.log(f"{changed} instance reference(s) repointed") def apply_parameters(entries, dry_run): """The `parameters` block of an entry, written onto the copy. A copy is the project's own asset, so this is where it stops being a duplicate of the pack and starts being ours: the pack's instance is left exactly as it shipped and the numbers that differ live in ground.json, next to the reason they differ. Idempotent by comparison, not by memory: a value already set is not written again, so a rerun after an unrelated edit reports nothing. Every write is read back and checked. A misspelled parameter name is not an error to the engine - the getter answers with a zero colour for a name that does not exist and the setter does nothing at all - so without the read-back a typo here would look exactly like a successful run and leave the ground the colour it already was.""" changed = 0 for _, dest, parameters in entries: if not parameters: continue instance = unreal.load_asset(dest) if not isinstance(instance, unreal.MaterialInstanceConstant): raise RuntimeError(f"{dest} has parameters in ground.json but is a {type(instance).__name__}, " f"not a material instance") for kind, names in (("vector", parameters.get("vector") or {}), ("scalar", parameters.get("scalar") or {})): for name, value in names.items(): wanted = unreal.LinearColor(*value) if kind == "vector" else float(value) if read_parameter(instance, kind, name) == quantise(wanted): continue before = read_parameter(instance, kind, name) if not dry_run: write_parameter(instance, kind, name, wanted) after = read_parameter(instance, kind, name) if after != quantise(wanted): raise RuntimeError(f"{dest}: setting {kind} parameter {name!r} to {quantise(wanted)} " f"left it at {after}; is that the name the parent exposes?") unreal.log(f" {dest}: {name} {before} -> {quantise(wanted)}") changed += 1 if changed and not dry_run: unreal.MaterialEditingLibrary.update_material_instance(instance) asset_lib.save_asset(dest, only_if_is_dirty=False) unreal.log(f"{changed} parameter(s) set from ground.json") def quantise(value): """A value rounded to what a comparison should care about, so a float that survived a round trip through the asset still equals what was asked for.""" if isinstance(value, unreal.LinearColor): return tuple(round(getattr(value, c), 4) for c in ("r", "g", "b", "a")) return round(float(value), 4) def read_parameter(instance, kind, name): getter = (unreal.MaterialEditingLibrary.get_material_instance_vector_parameter_value if kind == "vector" else unreal.MaterialEditingLibrary.get_material_instance_scalar_parameter_value) return quantise(getter(instance, name)) def write_parameter(instance, kind, name, value): setter = (unreal.MaterialEditingLibrary.set_material_instance_vector_parameter_value if kind == "vector" else unreal.MaterialEditingLibrary.set_material_instance_scalar_parameter_value) setter(instance, name, value) def refresh_all(mapping): """Recompile every copy, functions first, then materials, then instances, whether or not this run changed anything. A material caches the textures its graph and its called functions reference, and an instance caches the defaults it inherits; both caches were captured while those still pointed into the pack, and repointing a function does not rewrite them. Without this pass the asset registry went on reporting five pack textures under the master and one under the instance after every reference in both had been repointed, and a rerun could not clear it because a rerun has nothing left to change.""" order = (unreal.MaterialFunction, unreal.Material, unreal.MaterialInstanceConstant) refreshed = 0 for kind in order: for dest in sorted(set(mapping.values())): asset = unreal.load_asset(dest) if not isinstance(asset, kind): continue if kind is unreal.MaterialFunction: unreal.MaterialEditingLibrary.update_material_function(asset) elif kind is unreal.Material: unreal.MaterialEditingLibrary.recompile_material(asset) else: unreal.MaterialEditingLibrary.update_material_instance(asset) # only_if_is_dirty=False: the point of the pass is to rewrite the saved package's import table, and # a recompile does not always mark the package dirty, so the default would skip the one save that # matters. asset_lib.save_asset(dest, only_if_is_dirty=False) refreshed += 1 unreal.log(f"{refreshed} material asset(s) recompiled and saved so their cached references are rebuilt") def report_ownership(destination, mapping): """What Content/Terrain still needs from outside itself, read from the asset registry rather than from what this script believes it did. A clean report means the packs could be deleted and the ground would still render; anything listed is a reference that did not get repointed.""" registry = unreal.AssetRegistryHelpers.get_asset_registry() # Without this the newly copied assets are not in the registry yet and every one of them comes back with no # dependencies at all - which an earlier version read as "clean" and reported as standing on its own while # the master material was still calling the pack's functions. An empty answer is unknown, not clean. registry.scan_paths_synchronous([destination], force_rescan=True) options = unreal.AssetRegistryDependencyOptions(include_soft_package_references=True, include_hard_package_references=True) outside, unknown = {}, [] for dest in sorted(set(mapping.values())): dependencies = registry.get_dependencies(unreal.Name(dest), options) if dependencies is None: unknown.append(dest) continue leaks = sorted({str(name) for name in dependencies if str(name).startswith("/Game/") and not str(name).startswith(destination + "/")}) if leaks: outside[dest] = leaks if unknown: unreal.log_warning(f"{len(unknown)} asset(s) are not in the asset registry, so nothing can be said " f"about what they reference: {', '.join(unknown)}") if not outside: verdict = "stands on its own" if not unknown else "has no known references outside itself, but see above" unreal.log(f"{destination} {verdict}") return unreal.log_warning(f"{destination}: {len(outside)} asset(s) still reference something outside it") for dest, leaks in outside.items(): unreal.log_warning(f" {dest} -> {', '.join(leaks)}") unreal.log_warning("the packs cannot be deleted while those remain") def main(): # UE passes its own arguments through sys.argv under -run=pythonscript, so only the flags this script knows # are read out of it rather than handing the lot to argparse. argv = sys.argv[1:] dry_run = "--dry-run" in argv ground = GROUND_PATH if "--ground" in argv: ground = argv[argv.index("--ground") + 1] destination, entries = load_ground(ground) unreal.log(f"{len(entries)} ground assets from {os.path.relpath(ground, PROJECT_ROOT)} into {destination}" + (" (dry run)" if dry_run else "")) mapping = copy_assets(entries, dry_run) repoint_graphs(mapping, dry_run) repoint_instances(mapping, dry_run) apply_parameters(entries, dry_run) if not dry_run: refresh_all(mapping) unreal.EditorLoadingAndSavingUtils.save_dirty_packages(False, True) report_ownership(destination, mapping) main()