Tooling
This commit is contained in:
Rainer Leit
@@ -0,0 +1,324 @@
|
||||
package main
|
||||
|
||||
// `mapart substances` turns the raw Fab/Quixel downloads into the handful of maps the landscape actually
|
||||
// samples, at the resolution it actually needs.
|
||||
//
|
||||
// Why it exists. A downloaded set is nine maps at 4K - AO, BaseColor, Bump, Cavity, Displacement, Gloss,
|
||||
// Normal, Roughness, Specular - about 110 MB a set, and the landscape material samples three of them. Worse,
|
||||
// a UTexture2D keeps its *source* inside the uasset, so importing 4K would put hundreds of megabytes through
|
||||
// LFS to render ground that is almost always seen at grazing distance: the scans are 2 m across, so 4K is
|
||||
// 2048 pixels per metre. This writes 2K, three maps, and nothing else.
|
||||
//
|
||||
// Normals are downsampled and then **renormalised**. Averaging four unit vectors gives a shorter one, and a
|
||||
// normal map whose vectors are not unit length lights slightly flat - not obviously wrong, just quietly
|
||||
// duller everywhere, which is the kind of thing nobody finds later.
|
||||
//
|
||||
// The physical size comes out of the set's own metadata rather than being typed here. It is what the material
|
||||
// needs to tile the texture life-size, and a number copied by hand is a number that goes stale when somebody
|
||||
// swaps a substance for one scanned at a different scale.
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"image"
|
||||
"image/jpeg"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type substanceSet struct {
|
||||
Name string `json:"name"`
|
||||
Folder string `json:"folder"`
|
||||
Layer string `json:"layer"`
|
||||
PhysicalM float64 `json:"physical_m"` // 0 = read it from the set's metadata
|
||||
Note string `json:"note"`
|
||||
}
|
||||
|
||||
type substanceConfig struct {
|
||||
SourceDir string `json:"source_dir"`
|
||||
OutputDir string `json:"output_dir"`
|
||||
Resolution int `json:"resolution"`
|
||||
Maps []string `json:"maps"`
|
||||
JpegQuality int `json:"jpeg_quality"`
|
||||
Sets []substanceSet `json:"sets"`
|
||||
}
|
||||
|
||||
type groundManifest struct {
|
||||
Substances substanceConfig `json:"substances"`
|
||||
}
|
||||
|
||||
// The per-asset metadata Quixel ships beside the maps. Only two fields matter here.
|
||||
type quixelMeta struct {
|
||||
ID string `json:"id"`
|
||||
Maps []struct {
|
||||
Name string `json:"name"`
|
||||
Type string `json:"type"`
|
||||
PhysicalSize string `json:"physicalSize"`
|
||||
Resolution string `json:"resolution"`
|
||||
MimeType string `json:"mimeType"`
|
||||
} `json:"maps"`
|
||||
}
|
||||
|
||||
type substanceReport struct {
|
||||
Name string `json:"name"`
|
||||
Layer string `json:"layer"`
|
||||
Source string `json:"source"`
|
||||
PhysicalM float64 `json:"physical_m"`
|
||||
Written []string `json:"written"`
|
||||
FromPx int `json:"from_px"`
|
||||
ToPx int `json:"to_px"`
|
||||
Seconds float64 `json:"seconds"`
|
||||
}
|
||||
|
||||
func substances(root string) error {
|
||||
groundPath := filepath.Join(root, "RawContent", "Terrain", "ground.json")
|
||||
var ground groundManifest
|
||||
if err := readJSON(groundPath, &ground); err != nil {
|
||||
return err
|
||||
}
|
||||
cfg := ground.Substances
|
||||
if len(cfg.Sets) == 0 {
|
||||
return fmt.Errorf("%s: no substances.sets, nothing to extract", groundPath)
|
||||
}
|
||||
if cfg.Resolution <= 0 {
|
||||
cfg.Resolution = 2048
|
||||
}
|
||||
if cfg.JpegQuality <= 0 {
|
||||
cfg.JpegQuality = 92
|
||||
}
|
||||
if len(cfg.Maps) == 0 {
|
||||
cfg.Maps = []string{"BaseColor", "Normal", "Roughness"}
|
||||
}
|
||||
srcRoot := filepath.Join(root, filepath.FromSlash(cfg.SourceDir))
|
||||
outDir := filepath.Join(root, filepath.FromSlash(cfg.OutputDir))
|
||||
if err := os.MkdirAll(outDir, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fmt.Printf("substances: %d set(s) -> %d px, maps %s\n", len(cfg.Sets), cfg.Resolution, strings.Join(cfg.Maps, ", "))
|
||||
|
||||
reports := make([]substanceReport, 0, len(cfg.Sets))
|
||||
for _, set := range cfg.Sets {
|
||||
started := time.Now()
|
||||
dir := filepath.Join(srcRoot, filepath.FromSlash(set.Folder))
|
||||
extracted, err := findExtracted(dir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("substance %q: %w", set.Name, err)
|
||||
}
|
||||
|
||||
physical := set.PhysicalM
|
||||
if physical <= 0 {
|
||||
physical, err = physicalSize(extracted)
|
||||
if err != nil {
|
||||
return fmt.Errorf("substance %q: %w", set.Name, err)
|
||||
}
|
||||
}
|
||||
|
||||
rep := substanceReport{Name: set.Name, Layer: set.Layer, Source: set.Folder, PhysicalM: physical, ToPx: cfg.Resolution}
|
||||
for _, kind := range cfg.Maps {
|
||||
srcFile, err := findMap(extracted, kind)
|
||||
if err != nil {
|
||||
return fmt.Errorf("substance %q: %w", set.Name, err)
|
||||
}
|
||||
im, err := load(srcFile)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s: %w", srcFile, err)
|
||||
}
|
||||
rep.FromPx = im.Bounds().Dx()
|
||||
|
||||
isNormal := strings.EqualFold(kind, "Normal")
|
||||
out, err := downsampleSurface(im, cfg.Resolution, isNormal)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s: %w", srcFile, err)
|
||||
}
|
||||
|
||||
// Normals as PNG: the source is already JPEG and a second lossy pass on a vector field shows up as
|
||||
// blocky lighting on flat ground. Colour and roughness stay JPEG, where it does not.
|
||||
name := fmt.Sprintf("T_%s_%s", set.Name, kind)
|
||||
var written string
|
||||
if isNormal {
|
||||
written = name + ".png"
|
||||
err = writePNG(filepath.Join(outDir, written), out)
|
||||
} else {
|
||||
written = name + ".jpg"
|
||||
err = writeJPEG(filepath.Join(outDir, written), out, cfg.JpegQuality)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rep.Written = append(rep.Written, written)
|
||||
}
|
||||
rep.Seconds = time.Since(started).Seconds()
|
||||
fmt.Printf(" %-26s %-11s %4d -> %4d px, %.2f m scan, %d map(s), %.1fs\n",
|
||||
set.Name, set.Layer, rep.FromPx, rep.ToPx, rep.PhysicalM, len(rep.Written), rep.Seconds)
|
||||
reports = append(reports, rep)
|
||||
}
|
||||
|
||||
if err := writeJSON(filepath.Join(outDir, "substances.json"), map[string]any{
|
||||
"when": time.Now().UTC().Format(time.RFC3339),
|
||||
"resolution": cfg.Resolution,
|
||||
"maps": cfg.Maps,
|
||||
"sets": reports,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%d substance(s) into %s\n", len(reports), outDir)
|
||||
return nil
|
||||
}
|
||||
|
||||
// findExtracted locates the ..._extracted folder inside a Fab download, whose name nobody chose and which is
|
||||
// the only place the maps and the metadata actually live.
|
||||
func findExtracted(dir string) (string, error) {
|
||||
var found string
|
||||
err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if info.IsDir() && strings.HasSuffix(info.Name(), "_extracted") {
|
||||
found = path
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if found == "" {
|
||||
return "", fmt.Errorf("no ..._extracted folder under %s - is this a Fab texture-set download?", dir)
|
||||
}
|
||||
return found, nil
|
||||
}
|
||||
|
||||
func physicalSize(extracted string) (float64, error) {
|
||||
entries, err := os.ReadDir(extracted)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
for _, e := range entries {
|
||||
if !strings.EqualFold(filepath.Ext(e.Name()), ".json") {
|
||||
continue
|
||||
}
|
||||
var meta quixelMeta
|
||||
if err := readJSON(filepath.Join(extracted, e.Name()), &meta); err != nil {
|
||||
continue
|
||||
}
|
||||
for _, m := range meta.Maps {
|
||||
// "2x2" metres. Square scans only; a non-square one would need two numbers and the material
|
||||
// would need to know about both, so it is refused rather than silently halved.
|
||||
if m.PhysicalSize == "" {
|
||||
continue
|
||||
}
|
||||
parts := strings.Split(strings.ToLower(m.PhysicalSize), "x")
|
||||
if len(parts) != 2 {
|
||||
continue
|
||||
}
|
||||
w, err1 := strconv.ParseFloat(strings.TrimSpace(parts[0]), 64)
|
||||
h, err2 := strconv.ParseFloat(strings.TrimSpace(parts[1]), 64)
|
||||
if err1 != nil || err2 != nil || w <= 0 {
|
||||
continue
|
||||
}
|
||||
if math.Abs(w-h) > 1e-6 {
|
||||
return 0, fmt.Errorf("scan is %s m, not square; the landscape material tiles with one number", m.PhysicalSize)
|
||||
}
|
||||
return w, nil
|
||||
}
|
||||
}
|
||||
return 0, fmt.Errorf("no physicalSize in the metadata under %s; set physical_m in ground.json instead", extracted)
|
||||
}
|
||||
|
||||
func findMap(extracted, kind string) (string, error) {
|
||||
entries, err := os.ReadDir(extracted)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
want := "_" + strings.ToLower(kind) + "."
|
||||
var best string
|
||||
for _, e := range entries {
|
||||
name := strings.ToLower(e.Name())
|
||||
if strings.Contains(name, want) && (strings.HasSuffix(name, ".jpg") || strings.HasSuffix(name, ".png")) {
|
||||
best = filepath.Join(extracted, e.Name())
|
||||
}
|
||||
}
|
||||
if best == "" {
|
||||
return "", fmt.Errorf("no %s map in %s", kind, extracted)
|
||||
}
|
||||
return best, nil
|
||||
}
|
||||
|
||||
// downsampleSurface box-filters to `size` square. Colour is averaged in linear light like the map art;
|
||||
// a normal map is averaged as a vector and renormalised, which is the whole reason this is not one function.
|
||||
func downsampleSurface(im image.Image, size int, isNormal bool) (*image.RGBA, error) {
|
||||
read, err := rgbAccess(im)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
b := im.Bounds()
|
||||
w, h := b.Dx(), b.Dy()
|
||||
if size > w {
|
||||
size = w
|
||||
}
|
||||
sums := make([]float64, size*size*3)
|
||||
counts := make([]uint32, size*size)
|
||||
for y := 0; y < h; y++ {
|
||||
oy := y * size / h
|
||||
for x := 0; x < w; x++ {
|
||||
ox := x * size / w
|
||||
r, g, bl := read(b.Min.X+x, b.Min.Y+y)
|
||||
i := oy*size + ox
|
||||
if isNormal {
|
||||
// To -1..1 before averaging: the midpoint of two opposite normals is flat, and the midpoint
|
||||
// of their 0..1 encodings is flat too, but only if the average happens in the signed space.
|
||||
sums[i*3+0] += float64(r)/127.5 - 1
|
||||
sums[i*3+1] += float64(g)/127.5 - 1
|
||||
sums[i*3+2] += float64(bl)/127.5 - 1
|
||||
} else {
|
||||
sums[i*3+0] += float64(srgbToLinear[r])
|
||||
sums[i*3+1] += float64(srgbToLinear[g])
|
||||
sums[i*3+2] += float64(srgbToLinear[bl])
|
||||
}
|
||||
counts[i]++
|
||||
}
|
||||
}
|
||||
out := image.NewRGBA(image.Rect(0, 0, size, size))
|
||||
for i := 0; i < size*size; i++ {
|
||||
n := float64(counts[i])
|
||||
if n == 0 {
|
||||
n = 1
|
||||
}
|
||||
if isNormal {
|
||||
x, y, z := sums[i*3+0]/n, sums[i*3+1]/n, sums[i*3+2]/n
|
||||
length := math.Sqrt(x*x + y*y + z*z)
|
||||
if length < 1e-9 {
|
||||
x, y, z, length = 0, 0, 1, 1
|
||||
}
|
||||
x, y, z = x/length, y/length, z/length
|
||||
out.Pix[i*4+0] = encodeUnit(x)
|
||||
out.Pix[i*4+1] = encodeUnit(y)
|
||||
out.Pix[i*4+2] = encodeUnit(z)
|
||||
} else {
|
||||
out.Pix[i*4+0] = encodeSrgb(float32(sums[i*3+0] / n))
|
||||
out.Pix[i*4+1] = encodeSrgb(float32(sums[i*3+1] / n))
|
||||
out.Pix[i*4+2] = encodeSrgb(float32(sums[i*3+2] / n))
|
||||
}
|
||||
out.Pix[i*4+3] = 255
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func encodeUnit(v float64) uint8 {
|
||||
return uint8(math.Round(math.Max(0, math.Min(255, (v+1)*127.5))))
|
||||
}
|
||||
|
||||
func writeJPEG(path string, im image.Image, quality int) error {
|
||||
f, err := os.Create(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := jpeg.Encode(f, im, &jpeg.Options{Quality: quality}); err != nil {
|
||||
f.Close()
|
||||
return err
|
||||
}
|
||||
return f.Close()
|
||||
}
|
||||
Reference in New Issue
Block a user