Tooling
This commit is contained in:
Rainer Leit
@@ -0,0 +1,485 @@
|
||||
package main
|
||||
|
||||
// `mapart biomes` turns the two categorical maps of the planet into the smooth 0..1 masks the landscape's
|
||||
// paint layers are built from: one greyscale PNG per biome, at the source's own resolution, already blurred.
|
||||
//
|
||||
// Why this is here and not in generate_region_tiles.py, which is what consumes it. The class source is the
|
||||
// painting - 7738 x 3761 of RGB - and the engine's Python has no image library that can decode it: heightmap_io
|
||||
// is greyscale-only and unfilters a byte at a time. The same constraint that made this tool Go in the first
|
||||
// place. What it hands back is 8-bit greyscale, which is heightmap_io's fast path.
|
||||
//
|
||||
// Why blurred here rather than per tile. A biome boundary has to be a gradient or the ground has a drawn line
|
||||
// on it, and a blur computed per tile is a blur that disagrees with itself across a tile seam unless every tile
|
||||
// carries a margin the width of the blur - 200 vertices at 400 m and 2 m quads, a third more area on every one
|
||||
// of ninety-eight tiles. Blurring once, globally, in the source's own pixels, makes the field smooth *before*
|
||||
// anything samples it, so a tile can read it with plain bilinear interpolation at its global coordinates and
|
||||
// two tiles agree at a shared vertex by construction. It is the same reasoning as sampling the height by global
|
||||
// position, applied a step earlier.
|
||||
//
|
||||
// Both sources are read by identity in normalised u,v, which is measured rather than assumed: the painting is
|
||||
// 7738 x 3761 and the heightmap is 8192 x 4096, and the two candidate registrations were tested against each
|
||||
// other on land/sea agreement - identity scored 98.09% against 96.14% for the alternative, and won in every
|
||||
// latitude band including the polar ones, which is where a vertical scale error shows first.
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"image"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ---------------------------------------------------------------------------------------------------------
|
||||
// The two legends
|
||||
|
||||
type legendClass struct {
|
||||
Name string `json:"name"`
|
||||
RGB []int `json:"rgb"`
|
||||
Sea bool `json:"sea"`
|
||||
}
|
||||
|
||||
type paintLegend struct {
|
||||
Classes []legendClass `json:"classes"`
|
||||
}
|
||||
|
||||
// koppenClass is one row of Tools/Orogen/js/koppen.js.
|
||||
//
|
||||
// Parsed out of the JavaScript rather than copied into a JSON beside it, because the browser twin is where that
|
||||
// table is *used* and two copies of a palette is how one of them goes stale. The parse is strict and the caller
|
||||
// checks the count: a table that has moved or been reformatted fails loudly here rather than silently matching
|
||||
// every pixel to the wrong biome.
|
||||
type koppenClass struct {
|
||||
Code string
|
||||
Name string
|
||||
SRGB [3]uint8
|
||||
}
|
||||
|
||||
// The colours in koppen.js are linear 0..1 and the browser writes them through an sRGB encode, which is why a
|
||||
// naive read of the exported PNG matches nothing: the observed "ocean" is 147,177,211 where the table says
|
||||
// 0.29,0.44,0.65. Encoding the table the same way reproduces every observed colour to within 1.4/255.
|
||||
func linearToSRGB8(c float64) uint8 {
|
||||
var s float64
|
||||
if c <= 0.0031308 {
|
||||
s = c * 12.92
|
||||
} else {
|
||||
s = 1.055*math.Pow(c, 1/2.4) - 0.055
|
||||
}
|
||||
return uint8(math.Round(math.Max(0, math.Min(1, s)) * 255))
|
||||
}
|
||||
|
||||
var koppenRow = regexp.MustCompile(`\{\s*code:\s*'([^']+)'\s*,\s*name:\s*'([^']+)'\s*,\s*color:\s*\[([^\]]+)\]`)
|
||||
|
||||
func parseKoppen(path string) ([]koppenClass, error) {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
matches := koppenRow.FindAllStringSubmatch(string(data), -1)
|
||||
out := make([]koppenClass, 0, len(matches))
|
||||
for _, m := range matches {
|
||||
parts := strings.Split(m[3], ",")
|
||||
if len(parts) != 3 {
|
||||
continue
|
||||
}
|
||||
var rgb [3]uint8
|
||||
ok := true
|
||||
for i, p := range parts {
|
||||
f, err := strconv.ParseFloat(strings.TrimSpace(p), 64)
|
||||
if err != nil {
|
||||
ok = false
|
||||
break
|
||||
}
|
||||
rgb[i] = linearToSRGB8(f)
|
||||
}
|
||||
if ok {
|
||||
out = append(out, koppenClass{Code: m[1], Name: m[2], SRGB: rgb})
|
||||
}
|
||||
}
|
||||
if len(out) < 20 {
|
||||
return nil, fmt.Errorf("%s: parsed only %d Koppen classes, expected about 31 - has the table been "+
|
||||
"reformatted? Matching against a partial palette would put every unmatched pixel in the wrong biome", path, len(out))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------------------------------
|
||||
// Classification
|
||||
|
||||
// classify turns an image into a per-pixel index into `palette`, by exact match where possible and nearest
|
||||
// colour otherwise. It returns how far the worst pixel had to travel: on the painting that is 0, because a
|
||||
// painted map is made of the legend's own colours and nothing else, and a number above a few units means the
|
||||
// image is a *render* of a classification rather than the classification itself - which is the difference
|
||||
// between data and a picture of data, and the reason the Orogen class export is not used here.
|
||||
func classify(im image.Image, palette [][3]uint8) ([]uint8, float64, float64, error) {
|
||||
if len(palette) == 0 || len(palette) > 255 {
|
||||
return nil, 0, 0, fmt.Errorf("classify: %d palette entries, need 1..255", len(palette))
|
||||
}
|
||||
read, err := rgbAccess(im)
|
||||
if err != nil {
|
||||
return nil, 0, 0, err
|
||||
}
|
||||
b := im.Bounds()
|
||||
w, h := b.Dx(), b.Dy()
|
||||
out := make([]uint8, w*h)
|
||||
exact := map[[3]uint8]uint8{}
|
||||
for i, p := range palette {
|
||||
exact[p] = uint8(i)
|
||||
}
|
||||
// Cached per distinct colour, but `far` counts *pixels*: an image whose boundaries are anti-aliased has
|
||||
// few distinct intermediate colours and a great many pixels wearing them, and it is the pixel count that
|
||||
// says whether the classification can be trusted.
|
||||
type match struct {
|
||||
index uint8
|
||||
far bool
|
||||
}
|
||||
worst := 0.0
|
||||
far := 0
|
||||
cache := map[[3]uint8]match{}
|
||||
for y := 0; y < h; y++ {
|
||||
for x := 0; x < w; x++ {
|
||||
r, g, bl := read(b.Min.X+x, b.Min.Y+y)
|
||||
key := [3]uint8{r, g, bl}
|
||||
if idx, ok := exact[key]; ok {
|
||||
out[y*w+x] = idx
|
||||
continue
|
||||
}
|
||||
m, ok := cache[key]
|
||||
if !ok {
|
||||
best, bestD := uint8(0), math.MaxFloat64
|
||||
for i, p := range palette {
|
||||
d := sq(float64(r)-float64(p[0])) + sq(float64(g)-float64(p[1])) + sq(float64(bl)-float64(p[2]))
|
||||
if d < bestD {
|
||||
bestD, best = d, uint8(i)
|
||||
}
|
||||
}
|
||||
d := math.Sqrt(bestD)
|
||||
if d > worst {
|
||||
worst = d
|
||||
}
|
||||
m = match{index: best, far: d > farThreshold}
|
||||
cache[key] = m
|
||||
}
|
||||
if m.far {
|
||||
far++
|
||||
}
|
||||
out[y*w+x] = m.index
|
||||
}
|
||||
}
|
||||
return out, worst, 100 * float64(far) / float64(w*h), nil
|
||||
}
|
||||
|
||||
// How far a colour may be from the nearest palette entry before it counts as "not really that class". Eight is
|
||||
// well past the rounding a PNG encode can introduce and well short of the distance between two palette colours.
|
||||
const farThreshold = 8.0
|
||||
|
||||
func sq(v float64) float64 { return v * v }
|
||||
|
||||
// ---------------------------------------------------------------------------------------------------------
|
||||
// Masks
|
||||
|
||||
// mask builds the 0/1 indicator of a set of palette indices, then blurs it. Two box passes rather than one:
|
||||
// a single box leaves visible straight edges where the kernel enters a blob, and two is a good enough triangle
|
||||
// filter for ground that is about to be broken up by noise anyway.
|
||||
//
|
||||
// X wraps and Y clamps, because the source is a cylinder with no route over its poles - the same rule the map
|
||||
// view's projection follows.
|
||||
func mask(index []uint8, w, h int, want map[uint8]bool, radius int) []float32 {
|
||||
field := make([]float32, w*h)
|
||||
for i, v := range index {
|
||||
if want[v] {
|
||||
field[i] = 1
|
||||
}
|
||||
}
|
||||
if radius < 1 {
|
||||
return field
|
||||
}
|
||||
field = boxBlur(field, w, h, radius)
|
||||
return boxBlur(field, w, h, radius)
|
||||
}
|
||||
|
||||
func boxBlur(src []float32, w, h, radius int) []float32 {
|
||||
tmp := make([]float32, w*h)
|
||||
out := make([]float32, w*h)
|
||||
window := float32(2*radius + 1)
|
||||
|
||||
// Horizontal, wrapping. A running sum, so the cost is per pixel and not per pixel per tap.
|
||||
for y := 0; y < h; y++ {
|
||||
row := src[y*w : (y+1)*w]
|
||||
var sum float32
|
||||
for k := -radius; k <= radius; k++ {
|
||||
sum += row[((k%w)+w)%w]
|
||||
}
|
||||
dst := tmp[y*w : (y+1)*w]
|
||||
for x := 0; x < w; x++ {
|
||||
dst[x] = sum / window
|
||||
sum -= row[(((x-radius)%w)+w)%w]
|
||||
sum += row[(((x+radius+1)%w)+w)%w]
|
||||
}
|
||||
}
|
||||
|
||||
// Vertical, clamping.
|
||||
at := func(x, y int) float32 {
|
||||
if y < 0 {
|
||||
y = 0
|
||||
} else if y >= h {
|
||||
y = h - 1
|
||||
}
|
||||
return tmp[y*w+x]
|
||||
}
|
||||
for x := 0; x < w; x++ {
|
||||
var sum float32
|
||||
for k := -radius; k <= radius; k++ {
|
||||
sum += at(x, k)
|
||||
}
|
||||
for y := 0; y < h; y++ {
|
||||
out[y*w+x] = sum / window
|
||||
sum -= at(x, y-radius)
|
||||
sum += at(x, y+radius+1)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------------------------------
|
||||
// The command
|
||||
|
||||
type biomeLayer struct {
|
||||
Name string `json:"name"`
|
||||
Rule string `json:"rule"`
|
||||
Classes []string `json:"classes"`
|
||||
Koppen []string `json:"koppen"`
|
||||
Enabled *bool `json:"enabled"`
|
||||
}
|
||||
|
||||
type biomeConfig struct {
|
||||
ClassImage string `json:"class_image"`
|
||||
ClassLegend string `json:"class_legend"`
|
||||
ClimateImage string `json:"climate_image"`
|
||||
KoppenJS string `json:"koppen_js"`
|
||||
MasksDir string `json:"masks_dir"`
|
||||
BlendM float64 `json:"blend_m"`
|
||||
}
|
||||
|
||||
type regionLayers struct {
|
||||
Biomes biomeConfig `json:"biomes"`
|
||||
Paint []biomeLayer `json:"paint"`
|
||||
}
|
||||
|
||||
type regionWithLayers struct {
|
||||
Region
|
||||
Layers regionLayers `json:"layers"`
|
||||
}
|
||||
|
||||
type maskReport struct {
|
||||
Layer string `json:"layer"`
|
||||
Rule string `json:"rule"`
|
||||
Of string `json:"of"`
|
||||
Output string `json:"output"`
|
||||
Width int `json:"width"`
|
||||
Height int `json:"height"`
|
||||
RadiusPx int `json:"radius_px"`
|
||||
CoverPct float64 `json:"cover_pct"`
|
||||
Enabled bool `json:"enabled"`
|
||||
}
|
||||
|
||||
type biomeReport struct {
|
||||
When string `json:"when"`
|
||||
BlendM float64 `json:"blend_m"`
|
||||
ClassWorst float64 `json:"class_worst_colour_distance"`
|
||||
ClassFarPct float64 `json:"class_unmatched_pct"`
|
||||
ClimateWorst float64 `json:"climate_worst_colour_distance"`
|
||||
ClimateFarPct float64 `json:"climate_unmatched_pct"`
|
||||
Masks []maskReport `json:"masks"`
|
||||
}
|
||||
|
||||
func biomes(root string) error {
|
||||
regionPath := filepath.Join(root, "RawContent", "World", "Region.json")
|
||||
var region regionWithLayers
|
||||
if err := readJSON(regionPath, ®ion); err != nil {
|
||||
return err
|
||||
}
|
||||
cfg := region.Layers.Biomes
|
||||
if cfg.ClassImage == "" {
|
||||
return fmt.Errorf("%s: layers.biomes has no class_image; nothing to classify", regionPath)
|
||||
}
|
||||
if cfg.MasksDir == "" {
|
||||
cfg.MasksDir = "RawContent/World/Biomes"
|
||||
}
|
||||
if cfg.BlendM <= 0 {
|
||||
cfg.BlendM = 400
|
||||
}
|
||||
outDir := filepath.Join(root, filepath.FromSlash(cfg.MasksDir))
|
||||
if err := os.MkdirAll(outDir, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
rep := biomeReport{When: time.Now().UTC().Format(time.RFC3339), BlendM: cfg.BlendM}
|
||||
worldW := region.widthM()
|
||||
|
||||
// --- the painted classes
|
||||
var legend paintLegend
|
||||
if err := readJSON(filepath.Join(root, filepath.FromSlash(cfg.ClassLegend)), &legend); err != nil {
|
||||
return err
|
||||
}
|
||||
classNames := make([]string, len(legend.Classes))
|
||||
classPalette := make([][3]uint8, len(legend.Classes))
|
||||
for i, c := range legend.Classes {
|
||||
if len(c.RGB) != 3 {
|
||||
return fmt.Errorf("class %q has no rgb", c.Name)
|
||||
}
|
||||
classNames[i] = c.Name
|
||||
classPalette[i] = [3]uint8{uint8(c.RGB[0]), uint8(c.RGB[1]), uint8(c.RGB[2])}
|
||||
}
|
||||
classIm, err := load(filepath.Join(root, filepath.FromSlash(cfg.ClassImage)))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
classIndex, classWorst, classFar, err := classify(classIm, classPalette)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cw, ch := classIm.Bounds().Dx(), classIm.Bounds().Dy()
|
||||
rep.ClassWorst, rep.ClassFarPct = classWorst, classFar
|
||||
fmt.Printf("class %-38s %5dx%-5d %d classes, worst distance %.1f, %.3f%% of pixels unmatched\n",
|
||||
filepath.Base(cfg.ClassImage), cw, ch, len(classNames), classWorst, classFar)
|
||||
if classWorst > 8 {
|
||||
fmt.Printf(" WARNING: a painted map is made of its legend's own colours, so this should be 0.\n")
|
||||
fmt.Printf(" A number this size means the image is a *render* of a classification rather than the\n")
|
||||
fmt.Printf(" classification itself, and every pixel is being snapped to whatever is nearest.\n")
|
||||
}
|
||||
|
||||
// --- the Koppen climate, only if a layer asks for it
|
||||
var climateIndex []uint8
|
||||
var climateCodes []string
|
||||
var clw, clh int
|
||||
needsClimate := false
|
||||
for _, l := range region.Layers.Paint {
|
||||
if l.Rule == "climate" {
|
||||
needsClimate = true
|
||||
}
|
||||
}
|
||||
if needsClimate {
|
||||
if cfg.ClimateImage == "" || cfg.KoppenJS == "" {
|
||||
return fmt.Errorf("a layer has rule \"climate\" but layers.biomes has no climate_image/koppen_js")
|
||||
}
|
||||
kop, err := parseKoppen(filepath.Join(root, filepath.FromSlash(cfg.KoppenJS)))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
climatePalette := make([][3]uint8, len(kop))
|
||||
climateCodes = make([]string, len(kop))
|
||||
for i, k := range kop {
|
||||
climatePalette[i] = k.SRGB
|
||||
climateCodes[i] = k.Code
|
||||
}
|
||||
climateIm, err := load(filepath.Join(root, filepath.FromSlash(cfg.ClimateImage)))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var worst, farPct float64
|
||||
climateIndex, worst, farPct, err = classify(climateIm, climatePalette)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
clw, clh = climateIm.Bounds().Dx(), climateIm.Bounds().Dy()
|
||||
rep.ClimateWorst, rep.ClimateFarPct = worst, farPct
|
||||
fmt.Printf("climate %-38s %5dx%-5d %d Koppen classes, worst distance %.1f, %.3f%% of pixels unmatched\n",
|
||||
filepath.Base(cfg.ClimateImage), clw, clh, len(kop), worst, farPct)
|
||||
// A render of a classification has anti-aliased boundaries, and a pixel halfway between two palette
|
||||
// colours is snapped to whichever is nearer - arbitrary, but only ever a pixel or two wide, and the
|
||||
// mask is blurred by tens of pixels afterwards. A large fraction would mean something else is wrong.
|
||||
if farPct > 5 {
|
||||
fmt.Printf(" WARNING: %.2f%% of the climate map is not close to any Koppen colour. Boundary\n", farPct)
|
||||
fmt.Printf(" anti-aliasing accounts for a fraction of a per cent; this is too much for that.\n")
|
||||
}
|
||||
}
|
||||
|
||||
indexOf := func(names []string, want string) int {
|
||||
for i, n := range names {
|
||||
if n == want {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
for _, layer := range region.Layers.Paint {
|
||||
if layer.Rule != "class" && layer.Rule != "climate" {
|
||||
continue // slope, altitude, beach and remainder are derived per tile from the height
|
||||
}
|
||||
enabled := layer.Enabled == nil || *layer.Enabled
|
||||
|
||||
var index []uint8
|
||||
var w, h int
|
||||
var names, wanted []string
|
||||
if layer.Rule == "class" {
|
||||
index, w, h, names, wanted = classIndex, cw, ch, classNames, layer.Classes
|
||||
} else {
|
||||
index, w, h, names, wanted = climateIndex, clw, clh, climateCodes, layer.Koppen
|
||||
}
|
||||
want := map[uint8]bool{}
|
||||
for _, n := range wanted {
|
||||
i := indexOf(names, n)
|
||||
if i < 0 {
|
||||
return fmt.Errorf("layer %q asks for %q, which is not in the %s legend (%s)",
|
||||
layer.Name, n, layer.Rule, strings.Join(names, ", "))
|
||||
}
|
||||
want[uint8(i)] = true
|
||||
}
|
||||
if len(want) == 0 {
|
||||
return fmt.Errorf("layer %q has rule %q but names no classes", layer.Name, layer.Rule)
|
||||
}
|
||||
|
||||
// The blur radius is in this image's own pixels, because the two sources are not the same resolution.
|
||||
metresPerPx := worldW / float64(w)
|
||||
radius := int(math.Round(cfg.BlendM / metresPerPx / 2))
|
||||
field := mask(index, w, h, want, radius)
|
||||
|
||||
var cover float64
|
||||
for _, v := range field {
|
||||
cover += float64(v)
|
||||
}
|
||||
cover = 100 * cover / float64(len(field))
|
||||
|
||||
out := image.NewGray(image.Rect(0, 0, w, h))
|
||||
for i, v := range field {
|
||||
out.Pix[i] = uint8(math.Round(float64(clamp01(v)) * 255))
|
||||
}
|
||||
name := "mask_" + strings.ToLower(layer.Name) + ".png"
|
||||
if err := writePNG(filepath.Join(outDir, name), out); err != nil {
|
||||
return err
|
||||
}
|
||||
state := ""
|
||||
if !enabled {
|
||||
state = " (not enabled yet)"
|
||||
}
|
||||
fmt.Printf(" %-10s %-8s %-28s -> %-22s r=%3d px %5.2f%% cover%s\n",
|
||||
layer.Name, layer.Rule, strings.Join(wanted, "+"), name, radius, cover, state)
|
||||
rep.Masks = append(rep.Masks, maskReport{
|
||||
Layer: layer.Name, Rule: layer.Rule, Of: strings.Join(wanted, "+"), Output: name,
|
||||
Width: w, Height: h, RadiusPx: radius, CoverPct: cover, Enabled: enabled,
|
||||
})
|
||||
}
|
||||
|
||||
sort.Slice(rep.Masks, func(i, j int) bool { return rep.Masks[i].Layer < rep.Masks[j].Layer })
|
||||
if err := writeJSON(filepath.Join(outDir, "biomes.json"), rep); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%d mask(s) into %s\n", len(rep.Masks), outDir)
|
||||
return nil
|
||||
}
|
||||
|
||||
func clamp01(v float32) float32 {
|
||||
if v < 0 {
|
||||
return 0
|
||||
}
|
||||
if v > 1 {
|
||||
return 1
|
||||
}
|
||||
return v
|
||||
}
|
||||
@@ -0,0 +1,696 @@
|
||||
// Command mapart turns planet-wide images into the layers of the world map: it downsamples what is already
|
||||
// coloured and renders shaded relief from the heightmap, writing PNGs that Scripts/Authoring/create_world_map.py
|
||||
// imports as textures.
|
||||
//
|
||||
// go run ./Tools/MapArt build # write RawContent/World/MapArt/map_<id>.png for every layer
|
||||
// go run ./Tools/MapArt check # land/sea agreement of every layer against the heightmap
|
||||
//
|
||||
// Why this is a Go tool and not part of the Python authoring set. The engine's Python has numpy and no PIL, and
|
||||
// Scripts/Authoring/heightmap_io.py's PNG decoder is greyscale-only with a per-byte unfilter loop - fine for a
|
||||
// 4081-square heightmap once, hopeless for 33 megapixels of RGB. Go's image/png does both in a few seconds.
|
||||
//
|
||||
// Why it is not part of Tools/Terrain. That is the generator: it decides what the ground IS. This decides what a
|
||||
// picture of the ground LOOKS like, downstream of every decision the generator has already made, and it will grow
|
||||
// the other way - towards compositing the overlay's marks, roads and labels onto a map sheet.
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"image"
|
||||
"image/color"
|
||||
_ "image/jpeg"
|
||||
"image/png"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ---------------------------------------------------------------------------------------------------------
|
||||
// Manifests
|
||||
|
||||
type Layer struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
File string `json:"file"`
|
||||
Render string `json:"render"`
|
||||
Default bool `json:"default"`
|
||||
Note string `json:"note"`
|
||||
}
|
||||
|
||||
type Relief struct {
|
||||
AzimuthDeg float64 `json:"light_azimuth_deg"`
|
||||
AltitudeDeg float64 `json:"light_altitude_deg"`
|
||||
Exaggeration float64 `json:"exaggeration"`
|
||||
LandTopM *float64 `json:"land_top_m"`
|
||||
ShadeStrength float64 `json:"shade_strength"`
|
||||
}
|
||||
|
||||
type Manifest struct {
|
||||
SourceDir string `json:"source_dir"`
|
||||
OutputDir string `json:"output_dir"`
|
||||
RegionPath string `json:"region"`
|
||||
Output struct {
|
||||
Width int `json:"width"`
|
||||
Height int `json:"height"`
|
||||
} `json:"output"`
|
||||
Package string `json:"package"`
|
||||
Definition string `json:"definition"`
|
||||
Level string `json:"level"`
|
||||
Layers []Layer `json:"layers"`
|
||||
Relief Relief `json:"relief"`
|
||||
}
|
||||
|
||||
// Region is the slice of RawContent/World/Region.json this tool needs. The world's size and the source's
|
||||
// elevation ramp are the generator's numbers, not ours, so they are read rather than repeated.
|
||||
type Region struct {
|
||||
Tiles struct {
|
||||
Columns int `json:"columns"`
|
||||
Rows int `json:"rows"`
|
||||
Vertices int `json:"vertices"`
|
||||
} `json:"tiles"`
|
||||
QuadCm float64 `json:"quad_cm"`
|
||||
ElevationM MinMax `json:"elevation_m"`
|
||||
SeaLevelM float64 `json:"sea_level_m"`
|
||||
Source struct {
|
||||
Path string `json:"path"`
|
||||
ElevationM MinMax `json:"elevation_m"`
|
||||
SeaScale float64 `json:"sea_scale"`
|
||||
Window struct {
|
||||
X int `json:"x"`
|
||||
Y int `json:"y"`
|
||||
Width int `json:"width"`
|
||||
Height int `json:"height"`
|
||||
} `json:"window"`
|
||||
} `json:"source"`
|
||||
}
|
||||
|
||||
type MinMax struct {
|
||||
Min float64 `json:"min"`
|
||||
Max float64 `json:"max"`
|
||||
}
|
||||
|
||||
func (r Region) quadsX() int { return (r.Tiles.Vertices - 1) * r.Tiles.Columns }
|
||||
func (r Region) quadsY() int { return (r.Tiles.Vertices - 1) * r.Tiles.Rows }
|
||||
func (r Region) widthM() float64 { return float64(r.quadsX()) * r.QuadCm / 100 }
|
||||
func (r Region) heightM() float64 { return float64(r.quadsY()) * r.QuadCm / 100 }
|
||||
|
||||
// sourceMetres turns a raw 16-bit sample into metres the way create_region_world.py does: the source's own ramp,
|
||||
// then sea_scale on everything below sea level. Anything that reads a height here must agree with the landscape
|
||||
// or the map and the ground tell different stories about the same place.
|
||||
func (r Region) sourceMetres(v uint16) float64 {
|
||||
e := r.Source.ElevationM
|
||||
m := e.Min + float64(v)/65535.0*(e.Max-e.Min)
|
||||
if m < 0 {
|
||||
m *= r.Source.SeaScale
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------------------------------
|
||||
// sRGB. Averaging encoded sRGB darkens a downsample; these two tables are the whole fix and cost nothing.
|
||||
|
||||
var srgbToLinear [256]float32
|
||||
var linearToSrgb [4096]uint8
|
||||
|
||||
func init() {
|
||||
for i := 0; i < 256; i++ {
|
||||
c := float64(i) / 255
|
||||
if c <= 0.04045 {
|
||||
srgbToLinear[i] = float32(c / 12.92)
|
||||
} else {
|
||||
srgbToLinear[i] = float32(math.Pow((c+0.055)/1.055, 2.4))
|
||||
}
|
||||
}
|
||||
for i := range linearToSrgb {
|
||||
c := float64(i) / float64(len(linearToSrgb)-1)
|
||||
var s float64
|
||||
if c <= 0.0031308 {
|
||||
s = c * 12.92
|
||||
} else {
|
||||
s = 1.055*math.Pow(c, 1/2.4) - 0.055
|
||||
}
|
||||
linearToSrgb[i] = uint8(math.Round(s * 255))
|
||||
}
|
||||
}
|
||||
|
||||
func encodeSrgb(linear float32) uint8 {
|
||||
if linear <= 0 {
|
||||
return 0
|
||||
}
|
||||
if linear >= 1 {
|
||||
return 255
|
||||
}
|
||||
return linearToSrgb[int(linear*float32(len(linearToSrgb)-1)+0.5)]
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------------------------------
|
||||
// Image access. The type switch is the point: At() through the image.Image interface costs an interface call and
|
||||
// a colour conversion per pixel, which over 33 megapixels is the difference between seconds and minutes.
|
||||
|
||||
type rgbReader func(x, y int) (r, g, b uint8)
|
||||
|
||||
func rgbAccess(im image.Image) (rgbReader, error) {
|
||||
switch src := im.(type) {
|
||||
case *image.NRGBA:
|
||||
return func(x, y int) (uint8, uint8, uint8) {
|
||||
i := src.PixOffset(x, y)
|
||||
return src.Pix[i], src.Pix[i+1], src.Pix[i+2]
|
||||
}, nil
|
||||
case *image.RGBA: // premultiplied; opaque map art, so the difference never shows, but be honest about alpha
|
||||
return func(x, y int) (uint8, uint8, uint8) {
|
||||
i := src.PixOffset(x, y)
|
||||
a := src.Pix[i+3]
|
||||
if a == 0 || a == 255 {
|
||||
return src.Pix[i], src.Pix[i+1], src.Pix[i+2]
|
||||
}
|
||||
un := func(c uint8) uint8 { return uint8(int(c) * 255 / int(a)) }
|
||||
return un(src.Pix[i]), un(src.Pix[i+1]), un(src.Pix[i+2])
|
||||
}, nil
|
||||
case *image.YCbCr: // the .jpg templates
|
||||
return func(x, y int) (uint8, uint8, uint8) {
|
||||
return color.YCbCrToRGB(src.Y[src.YOffset(x, y)], src.Cb[src.COffset(x, y)], src.Cr[src.COffset(x, y)])
|
||||
}, nil
|
||||
case *image.Gray:
|
||||
return func(x, y int) (uint8, uint8, uint8) {
|
||||
v := src.Pix[src.PixOffset(x, y)]
|
||||
return v, v, v
|
||||
}, nil
|
||||
case *image.Gray16:
|
||||
return func(x, y int) (uint8, uint8, uint8) {
|
||||
v := src.Pix[src.PixOffset(x, y)]
|
||||
return v, v, v
|
||||
}, nil
|
||||
case *image.Paletted:
|
||||
return func(x, y int) (uint8, uint8, uint8) {
|
||||
r, g, b, _ := src.Palette[src.Pix[src.PixOffset(x, y)]].RGBA()
|
||||
return uint8(r >> 8), uint8(g >> 8), uint8(b >> 8)
|
||||
}, nil
|
||||
}
|
||||
return nil, fmt.Errorf("unsupported image type %T", im)
|
||||
}
|
||||
|
||||
// grey16Access reads the 16-bit sample a heightmap carries. A heightmap that came back 8-bit is refused rather
|
||||
// than stretched: 256 levels over 11 km is 43 m a step, and a relief map built from that is terracing, not terrain.
|
||||
func grey16Access(im image.Image) (func(x, y int) uint16, error) {
|
||||
switch src := im.(type) {
|
||||
case *image.Gray16:
|
||||
return func(x, y int) uint16 {
|
||||
i := src.PixOffset(x, y)
|
||||
return uint16(src.Pix[i])<<8 | uint16(src.Pix[i+1])
|
||||
}, nil
|
||||
case *image.NRGBA64:
|
||||
return func(x, y int) uint16 {
|
||||
i := src.PixOffset(x, y)
|
||||
return uint16(src.Pix[i])<<8 | uint16(src.Pix[i+1])
|
||||
}, nil
|
||||
}
|
||||
return nil, fmt.Errorf("heightmap is %T, not 16-bit greyscale", im)
|
||||
}
|
||||
|
||||
func load(path string) (image.Image, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer f.Close()
|
||||
im, _, err := image.Decode(f)
|
||||
return im, err
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------------------------------
|
||||
// Downsampling. One pass over the source accumulating into output bins: an exact box filter when the ratio is a
|
||||
// whole number, which it is for 8192 -> 4096, and a reasonable one when it is not.
|
||||
|
||||
func downsampleRGB(im image.Image, outW, outH int) (*image.RGBA, error) {
|
||||
read, err := rgbAccess(im)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
b := im.Bounds()
|
||||
srcW, srcH := b.Dx(), b.Dy()
|
||||
sums := make([]float32, outW*outH*3)
|
||||
counts := make([]uint32, outW*outH)
|
||||
for y := 0; y < srcH; y++ {
|
||||
oy := y * outH / srcH
|
||||
for x := 0; x < srcW; x++ {
|
||||
ox := x * outW / srcW
|
||||
r, g, bl := read(b.Min.X+x, b.Min.Y+y)
|
||||
i := oy*outW + ox
|
||||
sums[i*3+0] += srgbToLinear[r]
|
||||
sums[i*3+1] += srgbToLinear[g]
|
||||
sums[i*3+2] += srgbToLinear[bl]
|
||||
counts[i]++
|
||||
}
|
||||
}
|
||||
out := image.NewRGBA(image.Rect(0, 0, outW, outH))
|
||||
for i := 0; i < outW*outH; i++ {
|
||||
n := float32(counts[i])
|
||||
if n == 0 {
|
||||
n = 1
|
||||
}
|
||||
out.Pix[i*4+0] = encodeSrgb(sums[i*3+0] / n)
|
||||
out.Pix[i*4+1] = encodeSrgb(sums[i*3+1] / n)
|
||||
out.Pix[i*4+2] = encodeSrgb(sums[i*3+2] / n)
|
||||
out.Pix[i*4+3] = 255
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// downsampleHeights averages in metres, not in sample values, because sea_scale makes the two different curves.
|
||||
func downsampleHeights(im image.Image, outW, outH int, region Region) ([]float32, error) {
|
||||
read, err := grey16Access(im)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
b := im.Bounds()
|
||||
srcW, srcH := b.Dx(), b.Dy()
|
||||
|
||||
var table [65536]float32 // one lookup beats a branch and two multiplies per source pixel
|
||||
for v := 0; v < 65536; v++ {
|
||||
table[v] = float32(region.sourceMetres(uint16(v)))
|
||||
}
|
||||
|
||||
sums := make([]float32, outW*outH)
|
||||
counts := make([]uint32, outW*outH)
|
||||
for y := 0; y < srcH; y++ {
|
||||
oy := y * outH / srcH
|
||||
for x := 0; x < srcW; x++ {
|
||||
ox := x * outW / srcW
|
||||
i := oy*outW + ox
|
||||
sums[i] += table[read(b.Min.X+x, b.Min.Y+y)]
|
||||
counts[i]++
|
||||
}
|
||||
}
|
||||
for i := range sums {
|
||||
if counts[i] > 0 {
|
||||
sums[i] /= float32(counts[i])
|
||||
}
|
||||
}
|
||||
return sums, nil
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------------------------------
|
||||
// The relief render.
|
||||
|
||||
type stop struct {
|
||||
at float64
|
||||
r, g, b float64
|
||||
}
|
||||
|
||||
// Hypsometric, the convention: green lowland through tan and brown to rock and snow. Read as fractions of the
|
||||
// land's own top, so it says nothing about absolute height - which is the honest thing, because Orogen's metres
|
||||
// are art (Region.json says so) and a ramp keyed to real metres would lie with more conviction.
|
||||
var landRamp = []stop{
|
||||
{0.00, 78, 116, 68},
|
||||
{0.12, 108, 138, 76},
|
||||
{0.30, 158, 158, 94},
|
||||
{0.50, 168, 134, 92},
|
||||
{0.70, 146, 118, 106},
|
||||
{0.88, 186, 186, 190},
|
||||
{1.00, 250, 250, 252},
|
||||
}
|
||||
|
||||
// By depth, shallow to abyss. The shelf is the light band; it is where the coast pass does its work and it should
|
||||
// be visible as a band rather than melting into the deep.
|
||||
var seaRamp = []stop{
|
||||
{0.00, 122, 174, 200},
|
||||
{0.10, 86, 144, 186},
|
||||
{0.35, 48, 100, 152},
|
||||
{1.00, 16, 38, 78},
|
||||
}
|
||||
|
||||
func sample(ramp []stop, t float64) (float64, float64, float64) {
|
||||
if t <= ramp[0].at {
|
||||
return ramp[0].r, ramp[0].g, ramp[0].b
|
||||
}
|
||||
for i := 1; i < len(ramp); i++ {
|
||||
if t <= ramp[i].at {
|
||||
a, b := ramp[i-1], ramp[i]
|
||||
f := (t - a.at) / (b.at - a.at)
|
||||
return a.r + (b.r-a.r)*f, a.g + (b.g-a.g)*f, a.b + (b.b-a.b)*f
|
||||
}
|
||||
}
|
||||
last := ramp[len(ramp)-1]
|
||||
return last.r, last.g, last.b
|
||||
}
|
||||
|
||||
func renderRelief(heights []float32, w, h int, cellM float64, cfg Relief, sea float64) (*image.RGBA, float64, float64) {
|
||||
// The ramp's ceiling. The 99.5th percentile rather than the maximum, so one summit cannot flatten the tint
|
||||
// over a whole continent - the same reasoning as the terrain tool's palette.land_top_m.
|
||||
landTop := 0.0
|
||||
if cfg.LandTopM != nil {
|
||||
landTop = *cfg.LandTopM
|
||||
} else {
|
||||
land := make([]float32, 0, len(heights)/2)
|
||||
for _, m := range heights {
|
||||
if float64(m) > sea {
|
||||
land = append(land, m)
|
||||
}
|
||||
}
|
||||
if len(land) > 0 {
|
||||
sort.Slice(land, func(i, j int) bool { return land[i] < land[j] })
|
||||
landTop = float64(land[int(float64(len(land)-1)*0.995)])
|
||||
}
|
||||
}
|
||||
if landTop <= sea {
|
||||
landTop = sea + 1
|
||||
}
|
||||
|
||||
deepest := 0.0
|
||||
for _, m := range heights {
|
||||
if float64(m) < deepest {
|
||||
deepest = float64(m)
|
||||
}
|
||||
}
|
||||
if deepest >= 0 {
|
||||
deepest = -1
|
||||
}
|
||||
|
||||
az := cfg.AzimuthDeg * math.Pi / 180
|
||||
zen := (90 - cfg.AltitudeDeg) * math.Pi / 180
|
||||
cosZen, sinZen := math.Cos(zen), math.Sin(zen)
|
||||
at := func(x, y int) float64 {
|
||||
if y < 0 {
|
||||
y = 0
|
||||
} else if y >= h {
|
||||
y = h - 1
|
||||
}
|
||||
x = ((x % w) + w) % w // the map is a cylinder: the seam column is lit by its true neighbour
|
||||
return float64(heights[y*w+x])
|
||||
}
|
||||
|
||||
out := image.NewRGBA(image.Rect(0, 0, w, h))
|
||||
for y := 0; y < h; y++ {
|
||||
for x := 0; x < w; x++ {
|
||||
m := float64(heights[y*w+x])
|
||||
|
||||
var r, g, b float64
|
||||
shaded := false
|
||||
if m > sea {
|
||||
r, g, b = sample(landRamp, (m-sea)/(landTop-sea))
|
||||
shaded = true
|
||||
} else {
|
||||
r, g, b = sample(seaRamp, m/deepest)
|
||||
}
|
||||
|
||||
if shaded {
|
||||
// Horn's 3x3 slope and aspect, then the standard hillshade. Exaggerated, because a few hundred
|
||||
// metres of relief over 17 m pixels is under two degrees and an honest shade of it is flat grey.
|
||||
a, bb, c := at(x-1, y-1), at(x, y-1), at(x+1, y-1)
|
||||
d, _, f := at(x-1, y), at(x, y), at(x+1, y)
|
||||
gg, hh, ii := at(x-1, y+1), at(x, y+1), at(x+1, y+1)
|
||||
dzdx := ((c + 2*f + ii) - (a + 2*d + gg)) / (8 * cellM) * cfg.Exaggeration
|
||||
dzdy := ((gg + 2*hh + ii) - (a + 2*bb + c)) / (8 * cellM) * cfg.Exaggeration
|
||||
slope := math.Atan(math.Hypot(dzdx, dzdy))
|
||||
aspect := math.Atan2(dzdy, -dzdx)
|
||||
shade := cosZen*math.Cos(slope) + sinZen*math.Sin(slope)*math.Cos(az-aspect)
|
||||
if shade < 0 {
|
||||
shade = 0
|
||||
}
|
||||
// 0.5 is neutral, so flat ground keeps the tint it was given and only slopes move.
|
||||
factor := 1 + cfg.ShadeStrength*(2*shade-1)
|
||||
r, g, b = r*factor, g*factor, b*factor
|
||||
}
|
||||
|
||||
i := (y*w + x) * 4
|
||||
out.Pix[i+0] = clamp8(r)
|
||||
out.Pix[i+1] = clamp8(g)
|
||||
out.Pix[i+2] = clamp8(b)
|
||||
out.Pix[i+3] = 255
|
||||
}
|
||||
}
|
||||
return out, landTop, deepest
|
||||
}
|
||||
|
||||
func clamp8(v float64) uint8 {
|
||||
if v <= 0 {
|
||||
return 0
|
||||
}
|
||||
if v >= 255 {
|
||||
return 255
|
||||
}
|
||||
return uint8(v + 0.5)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------------------------------
|
||||
|
||||
type layerReport struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Source string `json:"source"`
|
||||
Output string `json:"output"`
|
||||
Render string `json:"render"`
|
||||
Default bool `json:"default"`
|
||||
SourceW int `json:"source_width"`
|
||||
SourceH int `json:"source_height"`
|
||||
LandTopM float64 `json:"land_top_m,omitempty"`
|
||||
DeepestM float64 `json:"deepest_m,omitempty"`
|
||||
AgreePct float64 `json:"land_sea_agreement_pct,omitempty"`
|
||||
Seconds float64 `json:"seconds"`
|
||||
}
|
||||
|
||||
type report struct {
|
||||
When string `json:"when"`
|
||||
Manifest string `json:"manifest"`
|
||||
Region string `json:"region"`
|
||||
WorldWidthM float64 `json:"world_width_m"`
|
||||
WorldHeightM float64 `json:"world_height_m"`
|
||||
MetresPerPx float64 `json:"metres_per_pixel"`
|
||||
Output [2]int `json:"output"`
|
||||
Layers []layerReport `json:"layers"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
command := "build"
|
||||
if len(os.Args) > 1 {
|
||||
command = os.Args[1]
|
||||
}
|
||||
|
||||
root, err := repoRoot()
|
||||
must(err)
|
||||
|
||||
// The biome masks read Region.json alone - they are about what the ground is made of, not about the map's
|
||||
// art - so they run before layers.json is even opened.
|
||||
if command == "biomes" {
|
||||
must(biomes(root))
|
||||
return
|
||||
}
|
||||
// The substances read RawContent/Terrain/ground.json and nothing else: they are what the ground is made
|
||||
// of rather than what a picture of it looks like.
|
||||
if command == "substances" {
|
||||
must(substances(root))
|
||||
return
|
||||
}
|
||||
|
||||
manifestPath := filepath.Join(root, "RawContent", "World", "MapArt", "layers.json")
|
||||
|
||||
var man Manifest
|
||||
must(readJSON(manifestPath, &man))
|
||||
var region Region
|
||||
regionPath := filepath.Join(root, filepath.FromSlash(man.RegionPath))
|
||||
must(readJSON(regionPath, ®ion))
|
||||
|
||||
outDir := filepath.Join(root, filepath.FromSlash(man.OutputDir))
|
||||
srcDir := filepath.Join(root, filepath.FromSlash(man.SourceDir))
|
||||
must(os.MkdirAll(outDir, 0o755))
|
||||
|
||||
metresPerPx := region.widthM() / float64(man.Output.Width)
|
||||
fmt.Printf("world %.2f x %.2f km, %d x %d output, %.2f m a pixel\n",
|
||||
region.widthM()/1000, region.heightM()/1000, man.Output.Width, man.Output.Height, metresPerPx)
|
||||
if ratio := region.widthM() / region.heightM(); math.Abs(ratio-float64(man.Output.Width)/float64(man.Output.Height)) > 0.01 {
|
||||
fmt.Printf("WARNING: the world is %.3f:1 and the output is %.3f:1, so the map is stretched\n",
|
||||
ratio, float64(man.Output.Width)/float64(man.Output.Height))
|
||||
}
|
||||
|
||||
switch command {
|
||||
case "build":
|
||||
build(man, region, srcDir, outDir, regionPath, metresPerPx)
|
||||
case "check":
|
||||
check(man, region, srcDir)
|
||||
default:
|
||||
fmt.Fprintf(os.Stderr, "usage: mapart [build|check|biomes]\n\n"+
|
||||
" build render the world map's layers from the planet images\n"+
|
||||
" check land/sea agreement of every map layer against the heightmap\n"+
|
||||
" biomes the landscape's biome masks, from the painting and the Koppen climate\n")
|
||||
os.Exit(2)
|
||||
}
|
||||
}
|
||||
|
||||
func build(man Manifest, region Region, srcDir, outDir, regionPath string, metresPerPx float64) {
|
||||
rep := report{
|
||||
When: time.Now().UTC().Format(time.RFC3339),
|
||||
Manifest: man.RegionPath,
|
||||
Region: regionPath,
|
||||
WorldWidthM: region.widthM(),
|
||||
WorldHeightM: region.heightM(),
|
||||
MetresPerPx: metresPerPx,
|
||||
Output: [2]int{man.Output.Width, man.Output.Height},
|
||||
}
|
||||
|
||||
for _, layer := range man.Layers {
|
||||
started := time.Now()
|
||||
srcPath := filepath.Join(srcDir, layer.File)
|
||||
im, err := load(srcPath)
|
||||
must(err)
|
||||
b := im.Bounds()
|
||||
|
||||
entry := layerReport{
|
||||
ID: layer.ID, Name: layer.Name, Source: layer.File, Render: layer.Render,
|
||||
Default: layer.Default, SourceW: b.Dx(), SourceH: b.Dy(),
|
||||
}
|
||||
|
||||
var out *image.RGBA
|
||||
switch layer.Render {
|
||||
case "copy", "":
|
||||
out, err = downsampleRGB(im, man.Output.Width, man.Output.Height)
|
||||
must(err)
|
||||
case "relief":
|
||||
heights, err := downsampleHeights(im, man.Output.Width, man.Output.Height, region)
|
||||
must(err)
|
||||
var top, deep float64
|
||||
out, top, deep = renderRelief(heights, man.Output.Width, man.Output.Height, metresPerPx, man.Relief, region.SeaLevelM)
|
||||
entry.LandTopM, entry.DeepestM = top, deep
|
||||
default:
|
||||
must(fmt.Errorf("layer %q: unknown render %q", layer.ID, layer.Render))
|
||||
}
|
||||
|
||||
outPath := filepath.Join(outDir, "map_"+layer.ID+".png")
|
||||
must(writePNG(outPath, out))
|
||||
entry.Output = "map_" + layer.ID + ".png"
|
||||
entry.Seconds = time.Since(started).Seconds()
|
||||
|
||||
extra := ""
|
||||
if layer.Render == "relief" {
|
||||
extra = fmt.Sprintf(" land tops at %.0f m, deepest %.0f m", entry.LandTopM, entry.DeepestM)
|
||||
}
|
||||
fmt.Printf(" %-10s %5dx%-5d -> %s %.1fs%s\n", layer.ID, b.Dx(), b.Dy(), entry.Output, entry.Seconds, extra)
|
||||
rep.Layers = append(rep.Layers, entry)
|
||||
}
|
||||
|
||||
must(writeJSON(filepath.Join(outDir, "mapart.json"), rep))
|
||||
fmt.Printf("%d layers into %s\n", len(rep.Layers), outDir)
|
||||
}
|
||||
|
||||
// check is the guard against the one failure this pipeline cannot see: a layer of a different planet. Nothing in
|
||||
// a PNG says which world it is, and every layer here is a different render of the same one, so the test is not a
|
||||
// hash but agreement - does this image call the sea the sea where the heightmap does.
|
||||
func check(man Manifest, region Region, srcDir string) {
|
||||
var heightLayer *Layer
|
||||
for i := range man.Layers {
|
||||
if man.Layers[i].Render == "relief" {
|
||||
heightLayer = &man.Layers[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
if heightLayer == nil {
|
||||
must(fmt.Errorf("no layer with render \"relief\", so there is no heightmap to check against"))
|
||||
}
|
||||
hm, err := load(filepath.Join(srcDir, heightLayer.File))
|
||||
must(err)
|
||||
readH, err := grey16Access(hm)
|
||||
must(err)
|
||||
hb := hm.Bounds()
|
||||
|
||||
fmt.Printf("checking against %s\n", heightLayer.File)
|
||||
worst := 100.0
|
||||
for _, layer := range man.Layers {
|
||||
if layer.Render == "relief" {
|
||||
continue
|
||||
}
|
||||
im, err := load(filepath.Join(srcDir, layer.File))
|
||||
must(err)
|
||||
read, err := rgbAccess(im)
|
||||
must(err)
|
||||
b := im.Bounds()
|
||||
|
||||
agree, total := 0, 0
|
||||
for y := 8; y < hb.Dy(); y += 16 {
|
||||
for x := 8; x < hb.Dx(); x += 16 {
|
||||
isSea := region.sourceMetres(readH(hb.Min.X+x, hb.Min.Y+y)) <= region.SeaLevelM
|
||||
// Scaled by fraction, so a layer at a different resolution still lines up.
|
||||
lx := b.Min.X + x*b.Dx()/hb.Dx()
|
||||
ly := b.Min.Y + y*b.Dy()/hb.Dy()
|
||||
r, g, bl := read(lx, ly)
|
||||
looksSea := int(bl) > int(r)+8 && int(bl) > int(g)+4
|
||||
total++
|
||||
if looksSea == isSea {
|
||||
agree++
|
||||
}
|
||||
}
|
||||
}
|
||||
pct := 100 * float64(agree) / float64(total)
|
||||
if pct < worst {
|
||||
worst = pct
|
||||
}
|
||||
verdict := "same planet"
|
||||
if pct < 85 {
|
||||
verdict = "SUSPECT - check this is the same planet, at the same longitude origin"
|
||||
}
|
||||
fmt.Printf(" %-10s %5.2f%% land/sea agreement %s\n", layer.ID, pct, verdict)
|
||||
}
|
||||
fmt.Printf("worst %.2f%%\n", worst)
|
||||
fmt.Println("A layer of ice or heavy cloud scores lower without being wrong; the test catches a different")
|
||||
fmt.Println("planet or a shifted seam, not a few per cent. Look at the map if a number surprises you.")
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------------------------------
|
||||
|
||||
func repoRoot() (string, error) {
|
||||
dir, err := os.Getwd()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
for {
|
||||
if _, err := os.Stat(filepath.Join(dir, "Salty.uproject")); err == nil {
|
||||
return dir, nil
|
||||
}
|
||||
parent := filepath.Dir(dir)
|
||||
if parent == dir {
|
||||
return "", fmt.Errorf("no Salty.uproject above %s; run this from inside the project", dir)
|
||||
}
|
||||
dir = parent
|
||||
}
|
||||
}
|
||||
|
||||
func readJSON(path string, into any) error {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := json.Unmarshal(data, into); err != nil {
|
||||
return fmt.Errorf("%s: %w", path, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func writeJSON(path string, value any) error {
|
||||
data, err := json.MarshalIndent(value, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(path, append(data, '\n'), 0o644)
|
||||
}
|
||||
|
||||
func writePNG(path string, im image.Image) error {
|
||||
f, err := os.Create(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
enc := png.Encoder{CompressionLevel: png.DefaultCompression}
|
||||
if err := enc.Encode(f, im); err != nil {
|
||||
f.Close()
|
||||
return err
|
||||
}
|
||||
return f.Close()
|
||||
}
|
||||
|
||||
func must(err error) {
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "mapart: %v\n", strings.TrimSpace(err.Error()))
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,324 @@
|
||||
package main
|
||||
|
||||
// `mapart substances` turns the raw Fab/Quixel downloads into the handful of maps the landscape actually
|
||||
// samples, at the resolution it actually needs.
|
||||
//
|
||||
// Why it exists. A downloaded set is nine maps at 4K - AO, BaseColor, Bump, Cavity, Displacement, Gloss,
|
||||
// Normal, Roughness, Specular - about 110 MB a set, and the landscape material samples three of them. Worse,
|
||||
// a UTexture2D keeps its *source* inside the uasset, so importing 4K would put hundreds of megabytes through
|
||||
// LFS to render ground that is almost always seen at grazing distance: the scans are 2 m across, so 4K is
|
||||
// 2048 pixels per metre. This writes 2K, three maps, and nothing else.
|
||||
//
|
||||
// Normals are downsampled and then **renormalised**. Averaging four unit vectors gives a shorter one, and a
|
||||
// normal map whose vectors are not unit length lights slightly flat - not obviously wrong, just quietly
|
||||
// duller everywhere, which is the kind of thing nobody finds later.
|
||||
//
|
||||
// The physical size comes out of the set's own metadata rather than being typed here. It is what the material
|
||||
// needs to tile the texture life-size, and a number copied by hand is a number that goes stale when somebody
|
||||
// swaps a substance for one scanned at a different scale.
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"image"
|
||||
"image/jpeg"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type substanceSet struct {
|
||||
Name string `json:"name"`
|
||||
Folder string `json:"folder"`
|
||||
Layer string `json:"layer"`
|
||||
PhysicalM float64 `json:"physical_m"` // 0 = read it from the set's metadata
|
||||
Note string `json:"note"`
|
||||
}
|
||||
|
||||
type substanceConfig struct {
|
||||
SourceDir string `json:"source_dir"`
|
||||
OutputDir string `json:"output_dir"`
|
||||
Resolution int `json:"resolution"`
|
||||
Maps []string `json:"maps"`
|
||||
JpegQuality int `json:"jpeg_quality"`
|
||||
Sets []substanceSet `json:"sets"`
|
||||
}
|
||||
|
||||
type groundManifest struct {
|
||||
Substances substanceConfig `json:"substances"`
|
||||
}
|
||||
|
||||
// The per-asset metadata Quixel ships beside the maps. Only two fields matter here.
|
||||
type quixelMeta struct {
|
||||
ID string `json:"id"`
|
||||
Maps []struct {
|
||||
Name string `json:"name"`
|
||||
Type string `json:"type"`
|
||||
PhysicalSize string `json:"physicalSize"`
|
||||
Resolution string `json:"resolution"`
|
||||
MimeType string `json:"mimeType"`
|
||||
} `json:"maps"`
|
||||
}
|
||||
|
||||
type substanceReport struct {
|
||||
Name string `json:"name"`
|
||||
Layer string `json:"layer"`
|
||||
Source string `json:"source"`
|
||||
PhysicalM float64 `json:"physical_m"`
|
||||
Written []string `json:"written"`
|
||||
FromPx int `json:"from_px"`
|
||||
ToPx int `json:"to_px"`
|
||||
Seconds float64 `json:"seconds"`
|
||||
}
|
||||
|
||||
func substances(root string) error {
|
||||
groundPath := filepath.Join(root, "RawContent", "Terrain", "ground.json")
|
||||
var ground groundManifest
|
||||
if err := readJSON(groundPath, &ground); err != nil {
|
||||
return err
|
||||
}
|
||||
cfg := ground.Substances
|
||||
if len(cfg.Sets) == 0 {
|
||||
return fmt.Errorf("%s: no substances.sets, nothing to extract", groundPath)
|
||||
}
|
||||
if cfg.Resolution <= 0 {
|
||||
cfg.Resolution = 2048
|
||||
}
|
||||
if cfg.JpegQuality <= 0 {
|
||||
cfg.JpegQuality = 92
|
||||
}
|
||||
if len(cfg.Maps) == 0 {
|
||||
cfg.Maps = []string{"BaseColor", "Normal", "Roughness"}
|
||||
}
|
||||
srcRoot := filepath.Join(root, filepath.FromSlash(cfg.SourceDir))
|
||||
outDir := filepath.Join(root, filepath.FromSlash(cfg.OutputDir))
|
||||
if err := os.MkdirAll(outDir, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fmt.Printf("substances: %d set(s) -> %d px, maps %s\n", len(cfg.Sets), cfg.Resolution, strings.Join(cfg.Maps, ", "))
|
||||
|
||||
reports := make([]substanceReport, 0, len(cfg.Sets))
|
||||
for _, set := range cfg.Sets {
|
||||
started := time.Now()
|
||||
dir := filepath.Join(srcRoot, filepath.FromSlash(set.Folder))
|
||||
extracted, err := findExtracted(dir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("substance %q: %w", set.Name, err)
|
||||
}
|
||||
|
||||
physical := set.PhysicalM
|
||||
if physical <= 0 {
|
||||
physical, err = physicalSize(extracted)
|
||||
if err != nil {
|
||||
return fmt.Errorf("substance %q: %w", set.Name, err)
|
||||
}
|
||||
}
|
||||
|
||||
rep := substanceReport{Name: set.Name, Layer: set.Layer, Source: set.Folder, PhysicalM: physical, ToPx: cfg.Resolution}
|
||||
for _, kind := range cfg.Maps {
|
||||
srcFile, err := findMap(extracted, kind)
|
||||
if err != nil {
|
||||
return fmt.Errorf("substance %q: %w", set.Name, err)
|
||||
}
|
||||
im, err := load(srcFile)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s: %w", srcFile, err)
|
||||
}
|
||||
rep.FromPx = im.Bounds().Dx()
|
||||
|
||||
isNormal := strings.EqualFold(kind, "Normal")
|
||||
out, err := downsampleSurface(im, cfg.Resolution, isNormal)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s: %w", srcFile, err)
|
||||
}
|
||||
|
||||
// Normals as PNG: the source is already JPEG and a second lossy pass on a vector field shows up as
|
||||
// blocky lighting on flat ground. Colour and roughness stay JPEG, where it does not.
|
||||
name := fmt.Sprintf("T_%s_%s", set.Name, kind)
|
||||
var written string
|
||||
if isNormal {
|
||||
written = name + ".png"
|
||||
err = writePNG(filepath.Join(outDir, written), out)
|
||||
} else {
|
||||
written = name + ".jpg"
|
||||
err = writeJPEG(filepath.Join(outDir, written), out, cfg.JpegQuality)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rep.Written = append(rep.Written, written)
|
||||
}
|
||||
rep.Seconds = time.Since(started).Seconds()
|
||||
fmt.Printf(" %-26s %-11s %4d -> %4d px, %.2f m scan, %d map(s), %.1fs\n",
|
||||
set.Name, set.Layer, rep.FromPx, rep.ToPx, rep.PhysicalM, len(rep.Written), rep.Seconds)
|
||||
reports = append(reports, rep)
|
||||
}
|
||||
|
||||
if err := writeJSON(filepath.Join(outDir, "substances.json"), map[string]any{
|
||||
"when": time.Now().UTC().Format(time.RFC3339),
|
||||
"resolution": cfg.Resolution,
|
||||
"maps": cfg.Maps,
|
||||
"sets": reports,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%d substance(s) into %s\n", len(reports), outDir)
|
||||
return nil
|
||||
}
|
||||
|
||||
// findExtracted locates the ..._extracted folder inside a Fab download, whose name nobody chose and which is
|
||||
// the only place the maps and the metadata actually live.
|
||||
func findExtracted(dir string) (string, error) {
|
||||
var found string
|
||||
err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if info.IsDir() && strings.HasSuffix(info.Name(), "_extracted") {
|
||||
found = path
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if found == "" {
|
||||
return "", fmt.Errorf("no ..._extracted folder under %s - is this a Fab texture-set download?", dir)
|
||||
}
|
||||
return found, nil
|
||||
}
|
||||
|
||||
func physicalSize(extracted string) (float64, error) {
|
||||
entries, err := os.ReadDir(extracted)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
for _, e := range entries {
|
||||
if !strings.EqualFold(filepath.Ext(e.Name()), ".json") {
|
||||
continue
|
||||
}
|
||||
var meta quixelMeta
|
||||
if err := readJSON(filepath.Join(extracted, e.Name()), &meta); err != nil {
|
||||
continue
|
||||
}
|
||||
for _, m := range meta.Maps {
|
||||
// "2x2" metres. Square scans only; a non-square one would need two numbers and the material
|
||||
// would need to know about both, so it is refused rather than silently halved.
|
||||
if m.PhysicalSize == "" {
|
||||
continue
|
||||
}
|
||||
parts := strings.Split(strings.ToLower(m.PhysicalSize), "x")
|
||||
if len(parts) != 2 {
|
||||
continue
|
||||
}
|
||||
w, err1 := strconv.ParseFloat(strings.TrimSpace(parts[0]), 64)
|
||||
h, err2 := strconv.ParseFloat(strings.TrimSpace(parts[1]), 64)
|
||||
if err1 != nil || err2 != nil || w <= 0 {
|
||||
continue
|
||||
}
|
||||
if math.Abs(w-h) > 1e-6 {
|
||||
return 0, fmt.Errorf("scan is %s m, not square; the landscape material tiles with one number", m.PhysicalSize)
|
||||
}
|
||||
return w, nil
|
||||
}
|
||||
}
|
||||
return 0, fmt.Errorf("no physicalSize in the metadata under %s; set physical_m in ground.json instead", extracted)
|
||||
}
|
||||
|
||||
func findMap(extracted, kind string) (string, error) {
|
||||
entries, err := os.ReadDir(extracted)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
want := "_" + strings.ToLower(kind) + "."
|
||||
var best string
|
||||
for _, e := range entries {
|
||||
name := strings.ToLower(e.Name())
|
||||
if strings.Contains(name, want) && (strings.HasSuffix(name, ".jpg") || strings.HasSuffix(name, ".png")) {
|
||||
best = filepath.Join(extracted, e.Name())
|
||||
}
|
||||
}
|
||||
if best == "" {
|
||||
return "", fmt.Errorf("no %s map in %s", kind, extracted)
|
||||
}
|
||||
return best, nil
|
||||
}
|
||||
|
||||
// downsampleSurface box-filters to `size` square. Colour is averaged in linear light like the map art;
|
||||
// a normal map is averaged as a vector and renormalised, which is the whole reason this is not one function.
|
||||
func downsampleSurface(im image.Image, size int, isNormal bool) (*image.RGBA, error) {
|
||||
read, err := rgbAccess(im)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
b := im.Bounds()
|
||||
w, h := b.Dx(), b.Dy()
|
||||
if size > w {
|
||||
size = w
|
||||
}
|
||||
sums := make([]float64, size*size*3)
|
||||
counts := make([]uint32, size*size)
|
||||
for y := 0; y < h; y++ {
|
||||
oy := y * size / h
|
||||
for x := 0; x < w; x++ {
|
||||
ox := x * size / w
|
||||
r, g, bl := read(b.Min.X+x, b.Min.Y+y)
|
||||
i := oy*size + ox
|
||||
if isNormal {
|
||||
// To -1..1 before averaging: the midpoint of two opposite normals is flat, and the midpoint
|
||||
// of their 0..1 encodings is flat too, but only if the average happens in the signed space.
|
||||
sums[i*3+0] += float64(r)/127.5 - 1
|
||||
sums[i*3+1] += float64(g)/127.5 - 1
|
||||
sums[i*3+2] += float64(bl)/127.5 - 1
|
||||
} else {
|
||||
sums[i*3+0] += float64(srgbToLinear[r])
|
||||
sums[i*3+1] += float64(srgbToLinear[g])
|
||||
sums[i*3+2] += float64(srgbToLinear[bl])
|
||||
}
|
||||
counts[i]++
|
||||
}
|
||||
}
|
||||
out := image.NewRGBA(image.Rect(0, 0, size, size))
|
||||
for i := 0; i < size*size; i++ {
|
||||
n := float64(counts[i])
|
||||
if n == 0 {
|
||||
n = 1
|
||||
}
|
||||
if isNormal {
|
||||
x, y, z := sums[i*3+0]/n, sums[i*3+1]/n, sums[i*3+2]/n
|
||||
length := math.Sqrt(x*x + y*y + z*z)
|
||||
if length < 1e-9 {
|
||||
x, y, z, length = 0, 0, 1, 1
|
||||
}
|
||||
x, y, z = x/length, y/length, z/length
|
||||
out.Pix[i*4+0] = encodeUnit(x)
|
||||
out.Pix[i*4+1] = encodeUnit(y)
|
||||
out.Pix[i*4+2] = encodeUnit(z)
|
||||
} else {
|
||||
out.Pix[i*4+0] = encodeSrgb(float32(sums[i*3+0] / n))
|
||||
out.Pix[i*4+1] = encodeSrgb(float32(sums[i*3+1] / n))
|
||||
out.Pix[i*4+2] = encodeSrgb(float32(sums[i*3+2] / n))
|
||||
}
|
||||
out.Pix[i*4+3] = 255
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func encodeUnit(v float64) uint8 {
|
||||
return uint8(math.Round(math.Max(0, math.Min(255, (v+1)*127.5))))
|
||||
}
|
||||
|
||||
func writeJPEG(path string, im image.Image, quality int) error {
|
||||
f, err := os.Create(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := jpeg.Encode(f, im, &jpeg.Options{Quality: quality}); err != nil {
|
||||
f.Close()
|
||||
return err
|
||||
}
|
||||
return f.Close()
|
||||
}
|
||||
Reference in New Issue
Block a user