This commit is contained in:
Rainer Leit
2026-09-25 17:02:24 +03:00
parent cc43ed8dc8
commit 9597629951
2149 changed files with 460234 additions and 1770 deletions
+696
View File
@@ -0,0 +1,696 @@
// Command mapart turns planet-wide images into the layers of the world map: it downsamples what is already
// coloured and renders shaded relief from the heightmap, writing PNGs that Scripts/Authoring/create_world_map.py
// imports as textures.
//
// go run ./Tools/MapArt build # write RawContent/World/MapArt/map_<id>.png for every layer
// go run ./Tools/MapArt check # land/sea agreement of every layer against the heightmap
//
// Why this is a Go tool and not part of the Python authoring set. The engine's Python has numpy and no PIL, and
// Scripts/Authoring/heightmap_io.py's PNG decoder is greyscale-only with a per-byte unfilter loop - fine for a
// 4081-square heightmap once, hopeless for 33 megapixels of RGB. Go's image/png does both in a few seconds.
//
// Why it is not part of Tools/Terrain. That is the generator: it decides what the ground IS. This decides what a
// picture of the ground LOOKS like, downstream of every decision the generator has already made, and it will grow
// the other way - towards compositing the overlay's marks, roads and labels onto a map sheet.
package main
import (
"encoding/json"
"fmt"
"image"
"image/color"
_ "image/jpeg"
"image/png"
"math"
"os"
"path/filepath"
"sort"
"strings"
"time"
)
// ---------------------------------------------------------------------------------------------------------
// Manifests
type Layer struct {
ID string `json:"id"`
Name string `json:"name"`
File string `json:"file"`
Render string `json:"render"`
Default bool `json:"default"`
Note string `json:"note"`
}
type Relief struct {
AzimuthDeg float64 `json:"light_azimuth_deg"`
AltitudeDeg float64 `json:"light_altitude_deg"`
Exaggeration float64 `json:"exaggeration"`
LandTopM *float64 `json:"land_top_m"`
ShadeStrength float64 `json:"shade_strength"`
}
type Manifest struct {
SourceDir string `json:"source_dir"`
OutputDir string `json:"output_dir"`
RegionPath string `json:"region"`
Output struct {
Width int `json:"width"`
Height int `json:"height"`
} `json:"output"`
Package string `json:"package"`
Definition string `json:"definition"`
Level string `json:"level"`
Layers []Layer `json:"layers"`
Relief Relief `json:"relief"`
}
// Region is the slice of RawContent/World/Region.json this tool needs. The world's size and the source's
// elevation ramp are the generator's numbers, not ours, so they are read rather than repeated.
type Region struct {
Tiles struct {
Columns int `json:"columns"`
Rows int `json:"rows"`
Vertices int `json:"vertices"`
} `json:"tiles"`
QuadCm float64 `json:"quad_cm"`
ElevationM MinMax `json:"elevation_m"`
SeaLevelM float64 `json:"sea_level_m"`
Source struct {
Path string `json:"path"`
ElevationM MinMax `json:"elevation_m"`
SeaScale float64 `json:"sea_scale"`
Window struct {
X int `json:"x"`
Y int `json:"y"`
Width int `json:"width"`
Height int `json:"height"`
} `json:"window"`
} `json:"source"`
}
type MinMax struct {
Min float64 `json:"min"`
Max float64 `json:"max"`
}
func (r Region) quadsX() int { return (r.Tiles.Vertices - 1) * r.Tiles.Columns }
func (r Region) quadsY() int { return (r.Tiles.Vertices - 1) * r.Tiles.Rows }
func (r Region) widthM() float64 { return float64(r.quadsX()) * r.QuadCm / 100 }
func (r Region) heightM() float64 { return float64(r.quadsY()) * r.QuadCm / 100 }
// sourceMetres turns a raw 16-bit sample into metres the way create_region_world.py does: the source's own ramp,
// then sea_scale on everything below sea level. Anything that reads a height here must agree with the landscape
// or the map and the ground tell different stories about the same place.
func (r Region) sourceMetres(v uint16) float64 {
e := r.Source.ElevationM
m := e.Min + float64(v)/65535.0*(e.Max-e.Min)
if m < 0 {
m *= r.Source.SeaScale
}
return m
}
// ---------------------------------------------------------------------------------------------------------
// sRGB. Averaging encoded sRGB darkens a downsample; these two tables are the whole fix and cost nothing.
var srgbToLinear [256]float32
var linearToSrgb [4096]uint8
func init() {
for i := 0; i < 256; i++ {
c := float64(i) / 255
if c <= 0.04045 {
srgbToLinear[i] = float32(c / 12.92)
} else {
srgbToLinear[i] = float32(math.Pow((c+0.055)/1.055, 2.4))
}
}
for i := range linearToSrgb {
c := float64(i) / float64(len(linearToSrgb)-1)
var s float64
if c <= 0.0031308 {
s = c * 12.92
} else {
s = 1.055*math.Pow(c, 1/2.4) - 0.055
}
linearToSrgb[i] = uint8(math.Round(s * 255))
}
}
func encodeSrgb(linear float32) uint8 {
if linear <= 0 {
return 0
}
if linear >= 1 {
return 255
}
return linearToSrgb[int(linear*float32(len(linearToSrgb)-1)+0.5)]
}
// ---------------------------------------------------------------------------------------------------------
// Image access. The type switch is the point: At() through the image.Image interface costs an interface call and
// a colour conversion per pixel, which over 33 megapixels is the difference between seconds and minutes.
type rgbReader func(x, y int) (r, g, b uint8)
func rgbAccess(im image.Image) (rgbReader, error) {
switch src := im.(type) {
case *image.NRGBA:
return func(x, y int) (uint8, uint8, uint8) {
i := src.PixOffset(x, y)
return src.Pix[i], src.Pix[i+1], src.Pix[i+2]
}, nil
case *image.RGBA: // premultiplied; opaque map art, so the difference never shows, but be honest about alpha
return func(x, y int) (uint8, uint8, uint8) {
i := src.PixOffset(x, y)
a := src.Pix[i+3]
if a == 0 || a == 255 {
return src.Pix[i], src.Pix[i+1], src.Pix[i+2]
}
un := func(c uint8) uint8 { return uint8(int(c) * 255 / int(a)) }
return un(src.Pix[i]), un(src.Pix[i+1]), un(src.Pix[i+2])
}, nil
case *image.YCbCr: // the .jpg templates
return func(x, y int) (uint8, uint8, uint8) {
return color.YCbCrToRGB(src.Y[src.YOffset(x, y)], src.Cb[src.COffset(x, y)], src.Cr[src.COffset(x, y)])
}, nil
case *image.Gray:
return func(x, y int) (uint8, uint8, uint8) {
v := src.Pix[src.PixOffset(x, y)]
return v, v, v
}, nil
case *image.Gray16:
return func(x, y int) (uint8, uint8, uint8) {
v := src.Pix[src.PixOffset(x, y)]
return v, v, v
}, nil
case *image.Paletted:
return func(x, y int) (uint8, uint8, uint8) {
r, g, b, _ := src.Palette[src.Pix[src.PixOffset(x, y)]].RGBA()
return uint8(r >> 8), uint8(g >> 8), uint8(b >> 8)
}, nil
}
return nil, fmt.Errorf("unsupported image type %T", im)
}
// grey16Access reads the 16-bit sample a heightmap carries. A heightmap that came back 8-bit is refused rather
// than stretched: 256 levels over 11 km is 43 m a step, and a relief map built from that is terracing, not terrain.
func grey16Access(im image.Image) (func(x, y int) uint16, error) {
switch src := im.(type) {
case *image.Gray16:
return func(x, y int) uint16 {
i := src.PixOffset(x, y)
return uint16(src.Pix[i])<<8 | uint16(src.Pix[i+1])
}, nil
case *image.NRGBA64:
return func(x, y int) uint16 {
i := src.PixOffset(x, y)
return uint16(src.Pix[i])<<8 | uint16(src.Pix[i+1])
}, nil
}
return nil, fmt.Errorf("heightmap is %T, not 16-bit greyscale", im)
}
func load(path string) (image.Image, error) {
f, err := os.Open(path)
if err != nil {
return nil, err
}
defer f.Close()
im, _, err := image.Decode(f)
return im, err
}
// ---------------------------------------------------------------------------------------------------------
// Downsampling. One pass over the source accumulating into output bins: an exact box filter when the ratio is a
// whole number, which it is for 8192 -> 4096, and a reasonable one when it is not.
func downsampleRGB(im image.Image, outW, outH int) (*image.RGBA, error) {
read, err := rgbAccess(im)
if err != nil {
return nil, err
}
b := im.Bounds()
srcW, srcH := b.Dx(), b.Dy()
sums := make([]float32, outW*outH*3)
counts := make([]uint32, outW*outH)
for y := 0; y < srcH; y++ {
oy := y * outH / srcH
for x := 0; x < srcW; x++ {
ox := x * outW / srcW
r, g, bl := read(b.Min.X+x, b.Min.Y+y)
i := oy*outW + ox
sums[i*3+0] += srgbToLinear[r]
sums[i*3+1] += srgbToLinear[g]
sums[i*3+2] += srgbToLinear[bl]
counts[i]++
}
}
out := image.NewRGBA(image.Rect(0, 0, outW, outH))
for i := 0; i < outW*outH; i++ {
n := float32(counts[i])
if n == 0 {
n = 1
}
out.Pix[i*4+0] = encodeSrgb(sums[i*3+0] / n)
out.Pix[i*4+1] = encodeSrgb(sums[i*3+1] / n)
out.Pix[i*4+2] = encodeSrgb(sums[i*3+2] / n)
out.Pix[i*4+3] = 255
}
return out, nil
}
// downsampleHeights averages in metres, not in sample values, because sea_scale makes the two different curves.
func downsampleHeights(im image.Image, outW, outH int, region Region) ([]float32, error) {
read, err := grey16Access(im)
if err != nil {
return nil, err
}
b := im.Bounds()
srcW, srcH := b.Dx(), b.Dy()
var table [65536]float32 // one lookup beats a branch and two multiplies per source pixel
for v := 0; v < 65536; v++ {
table[v] = float32(region.sourceMetres(uint16(v)))
}
sums := make([]float32, outW*outH)
counts := make([]uint32, outW*outH)
for y := 0; y < srcH; y++ {
oy := y * outH / srcH
for x := 0; x < srcW; x++ {
ox := x * outW / srcW
i := oy*outW + ox
sums[i] += table[read(b.Min.X+x, b.Min.Y+y)]
counts[i]++
}
}
for i := range sums {
if counts[i] > 0 {
sums[i] /= float32(counts[i])
}
}
return sums, nil
}
// ---------------------------------------------------------------------------------------------------------
// The relief render.
type stop struct {
at float64
r, g, b float64
}
// Hypsometric, the convention: green lowland through tan and brown to rock and snow. Read as fractions of the
// land's own top, so it says nothing about absolute height - which is the honest thing, because Orogen's metres
// are art (Region.json says so) and a ramp keyed to real metres would lie with more conviction.
var landRamp = []stop{
{0.00, 78, 116, 68},
{0.12, 108, 138, 76},
{0.30, 158, 158, 94},
{0.50, 168, 134, 92},
{0.70, 146, 118, 106},
{0.88, 186, 186, 190},
{1.00, 250, 250, 252},
}
// By depth, shallow to abyss. The shelf is the light band; it is where the coast pass does its work and it should
// be visible as a band rather than melting into the deep.
var seaRamp = []stop{
{0.00, 122, 174, 200},
{0.10, 86, 144, 186},
{0.35, 48, 100, 152},
{1.00, 16, 38, 78},
}
func sample(ramp []stop, t float64) (float64, float64, float64) {
if t <= ramp[0].at {
return ramp[0].r, ramp[0].g, ramp[0].b
}
for i := 1; i < len(ramp); i++ {
if t <= ramp[i].at {
a, b := ramp[i-1], ramp[i]
f := (t - a.at) / (b.at - a.at)
return a.r + (b.r-a.r)*f, a.g + (b.g-a.g)*f, a.b + (b.b-a.b)*f
}
}
last := ramp[len(ramp)-1]
return last.r, last.g, last.b
}
func renderRelief(heights []float32, w, h int, cellM float64, cfg Relief, sea float64) (*image.RGBA, float64, float64) {
// The ramp's ceiling. The 99.5th percentile rather than the maximum, so one summit cannot flatten the tint
// over a whole continent - the same reasoning as the terrain tool's palette.land_top_m.
landTop := 0.0
if cfg.LandTopM != nil {
landTop = *cfg.LandTopM
} else {
land := make([]float32, 0, len(heights)/2)
for _, m := range heights {
if float64(m) > sea {
land = append(land, m)
}
}
if len(land) > 0 {
sort.Slice(land, func(i, j int) bool { return land[i] < land[j] })
landTop = float64(land[int(float64(len(land)-1)*0.995)])
}
}
if landTop <= sea {
landTop = sea + 1
}
deepest := 0.0
for _, m := range heights {
if float64(m) < deepest {
deepest = float64(m)
}
}
if deepest >= 0 {
deepest = -1
}
az := cfg.AzimuthDeg * math.Pi / 180
zen := (90 - cfg.AltitudeDeg) * math.Pi / 180
cosZen, sinZen := math.Cos(zen), math.Sin(zen)
at := func(x, y int) float64 {
if y < 0 {
y = 0
} else if y >= h {
y = h - 1
}
x = ((x % w) + w) % w // the map is a cylinder: the seam column is lit by its true neighbour
return float64(heights[y*w+x])
}
out := image.NewRGBA(image.Rect(0, 0, w, h))
for y := 0; y < h; y++ {
for x := 0; x < w; x++ {
m := float64(heights[y*w+x])
var r, g, b float64
shaded := false
if m > sea {
r, g, b = sample(landRamp, (m-sea)/(landTop-sea))
shaded = true
} else {
r, g, b = sample(seaRamp, m/deepest)
}
if shaded {
// Horn's 3x3 slope and aspect, then the standard hillshade. Exaggerated, because a few hundred
// metres of relief over 17 m pixels is under two degrees and an honest shade of it is flat grey.
a, bb, c := at(x-1, y-1), at(x, y-1), at(x+1, y-1)
d, _, f := at(x-1, y), at(x, y), at(x+1, y)
gg, hh, ii := at(x-1, y+1), at(x, y+1), at(x+1, y+1)
dzdx := ((c + 2*f + ii) - (a + 2*d + gg)) / (8 * cellM) * cfg.Exaggeration
dzdy := ((gg + 2*hh + ii) - (a + 2*bb + c)) / (8 * cellM) * cfg.Exaggeration
slope := math.Atan(math.Hypot(dzdx, dzdy))
aspect := math.Atan2(dzdy, -dzdx)
shade := cosZen*math.Cos(slope) + sinZen*math.Sin(slope)*math.Cos(az-aspect)
if shade < 0 {
shade = 0
}
// 0.5 is neutral, so flat ground keeps the tint it was given and only slopes move.
factor := 1 + cfg.ShadeStrength*(2*shade-1)
r, g, b = r*factor, g*factor, b*factor
}
i := (y*w + x) * 4
out.Pix[i+0] = clamp8(r)
out.Pix[i+1] = clamp8(g)
out.Pix[i+2] = clamp8(b)
out.Pix[i+3] = 255
}
}
return out, landTop, deepest
}
func clamp8(v float64) uint8 {
if v <= 0 {
return 0
}
if v >= 255 {
return 255
}
return uint8(v + 0.5)
}
// ---------------------------------------------------------------------------------------------------------
type layerReport struct {
ID string `json:"id"`
Name string `json:"name"`
Source string `json:"source"`
Output string `json:"output"`
Render string `json:"render"`
Default bool `json:"default"`
SourceW int `json:"source_width"`
SourceH int `json:"source_height"`
LandTopM float64 `json:"land_top_m,omitempty"`
DeepestM float64 `json:"deepest_m,omitempty"`
AgreePct float64 `json:"land_sea_agreement_pct,omitempty"`
Seconds float64 `json:"seconds"`
}
type report struct {
When string `json:"when"`
Manifest string `json:"manifest"`
Region string `json:"region"`
WorldWidthM float64 `json:"world_width_m"`
WorldHeightM float64 `json:"world_height_m"`
MetresPerPx float64 `json:"metres_per_pixel"`
Output [2]int `json:"output"`
Layers []layerReport `json:"layers"`
}
func main() {
command := "build"
if len(os.Args) > 1 {
command = os.Args[1]
}
root, err := repoRoot()
must(err)
// The biome masks read Region.json alone - they are about what the ground is made of, not about the map's
// art - so they run before layers.json is even opened.
if command == "biomes" {
must(biomes(root))
return
}
// The substances read RawContent/Terrain/ground.json and nothing else: they are what the ground is made
// of rather than what a picture of it looks like.
if command == "substances" {
must(substances(root))
return
}
manifestPath := filepath.Join(root, "RawContent", "World", "MapArt", "layers.json")
var man Manifest
must(readJSON(manifestPath, &man))
var region Region
regionPath := filepath.Join(root, filepath.FromSlash(man.RegionPath))
must(readJSON(regionPath, &region))
outDir := filepath.Join(root, filepath.FromSlash(man.OutputDir))
srcDir := filepath.Join(root, filepath.FromSlash(man.SourceDir))
must(os.MkdirAll(outDir, 0o755))
metresPerPx := region.widthM() / float64(man.Output.Width)
fmt.Printf("world %.2f x %.2f km, %d x %d output, %.2f m a pixel\n",
region.widthM()/1000, region.heightM()/1000, man.Output.Width, man.Output.Height, metresPerPx)
if ratio := region.widthM() / region.heightM(); math.Abs(ratio-float64(man.Output.Width)/float64(man.Output.Height)) > 0.01 {
fmt.Printf("WARNING: the world is %.3f:1 and the output is %.3f:1, so the map is stretched\n",
ratio, float64(man.Output.Width)/float64(man.Output.Height))
}
switch command {
case "build":
build(man, region, srcDir, outDir, regionPath, metresPerPx)
case "check":
check(man, region, srcDir)
default:
fmt.Fprintf(os.Stderr, "usage: mapart [build|check|biomes]\n\n"+
" build render the world map's layers from the planet images\n"+
" check land/sea agreement of every map layer against the heightmap\n"+
" biomes the landscape's biome masks, from the painting and the Koppen climate\n")
os.Exit(2)
}
}
func build(man Manifest, region Region, srcDir, outDir, regionPath string, metresPerPx float64) {
rep := report{
When: time.Now().UTC().Format(time.RFC3339),
Manifest: man.RegionPath,
Region: regionPath,
WorldWidthM: region.widthM(),
WorldHeightM: region.heightM(),
MetresPerPx: metresPerPx,
Output: [2]int{man.Output.Width, man.Output.Height},
}
for _, layer := range man.Layers {
started := time.Now()
srcPath := filepath.Join(srcDir, layer.File)
im, err := load(srcPath)
must(err)
b := im.Bounds()
entry := layerReport{
ID: layer.ID, Name: layer.Name, Source: layer.File, Render: layer.Render,
Default: layer.Default, SourceW: b.Dx(), SourceH: b.Dy(),
}
var out *image.RGBA
switch layer.Render {
case "copy", "":
out, err = downsampleRGB(im, man.Output.Width, man.Output.Height)
must(err)
case "relief":
heights, err := downsampleHeights(im, man.Output.Width, man.Output.Height, region)
must(err)
var top, deep float64
out, top, deep = renderRelief(heights, man.Output.Width, man.Output.Height, metresPerPx, man.Relief, region.SeaLevelM)
entry.LandTopM, entry.DeepestM = top, deep
default:
must(fmt.Errorf("layer %q: unknown render %q", layer.ID, layer.Render))
}
outPath := filepath.Join(outDir, "map_"+layer.ID+".png")
must(writePNG(outPath, out))
entry.Output = "map_" + layer.ID + ".png"
entry.Seconds = time.Since(started).Seconds()
extra := ""
if layer.Render == "relief" {
extra = fmt.Sprintf(" land tops at %.0f m, deepest %.0f m", entry.LandTopM, entry.DeepestM)
}
fmt.Printf(" %-10s %5dx%-5d -> %s %.1fs%s\n", layer.ID, b.Dx(), b.Dy(), entry.Output, entry.Seconds, extra)
rep.Layers = append(rep.Layers, entry)
}
must(writeJSON(filepath.Join(outDir, "mapart.json"), rep))
fmt.Printf("%d layers into %s\n", len(rep.Layers), outDir)
}
// check is the guard against the one failure this pipeline cannot see: a layer of a different planet. Nothing in
// a PNG says which world it is, and every layer here is a different render of the same one, so the test is not a
// hash but agreement - does this image call the sea the sea where the heightmap does.
func check(man Manifest, region Region, srcDir string) {
var heightLayer *Layer
for i := range man.Layers {
if man.Layers[i].Render == "relief" {
heightLayer = &man.Layers[i]
break
}
}
if heightLayer == nil {
must(fmt.Errorf("no layer with render \"relief\", so there is no heightmap to check against"))
}
hm, err := load(filepath.Join(srcDir, heightLayer.File))
must(err)
readH, err := grey16Access(hm)
must(err)
hb := hm.Bounds()
fmt.Printf("checking against %s\n", heightLayer.File)
worst := 100.0
for _, layer := range man.Layers {
if layer.Render == "relief" {
continue
}
im, err := load(filepath.Join(srcDir, layer.File))
must(err)
read, err := rgbAccess(im)
must(err)
b := im.Bounds()
agree, total := 0, 0
for y := 8; y < hb.Dy(); y += 16 {
for x := 8; x < hb.Dx(); x += 16 {
isSea := region.sourceMetres(readH(hb.Min.X+x, hb.Min.Y+y)) <= region.SeaLevelM
// Scaled by fraction, so a layer at a different resolution still lines up.
lx := b.Min.X + x*b.Dx()/hb.Dx()
ly := b.Min.Y + y*b.Dy()/hb.Dy()
r, g, bl := read(lx, ly)
looksSea := int(bl) > int(r)+8 && int(bl) > int(g)+4
total++
if looksSea == isSea {
agree++
}
}
}
pct := 100 * float64(agree) / float64(total)
if pct < worst {
worst = pct
}
verdict := "same planet"
if pct < 85 {
verdict = "SUSPECT - check this is the same planet, at the same longitude origin"
}
fmt.Printf(" %-10s %5.2f%% land/sea agreement %s\n", layer.ID, pct, verdict)
}
fmt.Printf("worst %.2f%%\n", worst)
fmt.Println("A layer of ice or heavy cloud scores lower without being wrong; the test catches a different")
fmt.Println("planet or a shifted seam, not a few per cent. Look at the map if a number surprises you.")
}
// ---------------------------------------------------------------------------------------------------------
func repoRoot() (string, error) {
dir, err := os.Getwd()
if err != nil {
return "", err
}
for {
if _, err := os.Stat(filepath.Join(dir, "Salty.uproject")); err == nil {
return dir, nil
}
parent := filepath.Dir(dir)
if parent == dir {
return "", fmt.Errorf("no Salty.uproject above %s; run this from inside the project", dir)
}
dir = parent
}
}
func readJSON(path string, into any) error {
data, err := os.ReadFile(path)
if err != nil {
return err
}
if err := json.Unmarshal(data, into); err != nil {
return fmt.Errorf("%s: %w", path, err)
}
return nil
}
func writeJSON(path string, value any) error {
data, err := json.MarshalIndent(value, "", " ")
if err != nil {
return err
}
return os.WriteFile(path, append(data, '\n'), 0o644)
}
func writePNG(path string, im image.Image) error {
f, err := os.Create(path)
if err != nil {
return err
}
enc := png.Encoder{CompressionLevel: png.DefaultCompression}
if err := enc.Encode(f, im); err != nil {
f.Close()
return err
}
return f.Close()
}
func must(err error) {
if err != nil {
fmt.Fprintf(os.Stderr, "mapart: %v\n", strings.TrimSpace(err.Error()))
os.Exit(1)
}
}