diff --git a/README.md b/README.md index dc4ac77..d4c6285 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ ASP.NET Core сервер векторных тайлов для клиенто - .NET 8 SDK - **Java 21+** в `PATH` (или путь в `TileServer:JavaPath`) — без Java тайлы не соберутся, уже готовые MBTiles сервер всё равно отдаст - Диск: 5–10× размер PBF. ЦФО до **z15** — ориентир несколько ГБ готового MBTiles плюс столько же на staging во время сборки -- RAM: ориентир `TileServer:JvmMaxHeap` ≈ 0.5× размер PBF (по умолчанию `8g`) +- RAM: контейнер ~12g. `TileServer:JvmMaxHeap` — только Java heap, держите **~⅓ лимита** (по умолчанию `4g`). Остальное — mmap/файлы Planetiler и Kestrel. Код **137** = OOM killer, не переполнение `-Xmx`. Тайлы собираются через OpenMapTiles. Схема рассчитана на z0–z14 (дефолт MapTiler/Planetiler). Жёсткий потолок **planetiler-openmaptiles 3.16 — z15**; z16+ этот JAR не умеет. Ближе z15 MapLibre overzoom’ит те же тайлы до z18. diff --git a/context/2026-09-21_05-20-00_northwest-oom-137.md b/context/2026-09-21_05-20-00_northwest-oom-137.md new file mode 100644 index 0000000..88458eb --- /dev/null +++ b/context/2026-09-21_05-20-00_northwest-oom-137.md @@ -0,0 +1,9 @@ +# 2026-09-21 — СЗФО Failed, Planetiler 137 + +Баннер «ещё не готовы (ошибка)» — не «ещё качается». В логе: + +`Planetiler exited with code 137` на `northwestern-fed-district` при z15 ~91% (38M tiles / 3.6G, heap 4.7G/8.5G). 137 = SIGKILL от cgroup OOM killer. + +Причина: контейнер `mem_limit: 12g` без swap, JVM `-Xmx8g`. Heap + mmap `feature.db` (~6G) + staging mbtiles + Kestrel > 12g. + +Фикс: heap `4g`, `--threads=8`, `--mmap_temp=false`, повтор Failed и один retry на 137, баннер показывает `lastError`. После деплоя: `docker compose up -d --build`, затем `POST /api/v1/sync/northwestern-fed-district` если RunOnStartup уже прошёл. diff --git a/docker-compose.yml b/docker-compose.yml index 31a18d0..26a696b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,7 +13,8 @@ # docker run --rm -v "$(pwd)/data:/from:ro" -v tile-server-data:/to alpine \ # sh -c "cp -a /from/. /to/" # -# Tune TileServer__JvmMaxHeap / mem_limit to the machine. +# Heap must stay well below mem_limit: Planetiler maps temp files outside -Xmx. +# Exit 137 = cgroup OOM killer. 8g heap in a 12g box dies on NW FO z15. # After raising MaxZoom (z14→z15) the next startup rebuilds MBTiles; keep the volume. services: @@ -57,7 +58,9 @@ services: TZ: UTC TileServer__DataDirectory: /data TileServer__JavaPath: /opt/java/openjdk/bin/java - TileServer__JvmMaxHeap: 8g + TileServer__JvmMaxHeap: 4g + TileServer__PlanetilerThreads: "8" + TileServer__PlanetilerMmapTemp: "false" TileServer__Sync__RunOnStartup: "true" TileServer__Sync__Cron: "0 3 * * *" TileServer__Sync__TimeZone: UTC diff --git a/src/TileServer.Api/appsettings.json b/src/TileServer.Api/appsettings.json index 6e13237..0d154eb 100644 --- a/src/TileServer.Api/appsettings.json +++ b/src/TileServer.Api/appsettings.json @@ -12,7 +12,9 @@ "TileServer": { "DataDirectory": "../../data", "JavaPath": "java", - "JvmMaxHeap": "8g", + "JvmMaxHeap": "4g", + "PlanetilerThreads": 8, + "PlanetilerMmapTemp": false, "PlanetilerJarUrl": "https://github.com/openmaptiles/planetiler-openmaptiles/releases/download/v3.16/planetiler-openmaptiles.jar", "PlanetilerHttpTimeout": "15m", "DownloadStallTimeoutSeconds": 45, @@ -65,6 +67,15 @@ "MinZoom": 0, "MaxZoom": 15, "Enabled": true + }, + { + "Id": "ural-fed-district", + "Name": "Уральский федеральный округ", + "Url": "https://download.geofabrik.de/russia/ural-fed-district-latest.osm.pbf", + "Center": [60.0, 55.0], + "MinZoom": 0, + "MaxZoom": 15, + "Enabled": true } ] } diff --git a/src/TileServer.Application/Configuration/TileServerOptions.cs b/src/TileServer.Application/Configuration/TileServerOptions.cs index b8d720e..e539964 100644 --- a/src/TileServer.Application/Configuration/TileServerOptions.cs +++ b/src/TileServer.Application/Configuration/TileServerOptions.cs @@ -8,7 +8,20 @@ public sealed class TileServerOptions public string JavaPath { get; set; } = "java"; - public string JvmMaxHeap { get; set; } = "8g"; + /// + /// JVM heap only. Planetiler memory-maps temp features outside this budget. + /// Keep around one third of the container mem_limit (4g in a 12g box). + /// Exit 137 is the OOM killer, not a Java heap dump. + /// + public string JvmMaxHeap { get; set; } = "4g"; + + /// 0 = Planetiler default (all CPUs). Cap on small containers to limit RSS. + public int PlanetilerThreads { get; set; } = 8; + + /// + /// mmap of feature.db inflates RSS on large z15 extracts. Off by default for 12g hosts. + /// + public bool PlanetilerMmapTemp { get; set; } public string PlanetilerJarUrl { get; set; } = "https://github.com/openmaptiles/planetiler-openmaptiles/releases/download/v3.16/planetiler-openmaptiles.jar"; diff --git a/src/TileServer.Application/Sync/MapSyncService.cs b/src/TileServer.Application/Sync/MapSyncService.cs index f989e08..adf231c 100644 --- a/src/TileServer.Application/Sync/MapSyncService.cs +++ b/src/TileServer.Application/Sync/MapSyncService.cs @@ -129,6 +129,7 @@ public sealed class MapSyncService( { var state = await stateStore.GetAsync(extract.Id, ct).ConfigureAwait(false); var pbfPath = dataPaths.OsmPbf(extract.Id); + var previousStatus = state.Status; try { @@ -150,6 +151,7 @@ public sealed class MapSyncService( var metadata = tileStore.GetMetadata(extract.Id); var zoomRaised = metadata is not null && extract.MaxZoom > metadata.MaxZoom; var shouldBuild = download.Changed + || previousStatus == ExtractSyncState.Failed || (tilesMissing && options.Value.Sync.RebuildIfTilesMissing) || zoomRaised; if (!shouldBuild) @@ -168,6 +170,10 @@ public sealed class MapSyncService( extract.MaxZoom, metadata!.MaxZoom); } + else if (previousStatus == ExtractSyncState.Failed) + { + logger.LogInformation("Rebuilding {ExtractId}: previous run failed.", extract.Id); + } if (!planetiler.IsJavaAvailable()) { diff --git a/src/TileServer.Infrastructure/Tiles/PlanetilerExit.cs b/src/TileServer.Infrastructure/Tiles/PlanetilerExit.cs new file mode 100644 index 0000000..fe652c7 --- /dev/null +++ b/src/TileServer.Infrastructure/Tiles/PlanetilerExit.cs @@ -0,0 +1,22 @@ +namespace TileServer.Infrastructure.Tiles; + +public static class PlanetilerExit +{ + /// 128 + SIGKILL. Docker/cgroup OOM killer, not a Java heap dump. + public const int LinuxOomKill = 137; + + public static bool IsLikelyOom(int exitCode) => exitCode is LinuxOomKill or 9; + + public static string Describe(int exitCode, string extractId) + { + if (IsLikelyOom(exitCode)) + { + return + $"Planetiler was killed (exit {exitCode}, typically the Linux OOM killer) while building '{extractId}'. " + + "JVM -Xmx is only the heap; Planetiler also maps temp features. " + + "Keep TileServer:JvmMaxHeap around 1/3 of the container mem_limit (4g in 12g) and PlanetilerMmapTemp=false."; + } + + return $"Planetiler exited with code {exitCode} for extract '{extractId}'."; + } +} diff --git a/src/TileServer.Infrastructure/Tiles/PlanetilerTileBuilder.cs b/src/TileServer.Infrastructure/Tiles/PlanetilerTileBuilder.cs index fa0554e..411b705 100644 --- a/src/TileServer.Infrastructure/Tiles/PlanetilerTileBuilder.cs +++ b/src/TileServer.Infrastructure/Tiles/PlanetilerTileBuilder.cs @@ -15,6 +15,8 @@ public sealed class PlanetilerTileBuilder( IOptions options, ILogger logger) : ITileBuilder { + private const int OomRetryAttempts = 2; + public async Task BuildAsync(Extract extract, string osmPbfPath, string outputMbtilesPath, CancellationToken ct) { if (!File.Exists(osmPbfPath)) @@ -24,16 +26,64 @@ public sealed class PlanetilerTileBuilder( var jar = await bootstrapper.EnsureJarAsync(ct).ConfigureAwait(false); Directory.CreateDirectory(Path.GetDirectoryName(outputMbtilesPath)!); - if (File.Exists(outputMbtilesPath)) - { - File.Delete(outputMbtilesPath); - } + Directory.CreateDirectory(dataPaths.Tmp); var cfg = options.Value; - var java = cfg.JavaPath; + Exception? lastFailure = null; + for (var attempt = 1; attempt <= OomRetryAttempts; attempt++) + { + if (File.Exists(outputMbtilesPath)) + { + File.Delete(outputMbtilesPath); + } + + var psi = CreateStartInfo(cfg, jar, extract, osmPbfPath, outputMbtilesPath); + logger.LogInformation( + "Starting Planetiler for {ExtractId} (attempt {Attempt}/{Max}): {File} {Args}", + extract.Id, + attempt, + OomRetryAttempts, + cfg.JavaPath, + string.Join(' ', psi.ArgumentList)); + + var exitCode = await RunAsync(psi, extract.Id, ct).ConfigureAwait(false); + ClearTmp(); + + if (exitCode == 0) + { + if (!File.Exists(outputMbtilesPath)) + { + throw new InvalidOperationException($"Planetiler finished but '{outputMbtilesPath}' was not created."); + } + + return; + } + + lastFailure = new InvalidOperationException(PlanetilerExit.Describe(exitCode, extract.Id)); + if (!PlanetilerExit.IsLikelyOom(exitCode) || attempt == OomRetryAttempts) + { + throw lastFailure; + } + + logger.LogWarning( + lastFailure, + "Planetiler OOM for {ExtractId}; retrying after clearing tmp.", + extract.Id); + } + + throw lastFailure ?? new InvalidOperationException($"Planetiler failed for extract '{extract.Id}'."); + } + + private ProcessStartInfo CreateStartInfo( + TileServerOptions cfg, + string jar, + Extract extract, + string osmPbfPath, + string outputMbtilesPath) + { var psi = new ProcessStartInfo { - FileName = java, + FileName = cfg.JavaPath, RedirectStandardOutput = true, RedirectStandardError = true, UseShellExecute = false, @@ -59,28 +109,36 @@ public sealed class PlanetilerTileBuilder( psi.ArgumentList.Add($"--natural_earth_url={cfg.NaturalEarthUrl}"); psi.ArgumentList.Add($"--lake_centerlines_url={cfg.LakeCenterlinesUrl}"); psi.ArgumentList.Add($"--water_polygons_url={cfg.WaterPolygonsUrl}"); + psi.ArgumentList.Add($"--mmap_temp={cfg.PlanetilerMmapTemp.ToString().ToLowerInvariant()}"); + if (cfg.PlanetilerThreads > 0) + { + psi.ArgumentList.Add($"--threads={cfg.PlanetilerThreads}"); + } - logger.LogInformation("Starting Planetiler for {ExtractId}: {File} {Args}", extract.Id, java, string.Join(' ', psi.ArgumentList)); + return psi; + } + private async Task RunAsync(ProcessStartInfo psi, string extractId, CancellationToken ct) + { using var process = new Process { StartInfo = psi, EnableRaisingEvents = true }; process.OutputDataReceived += (_, e) => { if (!string.IsNullOrWhiteSpace(e.Data)) { - logger.LogInformation("[planetiler {ExtractId}] {Line}", extract.Id, e.Data); + logger.LogInformation("[planetiler {ExtractId}] {Line}", extractId, e.Data); } }; process.ErrorDataReceived += (_, e) => { if (!string.IsNullOrWhiteSpace(e.Data)) { - logger.LogInformation("[planetiler {ExtractId}] {Line}", extract.Id, e.Data); + logger.LogInformation("[planetiler {ExtractId}] {Line}", extractId, e.Data); } }; if (!process.Start()) { - throw new InvalidOperationException($"Failed to start '{java}'."); + throw new InvalidOperationException($"Failed to start '{psi.FileName}'."); } process.BeginOutputReadLine(); @@ -96,14 +154,31 @@ public sealed class PlanetilerTileBuilder( throw; } - if (process.ExitCode != 0) - { - throw new InvalidOperationException($"Planetiler exited with code {process.ExitCode} for extract '{extract.Id}'."); - } + return process.ExitCode; + } - if (!File.Exists(outputMbtilesPath)) + private void ClearTmp() + { + try { - throw new InvalidOperationException($"Planetiler finished but '{outputMbtilesPath}' was not created."); + if (!Directory.Exists(dataPaths.Tmp)) + { + return; + } + + foreach (var file in Directory.EnumerateFiles(dataPaths.Tmp)) + { + File.Delete(file); + } + + foreach (var dir in Directory.EnumerateDirectories(dataPaths.Tmp)) + { + Directory.Delete(dir, recursive: true); + } + } + catch (Exception ex) + { + logger.LogWarning(ex, "Could not clear Planetiler tmp {Tmp}", dataPaths.Tmp); } } diff --git a/tests/TileServer.UnitTests/PlanetilerExitTests.cs b/tests/TileServer.UnitTests/PlanetilerExitTests.cs new file mode 100644 index 0000000..7252154 --- /dev/null +++ b/tests/TileServer.UnitTests/PlanetilerExitTests.cs @@ -0,0 +1,23 @@ +using TileServer.Infrastructure.Tiles; + +namespace TileServer.UnitTests; + +public sealed class PlanetilerExitTests +{ + [Fact] + public void IsLikelyOom_DetectsLinuxOomKiller() + { + Assert.True(PlanetilerExit.IsLikelyOom(137)); + Assert.True(PlanetilerExit.IsLikelyOom(9)); + Assert.False(PlanetilerExit.IsLikelyOom(1)); + } + + [Fact] + public void Describe_Oom_MentionsHeapVersusRss() + { + var text = PlanetilerExit.Describe(137, "northwestern-fed-district"); + Assert.Contains("northwestern-fed-district", text, StringComparison.Ordinal); + Assert.Contains("OOM", text, StringComparison.Ordinal); + Assert.Contains("JvmMaxHeap", text, StringComparison.Ordinal); + } +} diff --git a/web-demo/src/App.tsx b/web-demo/src/App.tsx index 527eeaa..5587091 100644 --- a/web-demo/src/App.tsx +++ b/web-demo/src/App.tsx @@ -23,8 +23,14 @@ const STATUS_LABELS: Record = { Failed: "ошибка" }; -function formatStatus(value?: string): string { - return STATUS_LABELS[value ?? ""] ?? value ?? "—"; +function sourceBanner(item: SourceItem): string { + if (item.status === "Failed") { + const detail = item.lastError?.trim(); + return detail + ? `Сборка «${item.name}» не удалась: ${detail}` + : `Сборка «${item.name}» не удалась. Повтор — рестарт сервиса или POST /api/v1/sync/${item.id}.`; + } + return `Тайлы источника «${item.name}» ещё не готовы (${formatStatus(item.status)}). Первый прогон может занять часы.`; } function formatDate(value?: string | null): string { @@ -179,11 +185,7 @@ export function App() { styleKeyRef.current = styleKey; const pending = selectedSources.find((item) => item.status !== "Ready"); - setBanner( - pending - ? `Тайлы источника «${pending.name}» ещё не готовы (${formatStatus(pending.status)}). Первый прогон может занять часы.` - : "" - ); + setBanner(pending ? sourceBanner(pending) : ""); }, [access, personal, selectedSourceIds, sources, source, styleName, syncGotoFromMap, updateHud]); const refresh = useCallback(async () => { diff --git a/web-demo/src/demo.css b/web-demo/src/demo.css index 434f9c8..9661492 100644 --- a/web-demo/src/demo.css +++ b/web-demo/src/demo.css @@ -444,8 +444,8 @@ dd { .banner { position: absolute; z-index: var(--hud-z); - left: var(--space); - top: 5.4rem; + right: var(--space); + top: 10em; max-width: min(34rem, calc(100vw - 24px)); background: rgba(42, 31, 18, 0.92); color: #ffe0c2;