{ "schemaVersion": 1, "methodId": "flighttrove-adsb-lol-historical-derivation-v1", "purpose": "Exact algorithm and schema supplement for the ODbL 1.0 section 4.6(b) method-of-alterations offer, including downstream historical feature, scoring, selection, snapshot and outcome transformations.", "documentPermission": "FlightTrove permits copying, implementing and adapting this published method description and its algorithm excerpts, including commercial use, without an additional fee or restriction. This permission covers the published excerpts, not undisclosed FlightTrove application software. The derivative database and FlightTrove-added derivative contents are offered under ODbL 1.0.", "licenseUrl": "https://opendatacommons.org/licenses/odbl/1-0/", "dependencies": { "java": "21", "h3": "com.uber:h3:4.1.1", "postgresql": "16", "hashing": "Java standard-library SHA-256, SHA-1/UUIDv5 and MD5/UUIDv3 as specified in the included implementations" }, "normalization": "Algorithm text uses LF line endings. Geometry is a separate byte-exact additional-content artifact. No runtime data or secrets are read to build this supplement.", "packageSha256": "e170ff79aa946d4864bd265fe18186f99ca4e20739a196e2d59797d434a44876", "files": [ { "path": "backend/common/src/main/java/hu/gombos/flightspotter/common/geo/H3CoreFactory.java", "encoding": "UTF-8", "sha256": "fbad537497f2f8cf5b00a9680fdbb4afecab15c6a3f202f5ef4eaaa6dd840393", "content": "package hu.gombos.flightspotter.common.geo;\n\nimport com.uber.h3core.H3Core;\n\nimport java.io.IOException;\nimport java.util.Locale;\n\n/**\n * Central H3 native loader.\n *\n *

Development and tests use the upstream extract-to-temp behavior. Hardened\n * production images package libh3-java.so as an immutable image artifact and\n * select SYSTEM mode, avoiding executable temporary files entirely.\n */\npublic final class H3CoreFactory {\n static final String NATIVE_MODE_ENV = \"FLIGHTTROVE_H3_NATIVE_MODE\";\n\n private H3CoreFactory() {\n }\n\n public static H3Core create() {\n String mode = System.getenv().getOrDefault(NATIVE_MODE_ENV, \"EXTRACT\")\n .trim()\n .toUpperCase(Locale.ROOT);\n try {\n return switch (mode) {\n case \"EXTRACT\" -> H3Core.newInstance();\n case \"SYSTEM\" -> H3Core.newSystemInstance();\n default -> throw new IllegalStateException(\n \"Unsupported \" + NATIVE_MODE_ENV + \": \" + mode);\n };\n } catch (IOException | UnsatisfiedLinkError failure) {\n throw new IllegalStateException(\"Failed to initialize H3 native runtime in \" + mode + \" mode\", failure);\n }\n }\n}" }, { "path": "backend/common/src/main/java/hu/gombos/flightspotter/common/geo/H3Service.java", "encoding": "UTF-8", "sha256": "084cb0e731e71177350420c10c52b22687395758d1f6b8f53c1b5be88283fbb7", "content": "package hu.gombos.flightspotter.common.geo;\n\nimport java.util.List;\nimport java.util.Set;\n\npublic interface H3Service {\n record GeoPoint(double latitude, double longitude) {\n public GeoPoint {\n if (!Double.isFinite(latitude) || latitude < -90.0d || latitude > 90.0d) {\n throw new IllegalArgumentException(\"latitude must be finite and in [-90,90]\");\n }\n if (!Double.isFinite(longitude) || longitude < -180.0d || longitude > 180.0d) {\n throw new IllegalArgumentException(\"longitude must be finite and in [-180,180]\");\n }\n }\n }\n\n String latLonToCell(double lat, double lon, int resolution);\n\n Set gridDisk(String cell, int distance);\n\n /**\n * Canonical H3 hierarchy projection. Use this for parent/child identity; reconstructing a\n * coarser cell from a child centre is a geometric lookup and is not hierarchy-equivalent.\n */\n default String cellToParent(String cell, int parentResolution) {\n throw new UnsupportedOperationException(\"H3 cell parent projection is not supported by this implementation\");\n }\n\n /**\n * Optional polygon capability used by historical country-coverage derivation. Existing lightweight\n * H3 fakes only implementing point/disk operations remain source-compatible; production UberH3Service\n * overrides this method.\n */\n default Set polygonToCells(List outer, List> holes, int resolution) {\n throw new UnsupportedOperationException(\"polygon H3 coverage is not supported by this implementation\");\n }\n\n /**\n * Cell centre in geographic coordinates. This is suitable for rendering and geometric lookup,\n * but must never be used to reconstruct hierarchical parent identity.\n */\n default GeoPoint cellToCenter(String cell) {\n throw new UnsupportedOperationException(\"H3 cell centre is not supported by this implementation\");\n }\n\n /** Complete boundary vertices in geographic coordinates; never use the cell centre for containment. */\n default List cellToBoundary(String cell) {\n throw new UnsupportedOperationException(\"H3 cell boundary is not supported by this implementation\");\n }\n}\n" }, { "path": "backend/common/src/main/java/hu/gombos/flightspotter/common/geo/UberH3Service.java", "encoding": "UTF-8", "sha256": "8c08de3b06484f16b8d54a9a69a9138b6bd878c6b0b31e0f6845b08b54994b07", "content": "package hu.gombos.flightspotter.common.geo;\n\nimport com.uber.h3core.H3Core;\nimport com.uber.h3core.util.LatLng;\n\nimport java.util.List;\nimport java.util.Set;\n\npublic class UberH3Service implements H3Service {\n\n private final H3Core h3;\n\n public UberH3Service() {\n this.h3 = H3CoreFactory.create();\n }\n\n @Override\n public String latLonToCell(double lat, double lon, int resolution) {\n return h3.latLngToCellAddress(lat, lon, resolution);\n }\n\n @Override\n public Set gridDisk(String cell, int distance) {\n return Set.copyOf(h3.gridDisk(cell, distance));\n }\n\n @Override\n public String cellToParent(String cell, int parentResolution) {\n if (cell == null || cell.isBlank()) {\n throw new IllegalArgumentException(\"H3 cell is required\");\n }\n return h3.cellToParentAddress(cell, parentResolution);\n }\n\n @Override\n public Set polygonToCells(List outer, List> holes, int resolution) {\n if (outer == null || outer.size() < 3) {\n throw new IllegalArgumentException(\"polygon outer ring must contain at least three points\");\n }\n if (resolution < 0 || resolution > 15) {\n throw new IllegalArgumentException(\"H3 resolution must be in [0,15]\");\n }\n List outerLatLng = outer.stream()\n .map(point -> new LatLng(point.latitude(), point.longitude()))\n .toList();\n List> holeLatLng = holes == null\n ? List.of()\n : holes.stream()\n .map(ring -> ring.stream()\n .map(point -> new LatLng(point.latitude(), point.longitude()))\n .toList())\n .toList();\n return Set.copyOf(h3.polygonToCellAddresses(outerLatLng, holeLatLng, resolution));\n }\n\n @Override\n public GeoPoint cellToCenter(String cell) {\n if (cell == null || cell.isBlank()) {\n throw new IllegalArgumentException(\"H3 cell is required\");\n }\n LatLng center = h3.cellToLatLng(cell);\n return new GeoPoint(center.lat, center.lng);\n }\n\n @Override\n public List cellToBoundary(String cell) {\n if (cell == null || cell.isBlank()) {\n throw new IllegalArgumentException(\"H3 cell is required\");\n }\n return h3.cellToBoundary(cell).stream()\n .map(vertex -> new GeoPoint(vertex.lat, vertex.lng))\n .toList();\n }\n}\n" }, { "path": "backend/forecast-model-core/src/main/java/hu/gombos/flightspotter/forecastmodel/CoverageDomain.java", "encoding": "UTF-8", "sha256": "46388563cde3f3d501bbcc311b3480a4482993101d4c729c90878d0eece7e13b", "content": "package hu.gombos.flightspotter.forecastmodel;\nimport java.nio.charset.StandardCharsets;\nimport java.security.MessageDigest;\nimport java.security.NoSuchAlgorithmException;\nimport java.util.*;\nimport java.util.regex.Pattern;\npublic final class CoverageDomain {\n public static final String IDENTITY_VERSION=\"flighttrove-coverage-domain-v1\";\n private static final Pattern VERSION=Pattern.compile(\"[A-Za-z0-9._-]{1,120}\");\n private static final Pattern SHA256=Pattern.compile(\"[0-9a-f]{64}\");\n private static final Pattern H3=Pattern.compile(\"8[0-9a-f][0-9a-f]{13}\");\n private static final int MAX_CELLS=500_000;\n private final CoverageRegionId regionId; private final String coverageVersion; private final int h3Resolution;\n private final String geometryProvenanceVersion; private final String geometrySha256; private final int borderBufferRings;\n private final List orderedCells; private final Set cellSet; private final String evidenceCoverageHash; private final String canonicalSha256;\n public CoverageDomain(CoverageRegionId regionId,String coverageVersion,int h3Resolution,String geometryProvenanceVersion,String geometrySha256,int borderBufferRings,Collection cells){\n this.regionId=Objects.requireNonNull(regionId); this.coverageVersion=stableToken(coverageVersion,\"coverageVersion\");\n if(h3Resolution<0||h3Resolution>15) throw new IllegalArgumentException(\"h3Resolution must be in [0,15]\"); this.h3Resolution=h3Resolution;\n this.geometryProvenanceVersion=stableToken(geometryProvenanceVersion,\"geometryProvenanceVersion\");\n if(geometrySha256==null||!SHA256.matcher(geometrySha256).matches()) throw new IllegalArgumentException(\"geometrySha256 must be lowercase SHA-256\"); this.geometrySha256=geometrySha256;\n if(borderBufferRings<0||borderBufferRings>4) throw new IllegalArgumentException(\"borderBufferRings must be in [0,4]\"); this.borderBufferRings=borderBufferRings;\n Objects.requireNonNull(cells); if(cells.isEmpty()||cells.size()>MAX_CELLS) throw new IllegalArgumentException(\"coverage cell set must be non-empty and bounded\");\n TreeSet n=new TreeSet<>(); String prefix=\"8\"+Integer.toHexString(h3Resolution);\n for(String raw:cells){ if(raw==null) throw new IllegalArgumentException(\"coverage contains null H3 cell\"); String cell=raw.trim().toLowerCase(Locale.ROOT); if(!H3.matcher(cell).matches()||!cell.startsWith(prefix)) throw new IllegalArgumentException(\"coverage contains H3 cell outside declared resolution: \"+raw); if(!n.add(cell)) throw new IllegalArgumentException(\"coverage contains duplicate H3 cell: \"+cell); }\n orderedCells=List.copyOf(n); cellSet=Set.copyOf(n); evidenceCoverageHash=p7CompatibleHash(coverageVersion,orderedCells); canonicalSha256=DeterministicHashing.sha256(canonicalPayload());\n }\n public CoverageRegionId regionId(){return regionId;} public String coverageVersion(){return coverageVersion;} public int h3Resolution(){return h3Resolution;}\n public String geometryProvenanceVersion(){return geometryProvenanceVersion;} public String geometrySha256(){return geometrySha256;} public int borderBufferRings(){return borderBufferRings;}\n public List orderedH3Cells(){return orderedCells;} public int cellCount(){return orderedCells.size();} public String evidenceCoverageHash(){return evidenceCoverageHash;} public String canonicalSha256(){return canonicalSha256;}\n public boolean containsCell(String cell){return cell!=null&&cellSet.contains(cell.trim().toLowerCase(Locale.ROOT));}\n public boolean contains(CoverageDomain required){Objects.requireNonNull(required);return h3Resolution==required.h3Resolution&&cellSet.containsAll(required.cellSet);}\n public void requireContains(CoverageDomain required){if(!contains(required)) throw new IllegalArgumentException(\"coverage domain does not contain required domain\");}\n public String canonicalPayload(){StringBuilder out=new StringBuilder(IDENTITY_VERSION).append('\\n').append(\"region=\").append(regionId.value()).append('\\n').append(\"coverage_version=\").append(coverageVersion).append('\\n').append(\"h3_resolution=\").append(h3Resolution).append('\\n').append(\"geometry_provenance=\").append(geometryProvenanceVersion).append('\\n').append(\"geometry_sha256=\").append(geometrySha256).append('\\n').append(\"border_buffer_rings=\").append(borderBufferRings).append('\\n').append(\"evidence_coverage_hash=\").append(evidenceCoverageHash).append('\\n'); for(String cell:orderedCells)out.append(\"cell=\").append(cell).append('\\n'); return out.toString();}\n private static String p7CompatibleHash(String version,List cells){try{MessageDigest d=MessageDigest.getInstance(\"SHA-256\"); update(d,version); for(String c:cells)update(d,c); return HexFormat.of().formatHex(d.digest());}catch(NoSuchAlgorithmException e){throw new IllegalStateException(e);}}\n private static void update(MessageDigest d,String v){byte[] b=v.getBytes(StandardCharsets.UTF_8); d.update((byte)(b.length>>>24));d.update((byte)(b.length>>>16));d.update((byte)(b.length>>>8));d.update((byte)b.length);d.update(b);}\n private static String stableToken(String v,String f){if(v==null||!VERSION.matcher(v.trim()).matches())throw new IllegalArgumentException(f+\" must be a bounded stable token\");return v.trim();}\n}" }, { "path": "backend/forecast-model-core/src/main/java/hu/gombos/flightspotter/forecastmodel/CoverageDomainRegistry.java", "encoding": "UTF-8", "sha256": "ab2e505bf095d5ac3ffb4c43fe5ef9645034751cece2a83c88249bf85abe6e7a", "content": "package hu.gombos.flightspotter.forecastmodel;\nimport hu.gombos.flightspotter.common.geo.H3Service; import java.io.InputStream; import java.util.*;\npublic final class CoverageDomainRegistry {\n public static final String EUROPE_COVERAGE_VERSION=\"europe-v1-ne110m-admin0-r5-buffer1-v1\";\n public static final String EUROPE_GEOMETRY_PROVENANCE=\"natural-earth-ne110m-admin0-europe-union-v1\";\n public static final String EUROPE_GEOMETRY_SHA256=\"20b8eaffe91ce2eaab297708467d7cc6ec0957561597ecfc4113f61bf1d63b67\";\n public static final int EUROPE_H3_RESOLUTION=5, EUROPE_BORDER_BUFFER_RINGS=1; private static final String EUROPE_RESOURCE=\"/coverage/europe-v1.polygons\";\n private final Map domains;\n public CoverageDomainRegistry(Collection ds){Objects.requireNonNull(ds);LinkedHashMap m=new LinkedHashMap<>();for(CoverageDomain d:ds){Objects.requireNonNull(d);if(m.put(d.regionId(),d)!=null)throw new IllegalArgumentException(\"duplicate coverage region: \"+d.regionId());}if(m.isEmpty())throw new IllegalArgumentException(\"coverage registry cannot be empty\");domains=Map.copyOf(m);}\n public static CoverageDomainRegistry sourceControlled(H3Service h3){return new CoverageDomainRegistry(List.of(loadEuropeV1(h3)));}\n public static CoverageDomain loadEuropeV1(H3Service h3){InputStream r=CoverageDomainRegistry.class.getResourceAsStream(EUROPE_RESOURCE);if(r==null)throw new IllegalStateException(\"EUROPE_V1 geometry resource is missing\");return CoverageGeometryLoader.load(CoverageRegionId.EUROPE_V1,EUROPE_COVERAGE_VERSION,EUROPE_H3_RESOLUTION,EUROPE_GEOMETRY_PROVENANCE,EUROPE_GEOMETRY_SHA256,EUROPE_BORDER_BUFFER_RINGS,r,h3);}\n public CoverageDomain require(CoverageRegionId id){CoverageDomain d=domains.get(Objects.requireNonNull(id));if(d==null)throw new IllegalArgumentException(\"unknown coverage region: \"+id);return d;}\n public CoverageDomain require(String id){return require(CoverageRegionId.of(id));} public Map domains(){return domains;}\n}" }, { "path": "backend/forecast-model-core/src/main/java/hu/gombos/flightspotter/forecastmodel/CoverageGeometryLoader.java", "encoding": "UTF-8", "sha256": "a5523b88f42260781ee25a8891acf0bbdec3d68539f9fff6aca37908183c07a5", "content": "package hu.gombos.flightspotter.forecastmodel;\nimport hu.gombos.flightspotter.common.geo.H3Service;\nimport java.io.*; import java.nio.charset.StandardCharsets; import java.security.*; import java.util.*;\nfinal class CoverageGeometryLoader {\n private CoverageGeometryLoader(){}\n static CoverageDomain load(CoverageRegionId id,String version,int res,String prov,String expected,int buffer,InputStream source,H3Service h3){\n Objects.requireNonNull(source); Objects.requireNonNull(h3); byte[] bytes; try(source){bytes=source.readAllBytes();}catch(IOException e){throw new IllegalStateException(\"coverage geometry resource is unreadable\",e);}\n String actual=sha(bytes); if(!actual.equals(expected))throw new IllegalStateException(\"coverage geometry resource SHA-256 mismatch\");\n List> polygons=new ArrayList<>(); List anchors=new ArrayList<>();\n for(String raw:new String(bytes,StandardCharsets.UTF_8).lines().toList()){String line=raw.trim();if(line.isEmpty()||line.startsWith(\"#\"))continue;String[] p=line.split(\"\\\\|\",-1);if(p.length!=3||p[1].isBlank())throw new IllegalStateException(\"invalid coverage geometry row\");if(\"P\".equals(p[0])){List ring=new ArrayList<>();for(String v:p[2].split(\";\"))ring.add(point(v));if(ring.size()<4)throw new IllegalStateException(\"coverage polygon has fewer than four vertices\");polygons.add(List.copyOf(ring));}else if(\"A\".equals(p[0]))anchors.add(point(p[2]));else throw new IllegalStateException(\"unknown coverage geometry row type\");}\n if(polygons.isEmpty())throw new IllegalStateException(\"coverage geometry contains no polygons\"); LinkedHashSet base=new LinkedHashSet<>();\n for(var poly:polygons){Set cells=h3.polygonToCells(poly,List.of(),res);if(cells.isEmpty())throw new IllegalStateException(\"coverage polygon produced no H3 cells\");base.addAll(new TreeSet<>(cells));}\n for(var a:anchors)base.add(h3.latLonToCell(a.latitude(),a.longitude(),res)); LinkedHashSet buffered=new LinkedHashSet<>(base);if(buffer>0)for(String c:new TreeSet<>(base))buffered.addAll(h3.gridDisk(c,buffer));\n return new CoverageDomain(id,version,res,prov,actual,buffer,buffered);\n }\n private static H3Service.GeoPoint point(String token){String[] v=token.split(\",\",-1);if(v.length!=2)throw new IllegalStateException(\"invalid coverage coordinate\");try{double lat=Double.parseDouble(v[0]),lon=Double.parseDouble(v[1]);if(!Double.isFinite(lat)||lat<-90||lat>90||!Double.isFinite(lon)||lon<-180||lon>180)throw new IllegalStateException(\"coverage coordinate is outside WGS84 bounds\");return new H3Service.GeoPoint(lat,lon);}catch(NumberFormatException e){throw new IllegalStateException(\"invalid coverage coordinate\",e);}}\n private static String sha(byte[] b){try{return HexFormat.of().formatHex(MessageDigest.getInstance(\"SHA-256\").digest(b));}catch(NoSuchAlgorithmException e){throw new IllegalStateException(e);}}\n}" }, { "path": "backend/forecast-model-core/src/main/java/hu/gombos/flightspotter/forecastmodel/CoverageRegionId.java", "encoding": "UTF-8", "sha256": "e6870203029636beb41e9e7eda08a833a609d1fe23aa214c62b8e7742f0e16af", "content": "package hu.gombos.flightspotter.forecastmodel;\nimport java.util.Locale;\nimport java.util.Objects;\nimport java.util.regex.Pattern;\npublic record CoverageRegionId(String value) implements Comparable {\n private static final Pattern TOKEN = Pattern.compile(\"[A-Z][A-Z0-9_]{1,63}\");\n public static final CoverageRegionId EUROPE_V1 = new CoverageRegionId(\"EUROPE_V1\");\n public CoverageRegionId { Objects.requireNonNull(value,\"value\"); value=value.trim().toUpperCase(Locale.ROOT); if(!TOKEN.matcher(value).matches()) throw new IllegalArgumentException(\"coverage region id must be a bounded uppercase token\"); }\n public static CoverageRegionId of(String value){ return new CoverageRegionId(value); }\n @Override public int compareTo(CoverageRegionId other){ return value.compareTo(other.value); }\n @Override public String toString(){ return value; }\n}" }, { "path": "backend/forecast-model-core/src/main/java/hu/gombos/flightspotter/forecastmodel/DeterministicHashing.java", "encoding": "UTF-8", "sha256": "c5d15fae416659fba94aa52d86b3906c90e4816c5b6e1a425b8a691251d280cf", "content": "package hu.gombos.flightspotter.forecastmodel;\n\nimport java.nio.ByteBuffer;\nimport java.nio.charset.StandardCharsets;\nimport java.security.MessageDigest;\nimport java.security.NoSuchAlgorithmException;\nimport java.util.HexFormat;\nimport java.util.UUID;\n\nfinal class DeterministicHashing {\n private DeterministicHashing() {}\n\n static String sha256(String canonical) {\n return HexFormat.of().formatHex(digest(\"SHA-256\", canonical.getBytes(StandardCharsets.UTF_8)));\n }\n\n static UUID uuidV5(UUID namespace, String canonicalName) {\n MessageDigest sha1 = messageDigest(\"SHA-1\");\n sha1.update(ByteBuffer.allocate(16).putLong(namespace.getMostSignificantBits())\n .putLong(namespace.getLeastSignificantBits()).array());\n byte[] bytes = sha1.digest(canonicalName.getBytes(StandardCharsets.UTF_8));\n bytes[6] = (byte) ((bytes[6] & 0x0f) | 0x50);\n bytes[8] = (byte) ((bytes[8] & 0x3f) | 0x80);\n ByteBuffer value = ByteBuffer.wrap(bytes);\n return new UUID(value.getLong(), value.getLong());\n }\n\n private static byte[] digest(String algorithm, byte[] value) {\n return messageDigest(algorithm).digest(value);\n }\n\n private static MessageDigest messageDigest(String algorithm) {\n try {\n return MessageDigest.getInstance(algorithm);\n } catch (NoSuchAlgorithmException exception) {\n throw new IllegalStateException(algorithm + \" unavailable\", exception);\n }\n }\n}\n" }, { "path": "backend/forecast-model-core/src/main/java/hu/gombos/flightspotter/forecastmodel/P861DailySelectionPolicy.java", "encoding": "UTF-8", "sha256": "e6c530bdb5319dda5eaf4510661cc6fb16471cf41021bc2ef70937e030f52ca3", "content": "package hu.gombos.flightspotter.forecastmodel;\n\nimport java.math.BigDecimal;\nimport java.nio.charset.StandardCharsets;\nimport java.security.MessageDigest;\nimport java.time.Duration;\nimport java.time.Instant;\nimport java.time.ZoneOffset;\nimport java.util.ArrayList;\nimport java.util.Comparator;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Objects;\nimport java.util.TreeMap;\nimport java.util.HexFormat;\n\n/**\n * Frozen P8.6.1 daily selector experiment over the unchanged THINNED hourly top-two pool.\n *\n *

The selector accepts no label, outcome, target-day observation, guarded score or temporal\n * reliability input. It changes only the 48-to-5 daily selection structure. THINNED remains the\n * score and all hourly identities/ranks are inputs fixed upstream.

\n */\npublic final class P861DailySelectionPolicy {\n public static final String VERSION = \"p861-daily-selection-structure-v1\";\n public static final int PER_HOUR_CAP = 2;\n public static final int PER_DAY_CAP = 5;\n\n public enum Family {\n RAW_THINNED_BASELINE,\n ROUTE_ROUND_ROBIN,\n HOUR_RANK_ROUTE_ROUND_ROBIN\n }\n\n /** Label-free immutable selector row. */\n public record Candidate(String identity, String key, String routeKey, Instant target, Instant asOf,\n BigDecimal thinnedScore, long hourSupport, long corridorSupport, int hourlyRank) {\n public Candidate {\n Objects.requireNonNull(identity);\n Objects.requireNonNull(key);\n Objects.requireNonNull(routeKey);\n Objects.requireNonNull(target);\n Objects.requireNonNull(asOf);\n Objects.requireNonNull(thinnedScore);\n if (identity.isBlank() || key.isBlank() || routeKey.isBlank()\n || !asOf.equals(target.minus(Duration.ofHours(24)))\n || thinnedScore.signum() < 0 || thinnedScore.compareTo(BigDecimal.ONE) > 0\n || hourSupport < 1 || corridorSupport < 1\n || hourlyRank < 1 || hourlyRank > PER_HOUR_CAP) {\n throw new IllegalArgumentException(\"Invalid exact-T24 P861 selector candidate\");\n }\n }\n }\n\n /** Exact comparator used by the frozen THINNED daily baseline. */\n public static final Comparator THINNED_ORDER = Comparator\n .comparing(Candidate::thinnedScore).reversed()\n .thenComparing(Comparator.comparingLong(Candidate::hourSupport).reversed())\n .thenComparing(Comparator.comparingLong(Candidate::corridorSupport).reversed())\n .thenComparing(Candidate::key)\n .thenComparing(Candidate::target);\n\n /** Tests whether within-hour ordinal rank is more portable across hours than raw score level. */\n public static final Comparator HOUR_RANK_ORDER = Comparator\n .comparingInt(Candidate::hourlyRank)\n .thenComparing(THINNED_ORDER);\n\n private final Family family;\n\n public P861DailySelectionPolicy(Family family) {\n this.family = Objects.requireNonNull(family);\n }\n\n public Family family() {\n return family;\n }\n\n public String canonical() {\n return VERSION + \"\\nfamily=\" + family\n + \"\\nthinned_scorer=\" + RouteRateScore.Variant.THINNED.fingerprint()\n + \"\\ninput=frozen_hourly_top2;labels=forbidden;budget=5/day\"\n + \"\\nbaseline_order=score_desc,hour_support_desc,corridor_support_desc,key,target\"\n + \"\\nroute_group=ordered_R5_start_end\"\n + \"\\nroute_round_robin=one_candidate_per_route_per_round;no_fixed_route_cap;no_threshold\"\n + \"\\nhour_rank_variant=hourly_rank_asc_then_frozen_thinned_order\"\n + \"\\nround_heads=deterministically_sorted;fill_next_round_only_if_budget_remains\"\n + \"\\nno_score_mutation=true;no_named_dates=true;no_outcome_input=true\\n\";\n }\n\n public String fingerprint() {\n return sha256(canonical());\n }\n\n public List selectDay(List hourlyTopTwo) {\n Objects.requireNonNull(hourlyTopTwo);\n validatePool(hourlyTopTwo);\n return switch (family) {\n case RAW_THINNED_BASELINE -> hourlyTopTwo.stream().sorted(THINNED_ORDER).limit(PER_DAY_CAP).toList();\n case ROUTE_ROUND_ROBIN -> roundRobin(hourlyTopTwo, THINNED_ORDER, THINNED_ORDER);\n case HOUR_RANK_ROUTE_ROUND_ROBIN -> roundRobin(hourlyTopTwo, HOUR_RANK_ORDER, HOUR_RANK_ORDER);\n };\n }\n\n private static List roundRobin(List pool,\n Comparator perRouteOrder,\n Comparator roundHeadOrder) {\n Map> byRoute = new TreeMap<>();\n for (Candidate candidate : pool) {\n byRoute.computeIfAbsent(candidate.routeKey(), ignored -> new ArrayList<>()).add(candidate);\n }\n byRoute.values().forEach(rows -> rows.sort(perRouteOrder));\n\n List selected = new ArrayList<>(Math.min(PER_DAY_CAP, pool.size()));\n for (int round = 0; selected.size() < PER_DAY_CAP; round++) {\n List heads = new ArrayList<>();\n for (List routeRows : byRoute.values()) {\n if (round < routeRows.size()) heads.add(routeRows.get(round));\n }\n if (heads.isEmpty()) break;\n heads.sort(roundHeadOrder);\n for (Candidate candidate : heads) {\n if (selected.size() == PER_DAY_CAP) break;\n selected.add(candidate);\n }\n }\n return List.copyOf(selected);\n }\n\n private static void validatePool(List pool) {\n if (pool.size() > 24 * PER_HOUR_CAP\n || pool.stream().map(Candidate::identity).distinct().count() != pool.size()\n || pool.stream().map(c -> c.target().atZone(ZoneOffset.UTC).toLocalDate()).distinct().count() > 1) {\n throw new IllegalArgumentException(\"P861 requires one bounded unique daily hourly-top-two pool\");\n }\n Map perHour = new TreeMap<>();\n for (Candidate candidate : pool) perHour.merge(candidate.target(), 1, Integer::sum);\n if (perHour.values().stream().anyMatch(count -> count > PER_HOUR_CAP)) {\n throw new IllegalArgumentException(\"P861 hourly pool exceeds frozen top-two contract\");\n }\n }\n\n private static String sha256(String value) {\n try {\n return HexFormat.of().formatHex(MessageDigest.getInstance(\"SHA-256\")\n .digest(value.getBytes(StandardCharsets.UTF_8)));\n } catch (Exception failure) {\n throw new IllegalStateException(\"SHA-256 unavailable\", failure);\n }\n }\n}\n" }, { "path": "backend/forecast-model-core/src/main/java/hu/gombos/flightspotter/forecastmodel/P862FrozenPipeline.java", "encoding": "UTF-8", "sha256": "9489a97751fbc25f638d8afe5273fe13d6b29925192c384a98f7003633b4e8fc", "content": "package hu.gombos.flightspotter.forecastmodel;\n\nimport java.time.Duration;\nimport java.time.Instant;\nimport java.time.LocalDate;\nimport java.time.ZoneOffset;\nimport java.time.temporal.ChronoUnit;\nimport java.util.LinkedHashMap;\nimport java.util.Map;\n\n/** Frozen identities and causal invariants for the P8.6.2 prospective-only pipeline. */\npublic final class P862FrozenPipeline {\n public static final String CANDIDATE_GENERATOR_VERSION = \"p862-live-thinned-candidates-v1\";\n public static final String HOURLY_SELECTOR_VERSION = \"p862-frozen-hourly-top2-v1\";\n public static final String FEATURE_SOURCE_VERSION = \"p862-p5v2-durable-causal-r7-v3\";\n public static final String DAILY_DECISION_VERSION = \"p862-daily-portfolio-after-final-t24-snapshot-v1\";\n public static final String OUTCOME_MATURITY_VERSION = \"p862-authoritative-day-d-plus-2-v2\";\n public static final String METRICS_VERSION = \"p862-prospective-metrics-v2\";\n public static final int HOURLY_CAP = 2;\n public static final int DAILY_CAP = 5;\n public static final int WINDOW_DAYS = 28;\n public static final int WARMUP_COMPLETE_DAYS = 56;\n public static final Duration MIN_EVIDENCE_RETENTION = Duration.ofDays(90);\n\n private P862FrozenPipeline() {}\n\n public static Instant asOf(Instant target) {\n requireTarget(target);\n return target.minus(Duration.ofHours(24));\n }\n\n public static void requireExactT24(Instant target, Instant asOf) {\n requireTarget(target);\n if (asOf == null || !asOf.equals(target.minus(Duration.ofHours(24)))) {\n throw new IllegalArgumentException(\"P862 requires exact target-minus-24h asOf\");\n }\n }\n\n public static Instant earliestMaturity(LocalDate targetDay) {\n if (targetDay == null) throw new IllegalArgumentException(\"targetDay required\");\n return targetDay.plusDays(2).atStartOfDay(ZoneOffset.UTC).toInstant();\n }\n\n public static void requireMature(LocalDate targetDay, Instant authoritativeReadyAt, Instant now) {\n Instant minimum = earliestMaturity(targetDay);\n if (authoritativeReadyAt == null || now == null || authoritativeReadyAt.isBefore(minimum)\n || now.isBefore(authoritativeReadyAt)) {\n throw new IllegalStateException(\"P862 truth is not authoritatively mature\");\n }\n }\n\n public static Map fingerprints() {\n LinkedHashMap values = new LinkedHashMap<>();\n values.put(\"thinned_scorer\", RouteRateScore.Variant.THINNED.fingerprint());\n values.put(\"candidate_generator\", hash(CANDIDATE_GENERATOR_VERSION\n + \"|route>=2|hour>=1|corridor>=1|routeLimit=5000|corridorLimit=8\"));\n values.put(\"hourly_top2\", hash(HOURLY_SELECTOR_VERSION\n + \"|score_desc,hour_support_desc,corridor_support_desc,key_asc|cap=2\"));\n values.put(\"daily_selector\", new P861DailySelectionPolicy(\n P861DailySelectionPolicy.Family.ROUTE_ROUND_ROBIN).fingerprint());\n values.put(\"daily_decision\", hash(DAILY_DECISION_VERSION\n + \"|hourly_snapshots=exact_target_minus_24h\"\n + \"|portfolio_decision=after_all_24_target_day_snapshots\"\n + \"|selected_prediction_horizon=1_to_24h_not_exact_t24\"));\n values.put(\"causal_feature\", hash(FEATURE_SOURCE_VERSION\n + \"|summary_created_at<=asOf|last_observed_plus_20m<=asOf\"\n + \"|route_qualified_support_equals_route_support\"\n + \"|source_day=explicit_authoritative_complete_same_authority_ready_by_asOf\"\n + \"|truncated_r7_excluded_from_corridor_support\"\n + \"|weekday_exposure=explicit_authoritative_complete_same_authority_ready_by_asOf\"\n + \"|warmup=56_consecutive_complete_days_ending_start_minus_3\"\n + \"|retention=90_to_400d|p5v2-r7-first-entry\"));\n values.put(\"outcome_maturity\", hash(OUTCOME_MATURITY_VERSION\n + \"|not-before=D+2T00Z|explicit-authoritative-day-seal\"\n + \"|same-frozen-truth-authority-fingerprint|summary-created<=ready|incomplete!=negative\"));\n values.put(\"prospective_metrics\", hash(METRICS_VERSION\n + \"|precision,wilson95,halves,rolling5,daily,coverage,concentration,hourly-top2-top3\"\n + \"|all_672_target_hours_truth_mature_required|activation=false\"));\n return Map.copyOf(values);\n }\n\n\n private static void requireTarget(Instant target) {\n if (target == null || !target.equals(target.truncatedTo(ChronoUnit.HOURS))) {\n throw new IllegalArgumentException(\"P862 target must be an exact UTC hour\");\n }\n }\n\n private static String hash(String canonical) { return DeterministicHashing.sha256(canonical); }\n}\n" }, { "path": "backend/forecast-model-core/src/main/java/hu/gombos/flightspotter/forecastmodel/P862ProspectiveEvaluator.java", "encoding": "UTF-8", "sha256": "9286fbe0015cd6501bfa6d4326aff6c5b340fb93adf5bbb0e3e246e821968261", "content": "package hu.gombos.flightspotter.forecastmodel;\n\nimport java.time.LocalDate;\nimport java.util.ArrayList;\nimport java.util.Comparator;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.TreeMap;\n\n/** Deterministic P862 evidence evaluator. It has no activation capability. */\npublic final class P862ProspectiveEvaluator {\n public enum Outcome { UNMATURED, POSITIVE, NEGATIVE, INELIGIBLE_OVERFLOW, UNKNOWN_INCOMPLETE_TRUTH }\n public enum Decision { PROSPECTIVE_PASS, PROSPECTIVE_FAIL, INCOMPLETE_EVIDENCE }\n\n public record Observation(LocalDate day, String routeKey, Outcome outcome) {\n public Observation {\n if (day == null || routeKey == null || routeKey.isBlank() || outcome == null) {\n throw new IllegalArgumentException(\"Invalid P862 observation\");\n }\n }\n }\n\n public record Coverage(long expectedTargetHours, long forecastedTargetHours, long missedCausalInvocations,\n long candidateGenerationFailures, long maturedTargetHours,\n long unknownIncompleteOutcomes, long expectedBudget, long selected) {}\n\n public record Report(Decision decision, boolean productionActivationAuthorized, long selected, long positive,\n long negative, long unknown, double precision, double wilsonLower95,\n double firstHalfPrecision, double secondHalfPrecision, double minimumRollingFivePrecision,\n double minimumDailyPrecision, int minimumUniqueRoutesPerDay, int maximumSlotsPerRoutePerDay,\n long repeatedRouteSlots, Coverage coverage, List reasons) {}\n\n public Report evaluate(Coverage coverage, List input) {\n List observations = List.copyOf(input);\n long positive = observations.stream().filter(v -> v.outcome() == Outcome.POSITIVE).count();\n long negative = observations.stream().filter(v -> v.outcome() == Outcome.NEGATIVE).count();\n long unknown = observations.stream().filter(v -> v.outcome() == Outcome.UNMATURED\n || v.outcome() == Outcome.UNKNOWN_INCOMPLETE_TRUTH\n || v.outcome() == Outcome.INELIGIBLE_OVERFLOW).count();\n long known = positive + negative;\n double precision = ratio(positive, known);\n double wilson = wilsonLower(positive, known);\n TreeMap> daily = new TreeMap<>();\n observations.forEach(value -> daily.computeIfAbsent(value.day(), ignored -> new ArrayList<>()).add(value));\n List days = new ArrayList<>(daily.keySet());\n int split = (days.size() + 1) / 2;\n double first = precision(days.subList(0, split), daily);\n double second = precision(days.subList(split, days.size()), daily);\n double minimumDaily = days.stream().mapToDouble(day -> precision(List.of(day), daily)).min().orElse(0d);\n double minimumRolling = rollingFive(days, daily);\n int minRoutes = days.stream().mapToInt(day -> (int) daily.get(day).stream()\n .map(Observation::routeKey).distinct().count()).min().orElse(0);\n int maxSlots = days.stream().mapToInt(day -> daily.get(day).stream().collect(\n java.util.stream.Collectors.groupingBy(Observation::routeKey, java.util.stream.Collectors.counting()))\n .values().stream().mapToInt(Long::intValue).max().orElse(0)).max().orElse(0);\n long repeated = days.stream().mapToLong(day -> daily.get(day).size() - daily.get(day).stream()\n .map(Observation::routeKey).distinct().count()).sum();\n\n List reasons = new ArrayList<>();\n boolean complete = coverage.expectedTargetHours() == coverage.forecastedTargetHours()\n + coverage.missedCausalInvocations() + coverage.candidateGenerationFailures()\n && coverage.forecastedTargetHours() == coverage.expectedTargetHours()\n && coverage.maturedTargetHours() == coverage.expectedTargetHours()\n && coverage.missedCausalInvocations() == 0 && coverage.candidateGenerationFailures() == 0\n && coverage.unknownIncompleteOutcomes() == 0 && unknown == 0\n && coverage.selected() == coverage.expectedBudget() && observations.size() == coverage.selected();\n if (!complete) reasons.add(\"INCOMPLETE_COVERAGE_OR_TRUTH\");\n if (precision < .88d) reasons.add(\"PRECISION_BELOW_0_88\");\n if (wilson < .78d) reasons.add(\"WILSON_LOWER_BELOW_0_78\");\n if (first < .80d) reasons.add(\"FIRST_HALF_BELOW_0_80\");\n if (second < .80d) reasons.add(\"SECOND_HALF_BELOW_0_80\");\n if (minimumRolling < .80d) reasons.add(\"ROLLING_FIVE_BELOW_0_80\");\n if (minimumDaily < .40d) reasons.add(\"DAILY_BELOW_0_40\");\n Decision decision = !complete ? Decision.INCOMPLETE_EVIDENCE\n : reasons.isEmpty() ? Decision.PROSPECTIVE_PASS : Decision.PROSPECTIVE_FAIL;\n return new Report(decision, false, observations.size(), positive, negative, unknown, precision, wilson,\n first, second, minimumRolling, minimumDaily, minRoutes, maxSlots, repeated, coverage,\n List.copyOf(reasons));\n }\n\n private static double precision(List days, Map> daily) {\n long p = 0, n = 0;\n for (LocalDate day : days) for (Observation value : daily.get(day)) {\n if (value.outcome() == Outcome.POSITIVE) p++;\n if (value.outcome() == Outcome.NEGATIVE) n++;\n }\n return ratio(p, p + n);\n }\n\n private static double rollingFive(List days, Map> daily) {\n if (days.size() < 5) return 0d;\n double minimum = 1d;\n for (int i = 0; i <= days.size() - 5; i++) {\n minimum = Math.min(minimum, precision(days.subList(i, i + 5), daily));\n }\n return minimum;\n }\n\n private static double ratio(long numerator, long denominator) {\n return denominator == 0 ? 0d : (double) numerator / denominator;\n }\n\n static double wilsonLower(long successes, long total) {\n if (total == 0) return 0d;\n double z = 1.959963984540054d, p = (double) successes / total, z2 = z * z;\n return (p + z2 / (2d * total) - z * Math.sqrt(p * (1d - p) / total + z2 / (4d * total * total)))\n / (1d + z2 / total);\n }\n}\n" }, { "path": "backend/forecast-model-core/src/main/java/hu/gombos/flightspotter/forecastmodel/P862RegionalHistoricalPipeline.java", "encoding": "UTF-8", "sha256": "9c53de092bae5ffa1e3d9a2596e819ffb19696948f34956c928ede1ec6151ce8", "content": "package hu.gombos.flightspotter.forecastmodel;\nimport java.util.*;\n/** Region-aware P862 generation: frozen scoring/selection math, regional historical evidence authority. */\npublic final class P862RegionalHistoricalPipeline {\n public static final String EXPERIMENT_ID=\"p862-regional-historical-prospective-v1\";\n public static final String CANDIDATE_GENERATOR_VERSION=\"p862-regional-historical-thinned-candidates-v1\";\n public static final String FEATURE_SOURCE_VERSION=\"p862-qualified-regional-historical-r7-v1\";\n public static final String OUTCOME_MATURITY_VERSION=\"p862-regional-historical-day-authority-v1\";\n public static final String METRICS_VERSION=\"p862-regional-historical-prospective-metrics-v1\";\n private P862RegionalHistoricalPipeline(){}\n public static Map fingerprints(){\n LinkedHashMap v=new LinkedHashMap<>();\n v.put(\"thinned_scorer\",RouteRateScore.Variant.THINNED.fingerprint());\n v.put(\"candidate_generator\",hash(CANDIDATE_GENERATOR_VERSION+\"|route>=2|hour>=1|corridor>=1|routeLimit=5000|corridorLimit=8|source=qualified-regional-historical-days\"));\n v.put(\"hourly_top2\",P862FrozenPipeline.fingerprints().get(\"hourly_top2\"));\n v.put(\"daily_selector\",P862FrozenPipeline.fingerprints().get(\"daily_selector\"));\n v.put(\"daily_decision\",P862FrozenPipeline.fingerprints().get(\"daily_decision\"));\n v.put(\"causal_feature\",hash(FEATURE_SOURCE_VERSION+\"|day-seal-authority=REGIONAL_HISTORICAL_TRUTH_AUTHORITY_V1|dataset-state=READY|terminal-quality=QUALIFIED|ready<=asOf|route-qualified-support=route-support|weekday-hour=ready-hour-bins|corridor=ready-route-corridor-aggregates|warmup=56-consecutive-complete-days|retention=90-to-400d\"));\n v.put(\"outcome_maturity\",hash(OUTCOME_MATURITY_VERSION+\"|sealed-qualified-source-day|authority-fingerprint-match|gap-or-degraded=unknown|aggregate-ambiguity=ineligible|utc-boundary-hours=ineligible|never-false-negative\"));\n v.put(\"prospective_metrics\",hash(METRICS_VERSION+\"|precision,wilson95,halves,rolling5,daily,coverage,concentration,hourly-top2-top3|all_672_target_hours_truth_mature_required|activation=false\"));\n return Map.copyOf(v);\n }\n public static String combinedFingerprint(){StringBuilder c=new StringBuilder(EXPERIMENT_ID).append('\\n');fingerprints().entrySet().stream().sorted(Map.Entry.comparingByKey()).forEach(e->c.append(e.getKey()).append('=').append(e.getValue()).append('\\n'));return DeterministicHashing.sha256(c.toString());}\n private static String hash(String c){return DeterministicHashing.sha256(c);}\n}" }, { "path": "backend/forecast-model-core/src/main/java/hu/gombos/flightspotter/forecastmodel/RegionalHistoricalTruthAuthority.java", "encoding": "UTF-8", "sha256": "bee4f8b72099c8d243c3ce603e7a1178b1f657ee38d8e011e5b21592f6b4d590", "content": "package hu.gombos.flightspotter.forecastmodel;\n\nimport java.time.Duration;\nimport java.util.Objects;\nimport java.util.regex.Pattern;\n\n/** Versioned region-wide historical truth contract for T+24; independent from LIVE acquisition authority. */\npublic record RegionalHistoricalTruthAuthority(\n CoverageRegionId regionId,\n String regionCanonicalSha256,\n String coverageVersion,\n String evidenceCoverageHash,\n String provider,\n String sourceRepositoryPolicy,\n String licenseIdentifier,\n String sourceManifestPolicyVersion,\n String importerVersion,\n String parserVersion,\n String archiveSafetyProfile,\n String eligibilityVersion,\n String segmentationVersion,\n String derivationVersion,\n String completenessSemanticsVersion,\n Duration maturityAfterDayEnd,\n String sealSemanticsVersion\n) {\n public static final String GENERATION=\"REGIONAL_HISTORICAL_TRUTH_AUTHORITY_V1\";\n public static final String ADSB_LOL_REPOSITORY_POLICY=\"adsblol/globe_history_{yyyy}\";\n public static final String DEFAULT_MANIFEST_POLICY=\"P7_SOURCE_MANIFEST_SCHEMA_V1_DAY_SHA256_V1\";\n public static final String DEFAULT_COMPLETENESS=\"QUALIFIED_REGION_SOURCE_DAY_BOUNDARY_HOURS_INELIGIBLE_V1\";\n public static final String DEFAULT_SEAL=\"IMMUTABLE_DIGEST_SEAL_READY_AT_MAX_SOURCE_MATURITY_AND_CREATION_V1\";\n private static final Pattern STABLE=Pattern.compile(\"[A-Za-z0-9._:/|=,+{}-]{1,512}\");\n private static final Pattern SHA256=Pattern.compile(\"[0-9a-f]{64}\");\n\n public RegionalHistoricalTruthAuthority {\n Objects.requireNonNull(regionId,\"regionId\");\n requireSha(regionCanonicalSha256,\"regionCanonicalSha256\");\n requireStable(coverageVersion,\"coverageVersion\"); requireSha(evidenceCoverageHash,\"evidenceCoverageHash\");\n requireStable(provider,\"provider\"); requireStable(sourceRepositoryPolicy,\"sourceRepositoryPolicy\");\n requireStable(licenseIdentifier,\"licenseIdentifier\"); requireStable(sourceManifestPolicyVersion,\"sourceManifestPolicyVersion\");\n requireStable(importerVersion,\"importerVersion\"); requireStable(parserVersion,\"parserVersion\");\n requireStable(archiveSafetyProfile,\"archiveSafetyProfile\"); requireStable(eligibilityVersion,\"eligibilityVersion\");\n requireStable(segmentationVersion,\"segmentationVersion\"); requireStable(derivationVersion,\"derivationVersion\");\n requireStable(completenessSemanticsVersion,\"completenessSemanticsVersion\");\n Objects.requireNonNull(maturityAfterDayEnd,\"maturityAfterDayEnd\");\n if(maturityAfterDayEnd.isNegative()||maturityAfterDayEnd.compareTo(Duration.ofDays(14))>0) throw new IllegalArgumentException(\"maturityAfterDayEnd must be bounded to [0,14d]\");\n requireStable(sealSemanticsVersion,\"sealSemanticsVersion\");\n }\n public String fingerprint(){return DeterministicHashing.sha256(canonical());}\n public String canonical(){return GENERATION+'\\n'\n +\"region=\"+regionId.value()+'\\n'\n +\"region_canonical_sha256=\"+regionCanonicalSha256+'\\n'\n +\"coverage_version=\"+coverageVersion+'\\n'\n +\"evidence_coverage_hash=\"+evidenceCoverageHash+'\\n'\n +\"provider=\"+provider+'\\n'\n +\"source_repository_policy=\"+sourceRepositoryPolicy+'\\n'\n +\"license=\"+licenseIdentifier+'\\n'\n +\"source_manifest_policy=\"+sourceManifestPolicyVersion+'\\n'\n +\"importer=\"+importerVersion+'\\n'\n +\"parser=\"+parserVersion+'\\n'\n +\"archive_safety=\"+archiveSafetyProfile+'\\n'\n +\"eligibility=\"+eligibilityVersion+'\\n'\n +\"segmentation=\"+segmentationVersion+'\\n'\n +\"derivation=\"+derivationVersion+'\\n'\n +\"completeness=\"+completenessSemanticsVersion+'\\n'\n +\"maturity_after_day_end_seconds=\"+maturityAfterDayEnd.toSeconds()+'\\n'\n +\"seal=\"+sealSemanticsVersion+'\\n';}\n private static void requireSha(String v,String f){if(v==null||!SHA256.matcher(v).matches())throw new IllegalArgumentException(f+\" must be lowercase SHA-256\");}\n private static void requireStable(String v,String f){if(v==null||!STABLE.matcher(v.trim()).matches())throw new IllegalArgumentException(f+\" is missing, unstable, or unbounded\");}\n}" }, { "path": "backend/forecast-model-core/src/main/java/hu/gombos/flightspotter/forecastmodel/RouteRateScore.java", "encoding": "UTF-8", "sha256": "8b683b7688d2aa4d55b26eeb924438f29f7dc0185185626edf3c7404109088cb", "content": "package hu.gombos.flightspotter.forecastmodel;\n\nimport java.math.BigDecimal;\nimport java.math.RoundingMode;\n\n/** P8.5.3 offline hypotheses. These are ranking scores, not calibrated probabilities. */\npublic final class RouteRateScore {\n private RouteRateScore() {}\n\n public enum Variant {\n THINNED(\"p853-route-thinned-rate-v1\", 0),\n SHRUNK(\"p853-route-shrunk-rate-v1\", 1);\n\n private final String version;\n private final int extraExposure;\n Variant(String version, int extraExposure) {\n this.version = version;\n this.extraExposure = extraExposure;\n }\n public String version() { return version; }\n public String canonical() {\n return \"version=\" + version + \"\\nformula=1-exp(-nHour*nQualified*nCell/(nRoute^2*(occurrences+extraExposure)))\\n\"\n + \"extraExposure=\" + extraExposure + \"\\nrate_scale=18\\nscore_scale=10\\nrounding=HALF_EVEN\\n\"\n + \"exponential=StrictMath.expm1\\ncalibration=NONE\\n\";\n }\n public String fingerprint() { return DeterministicHashing.sha256(canonical()); }\n }\n\n public record Components(BigDecimal effectiveRate, BigDecimal score) {}\n\n public static Components calculate(Variant variant, long route, long qualified, long hour,\n long cell, long occurrences) {\n if (variant == null || route <= 0 || qualified < 0 || qualified > route || hour < 0\n || hour > route || cell < 0 || cell > route || occurrences <= 0) {\n throw new IllegalArgumentException(\"Invalid P853 recurrence counts\");\n }\n // Decimal products avoid both long overflow and intermediate double rounding.\n BigDecimal numerator = BigDecimal.valueOf(hour).multiply(BigDecimal.valueOf(qualified))\n .multiply(BigDecimal.valueOf(cell));\n BigDecimal denominator = BigDecimal.valueOf(route).pow(2)\n .multiply(BigDecimal.valueOf(occurrences).add(BigDecimal.valueOf(variant.extraExposure)));\n BigDecimal rate = numerator.divide(denominator, 18, RoundingMode.HALF_EVEN);\n BigDecimal score = BigDecimal.valueOf(-StrictMath.expm1(-rate.doubleValue()))\n .setScale(10, RoundingMode.HALF_EVEN);\n if (score.signum() < 0 || score.compareTo(BigDecimal.ONE) > 0) {\n throw new IllegalStateException(\"Invalid P853 score\");\n }\n return new Components(rate, score);\n }\n}\n" }, { "path": "backend/historical-bootstrap-tool/src/main/java/hu/gombos/flightspotter/historicalbootstrap/ArchiveFamilyResolver.java", "encoding": "UTF-8", "sha256": "de469041c40d97586e0e34cd2d75ef8c45e71f60699db63cfa10801d4ada1f4f", "content": "package hu.gombos.flightspotter.historicalbootstrap;\n\nimport java.util.ArrayList;\nimport java.util.Comparator;\nimport java.util.LinkedHashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.regex.Matcher;\nimport java.util.regex.Pattern;\n\nfinal class ArchiveFamilyResolver {\n private static final Pattern SPLIT = Pattern.compile(\"^(.*\\\\.tar)\\\\.([a-z]{2})$\");\n\n FamilyResolution resolve(List assets) {\n List unsplit = assets.stream()\n .filter(asset -> asset.name().endsWith(\".tar\"))\n .sorted(Comparator.comparing(SourceAsset::name).thenComparingLong(SourceAsset::assetId))\n .toList();\n\n Map> splitFamilies = new LinkedHashMap<>();\n for (SourceAsset asset : assets) {\n Matcher matcher = SPLIT.matcher(asset.name());\n if (matcher.matches()) {\n splitFamilies.computeIfAbsent(matcher.group(1), ignored -> new ArrayList<>())\n .add(new SplitAsset(asset, matcher.group(2)));\n }\n }\n\n if (!unsplit.isEmpty() && !splitFamilies.isEmpty()) {\n return FamilyResolution.failed(SourcePreflightStatus.ASSET_FAMILY_AMBIGUOUS);\n }\n if (unsplit.size() > 1 || splitFamilies.size() > 1) {\n return FamilyResolution.failed(SourcePreflightStatus.ASSET_FAMILY_AMBIGUOUS);\n }\n if (unsplit.size() == 1) {\n return FamilyResolution.resolved(new ResolvedArchiveFamily(ArchiveFamilyKind.UNSPLIT, unsplit));\n }\n if (splitFamilies.isEmpty()) {\n return FamilyResolution.failed(SourcePreflightStatus.NO_ARCHIVE_ASSETS);\n }\n\n List splitAssets = splitFamilies.values().iterator().next().stream()\n .sorted(Comparator.comparing(SplitAsset::suffix).thenComparing(value -> value.asset().name()))\n .toList();\n if (!contiguousFromAa(splitAssets)) {\n return FamilyResolution.failed(SourcePreflightStatus.SPLIT_SEQUENCE_INCOMPLETE);\n }\n return FamilyResolution.resolved(new ResolvedArchiveFamily(\n ArchiveFamilyKind.SPLIT,\n splitAssets.stream().map(SplitAsset::asset).toList()\n ));\n }\n\n private boolean contiguousFromAa(List assets) {\n if (assets.isEmpty() || !\"aa\".equals(assets.getFirst().suffix())) return false;\n String expected = \"aa\";\n for (SplitAsset asset : assets) {\n if (!expected.equals(asset.suffix())) return false;\n expected = nextSuffix(expected);\n }\n return true;\n }\n\n private String nextSuffix(String suffix) {\n char first = suffix.charAt(0);\n char second = suffix.charAt(1);\n if (second < 'z') return \"\" + first + (char) (second + 1);\n if (first < 'z') return \"\" + (char) (first + 1) + 'a';\n return \"{\"; // no valid two-letter suffix can follow zz\n }\n\n record FamilyResolution(SourcePreflightStatus status, ResolvedArchiveFamily family) {\n static FamilyResolution resolved(ResolvedArchiveFamily family) {\n return new FamilyResolution(SourcePreflightStatus.PREFLIGHT_OK, family);\n }\n\n static FamilyResolution failed(SourcePreflightStatus status) {\n return new FamilyResolution(status, null);\n }\n\n boolean ok() {\n return status == SourcePreflightStatus.PREFLIGHT_OK;\n }\n }\n\n private record SplitAsset(SourceAsset asset, String suffix) {\n }\n}\n" }, { "path": "backend/historical-bootstrap-tool/src/main/java/hu/gombos/flightspotter/historicalbootstrap/ArchiveSafetyLimits.java", "encoding": "UTF-8", "sha256": "5a574e8de612c60350ca461610561a418e1da1b8b5ad077d33015fde3fdd15e3", "content": "package hu.gombos.flightspotter.historicalbootstrap;\n\n/**\n * Hard safety bounds for the offline P7 historical parser. These are parser guardrails, not the\n * P7.4 measured machine/resource budget. P7.4 may recalibrate them only through a new semantic\n * generation after real-data evidence; the complete default profile is fingerprinted.\n */\nrecord ArchiveSafetyLimits(\n int maxTarEntries,\n long maxCompressedTraceBytes,\n int maxInflatedTraceBytes,\n int maxTracePoints,\n int maxCarryAircraft\n) {\n static final String SEMANTIC_VERSION = \"p7-archive-safety-v2\";\n\n static ArchiveSafetyLimits defaults() {\n return new ArchiveSafetyLimits(\n 1_000_000,\n 8L * 1024L * 1024L,\n 64 * 1024 * 1024,\n 500_000,\n 250_000\n );\n }\n\n ArchiveSafetyLimits {\n if (maxTarEntries < 1 || maxTarEntries > 5_000_000\n || maxCompressedTraceBytes < 1 || maxCompressedTraceBytes > 128L * 1024L * 1024L\n || maxInflatedTraceBytes < 1 || maxInflatedTraceBytes > 512 * 1024 * 1024\n || maxTracePoints < 1 || maxTracePoints > 2_000_000\n || maxCarryAircraft < 1 || maxCarryAircraft > 1_000_000) {\n throw new IllegalArgumentException(\"Invalid bounded historical archive limits\");\n }\n }\n\n String semanticFingerprintComponent() {\n return SEMANTIC_VERSION\n + \"|maxTarEntries=\" + maxTarEntries\n + \"|maxCompressedTraceBytes=\" + maxCompressedTraceBytes\n + \"|maxInflatedTraceBytes=\" + maxInflatedTraceBytes\n + \"|maxTracePoints=\" + maxTracePoints\n + \"|maxCarryAircraft=\" + maxCarryAircraft;\n }\n}\n" }, { "path": "backend/historical-bootstrap-tool/src/main/java/hu/gombos/flightspotter/historicalbootstrap/BootstrapCoverageSpec.java", "encoding": "UTF-8", "sha256": "18bfb979b707e567e5534ed9493de52f12ff339f298e11a999addd55e9d671c9", "content": "package hu.gombos.flightspotter.historicalbootstrap;\n\nimport hu.gombos.flightspotter.common.geo.H3Service;\nimport hu.gombos.flightspotter.forecastmodel.CoverageDomain;\n\nimport java.nio.charset.StandardCharsets;\nimport java.security.MessageDigest;\nimport java.security.NoSuchAlgorithmException;\nimport java.util.HexFormat;\nimport java.util.Objects;\nimport java.util.Set;\nimport java.util.TreeSet;\n\n/** Immutable H3 R5 coverage boundary used before any historical contribution is retained. */\nfinal class BootstrapCoverageSpec {\n private static final int H3_RESOLUTION = 5;\n private static final int CORRIDOR_H3_RESOLUTION = 7;\n\n private final String version;\n private final Set allowedH3R5;\n private final H3Service h3;\n private final String hash;\n\n BootstrapCoverageSpec(String version, Set allowedH3R5, H3Service h3) {\n if (version == null || version.isBlank() || version.length() > 80) {\n throw new IllegalArgumentException(\"coverage version is required and bounded\");\n }\n Objects.requireNonNull(allowedH3R5, \"allowedH3R5\");\n Objects.requireNonNull(h3, \"h3\");\n if (allowedH3R5.isEmpty() || allowedH3R5.size() > 500_000) {\n throw new IllegalArgumentException(\"coverage cell set must be non-empty and bounded\");\n }\n TreeSet normalized = new TreeSet<>();\n for (String cell : allowedH3R5) {\n if (cell == null || !cell.matches(\"85[0-9a-f]{13}\")) {\n throw new IllegalArgumentException(\"coverage contains non-R5 H3 cell\");\n }\n normalized.add(cell);\n }\n this.version = version.trim();\n this.allowedH3R5 = Set.copyOf(normalized);\n this.h3 = h3;\n this.hash = hash(this.version, normalized);\n }\n\n static BootstrapCoverageSpec fromDomain(CoverageDomain domain, H3Service h3) {\n Objects.requireNonNull(domain, \"domain\");\n if (domain.h3Resolution() != H3_RESOLUTION) {\n throw new IllegalArgumentException(\"P7 historical evidence requires an H3 R5 coverage domain\");\n }\n BootstrapCoverageSpec spec = new BootstrapCoverageSpec(\n domain.coverageVersion(), Set.copyOf(domain.orderedH3Cells()), h3);\n if (!spec.hash().equals(domain.evidenceCoverageHash())) {\n throw new IllegalStateException(\"regional coverage domain/P7 evidence hash mismatch\");\n }\n return spec;\n }\n\n String cellFor(double latitude, double longitude) {\n return h3.latLonToCell(latitude, longitude, H3_RESOLUTION);\n }\n\n String corridorCellFor(double latitude, double longitude) {\n return h3.latLonToCell(latitude, longitude, CORRIDOR_H3_RESOLUTION);\n }\n\n boolean containsCell(String cell) {\n return allowedH3R5.contains(cell);\n }\n\n String version() {\n return version;\n }\n\n String hash() {\n return hash;\n }\n\n Set allowedH3R5() {\n return allowedH3R5;\n }\n\n private static String hash(String version, Set cells) {\n try {\n MessageDigest digest = MessageDigest.getInstance(\"SHA-256\");\n update(digest, version);\n for (String cell : cells) {\n update(digest, cell);\n }\n return HexFormat.of().formatHex(digest.digest());\n } catch (NoSuchAlgorithmException exception) {\n throw new IllegalStateException(\"SHA-256 unavailable\", exception);\n }\n }\n\n private static void update(MessageDigest digest, String value) {\n byte[] bytes = value.getBytes(StandardCharsets.UTF_8);\n digest.update((byte) (bytes.length >>> 24));\n digest.update((byte) (bytes.length >>> 16));\n digest.update((byte) (bytes.length >>> 8));\n digest.update((byte) bytes.length);\n digest.update(bytes);\n }\n}\n" }, { "path": "backend/historical-bootstrap-tool/src/main/java/hu/gombos/flightspotter/historicalbootstrap/BootstrapSourceManifest.java", "encoding": "UTF-8", "sha256": "1054061fa6f1028fbf4e56a1ac4f6fae342d60f77059b4616e6608b133c10ab8", "content": "package hu.gombos.flightspotter.historicalbootstrap;\n\nimport java.nio.ByteBuffer;\nimport java.nio.charset.StandardCharsets;\nimport java.security.MessageDigest;\nimport java.security.NoSuchAlgorithmException;\nimport java.time.LocalDate;\nimport java.util.ArrayList;\nimport java.util.Comparator;\nimport java.util.LinkedHashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Objects;\n\nrecord BootstrapSourceManifest(\n int schemaVersion,\n String provider,\n DatasetWindow window,\n PreflightPolicy policy,\n List days,\n String manifestSha256\n) {\n BootstrapSourceManifest {\n if (schemaVersion != 1) throw new IllegalArgumentException(\"unsupported manifest schema version\");\n if (provider == null || provider.isBlank()) throw new IllegalArgumentException(\"provider is required\");\n Objects.requireNonNull(window, \"window\");\n Objects.requireNonNull(policy, \"policy\");\n if (days == null || days.size() != window.calendarDayCount()) {\n throw new IllegalArgumentException(\"manifest must contain exactly one row per calendar day\");\n }\n days = days.stream().sorted(Comparator.comparing(ResolvedSourceDay::sourceDate)).toList();\n LocalDate expected = window.fromDate();\n for (ResolvedSourceDay day : days) {\n if (!day.sourceDate().equals(expected)) {\n throw new IllegalArgumentException(\"manifest day sequence has a gap or duplicate at \" + expected);\n }\n if (day.role() != window.roleFor(day.sourceDate())) {\n throw new IllegalArgumentException(\"manifest day role does not match dataset window\");\n }\n expected = expected.plusDays(1);\n }\n String calculated = ManifestHasher.hash(schemaVersion, provider, window, policy, days);\n if (manifestSha256 == null) manifestSha256 = calculated;\n if (!manifestSha256.equals(calculated)) {\n throw new IllegalArgumentException(\"manifest SHA-256 does not match canonical content\");\n }\n }\n\n static BootstrapSourceManifest create(\n DatasetWindow window,\n PreflightPolicy policy,\n List days\n ) {\n return new BootstrapSourceManifest(1, \"ADSB_LOL\", window, policy, days, null);\n }\n\n String dayManifestSha256(LocalDate sourceDate) {\n if (sourceDate == null) throw new IllegalArgumentException(\"sourceDate is required\");\n ResolvedSourceDay day = days.stream()\n .filter(value -> value.sourceDate().equals(sourceDate))\n .findFirst()\n .orElseThrow(() -> new IllegalArgumentException(\"sourceDate is outside the manifest\"));\n return ManifestHasher.hashDay(schemaVersion, provider, window, policy, day);\n }\n\n Map toJsonModel() {\n Map root = new LinkedHashMap<>();\n root.put(\"schemaVersion\", schemaVersion);\n root.put(\"provider\", provider);\n root.put(\"manifestSha256\", manifestSha256);\n root.put(\"purpose\", window.purpose().name());\n root.put(\"fromDate\", window.fromDate().toString());\n root.put(\"toDate\", window.toDate().toString());\n root.put(\"asOfDate\", window.asOfDate() == null ? null : window.asOfDate().toString());\n root.put(\"lookbackDays\", window.lookbackDays());\n root.put(\"learningEnd\", date(window.learningEnd()));\n root.put(\"embargoDate\", date(window.embargoDate()));\n root.put(\"holdoutStart\", date(window.holdoutStart()));\n root.put(\"minimumArchiveBytes\", policy.minimumArchiveBytes());\n root.put(\"maximumArchiveBytes\", policy.maximumArchiveBytes());\n\n List> outputDays = new ArrayList<>();\n for (ResolvedSourceDay day : days) {\n Map value = new LinkedHashMap<>();\n value.put(\"sourceDate\", day.sourceDate().toString());\n value.put(\"dayManifestSha256\", dayManifestSha256(day.sourceDate()));\n value.put(\"role\", day.role().name());\n value.put(\"sourcePreflightStatus\", day.preflightStatus().name());\n value.put(\"overrideReason\", day.overrideReason());\n value.put(\"candidateReleases\", day.candidateReleases().stream()\n .map(BootstrapSourceManifest::releaseJson)\n .toList());\n if (day.selectedRelease() != null) {\n SourceRelease release = day.selectedRelease();\n value.put(\"selectedReleaseId\", release.releaseId());\n value.put(\"selectedArchiveFamily\", day.selectedFamily().kind().name());\n value.put(\"selectedArchiveBytes\", day.selectedFamily().totalBytes());\n value.put(\"selectedAssetIds\", day.selectedFamily().assets().stream()\n .map(SourceAsset::assetId)\n .toList());\n } else {\n value.put(\"selectedReleaseId\", null);\n value.put(\"selectedArchiveFamily\", null);\n value.put(\"selectedArchiveBytes\", null);\n value.put(\"selectedAssetIds\", List.of());\n }\n outputDays.add(value);\n }\n root.put(\"days\", outputDays);\n return root;\n }\n\n private static Map releaseJson(SourceRelease release) {\n Map value = new LinkedHashMap<>();\n value.put(\"repository\", release.repository());\n value.put(\"releaseId\", release.releaseId());\n value.put(\"releaseTag\", release.tag());\n value.put(\"releaseVariant\", release.variant().name());\n value.put(\"releaseDraft\", release.draft());\n value.put(\"releasePrerelease\", release.prerelease());\n value.put(\"assets\", release.assets().stream().map(BootstrapSourceManifest::assetJson).toList());\n return value;\n }\n\n private static Map assetJson(SourceAsset asset) {\n Map value = new LinkedHashMap<>();\n value.put(\"assetId\", asset.assetId());\n value.put(\"name\", asset.name());\n value.put(\"byteSize\", asset.byteSize());\n value.put(\"sha256\", asset.sha256());\n return value;\n }\n\n private static String date(LocalDate value) {\n return value == null ? null : value.toString();\n }\n\n private static final class ManifestHasher {\n private ManifestHasher() {\n }\n\n static String hash(\n int schemaVersion,\n String provider,\n DatasetWindow window,\n PreflightPolicy policy,\n List days\n ) {\n MessageDigest digest = sha256();\n putWindowAndPolicy(digest, schemaVersion, provider, window, policy);\n for (ResolvedSourceDay day : days.stream()\n .sorted(Comparator.comparing(ResolvedSourceDay::sourceDate)).toList()) {\n putDay(digest, day);\n }\n return java.util.HexFormat.of().formatHex(digest.digest());\n }\n\n static String hashDay(\n int schemaVersion,\n String provider,\n DatasetWindow window,\n PreflightPolicy policy,\n ResolvedSourceDay day\n ) {\n MessageDigest digest = sha256();\n put(digest, \"p7-day-manifest-v1\");\n putWindowAndPolicy(digest, schemaVersion, provider, window, policy);\n putDay(digest, day);\n return java.util.HexFormat.of().formatHex(digest.digest());\n }\n\n private static void putWindowAndPolicy(\n MessageDigest digest,\n int schemaVersion,\n String provider,\n DatasetWindow window,\n PreflightPolicy policy\n ) {\n put(digest, Integer.toString(schemaVersion));\n put(digest, provider);\n put(digest, window.purpose().name());\n put(digest, window.fromDate().toString());\n put(digest, window.toDate().toString());\n put(digest, date(window.asOfDate()));\n put(digest, window.lookbackDays() == null ? null : window.lookbackDays().toString());\n put(digest, date(window.learningEnd()));\n put(digest, date(window.embargoDate()));\n put(digest, date(window.holdoutStart()));\n put(digest, policy.minimumArchiveBytes() == null ? null : policy.minimumArchiveBytes().toString());\n put(digest, policy.maximumArchiveBytes() == null ? null : policy.maximumArchiveBytes().toString());\n }\n\n private static void putDay(MessageDigest digest, ResolvedSourceDay day) {\n put(digest, day.sourceDate().toString());\n put(digest, day.role().name());\n put(digest, day.preflightStatus().name());\n put(digest, day.overrideReason());\n for (SourceRelease candidate : day.candidateReleases()) {\n putRelease(digest, candidate);\n }\n put(digest, \"END_CANDIDATES\");\n SourceRelease selected = day.selectedRelease();\n if (selected == null) {\n put(digest, null);\n return;\n }\n put(digest, Long.toString(selected.releaseId()));\n put(digest, day.selectedFamily().kind().name());\n put(digest, Long.toString(day.selectedFamily().totalBytes()));\n for (SourceAsset asset : day.selectedFamily().assets()) {\n put(digest, Long.toString(asset.assetId()));\n }\n }\n\n private static void putRelease(MessageDigest digest, SourceRelease release) {\n put(digest, release.repository());\n put(digest, Long.toString(release.releaseId()));\n put(digest, release.tag());\n put(digest, release.variant().name());\n put(digest, Boolean.toString(release.draft()));\n put(digest, Boolean.toString(release.prerelease()));\n for (SourceAsset asset : release.assets()) {\n put(digest, Long.toString(asset.assetId()));\n put(digest, asset.name());\n put(digest, Long.toString(asset.byteSize()));\n put(digest, asset.sha256());\n }\n put(digest, \"END_RELEASE\");\n }\n\n private static MessageDigest sha256() {\n try {\n return MessageDigest.getInstance(\"SHA-256\");\n } catch (NoSuchAlgorithmException exception) {\n throw new IllegalStateException(\"SHA-256 unavailable\", exception);\n }\n }\n\n private static void put(MessageDigest digest, String value) {\n if (value == null) {\n digest.update(ByteBuffer.allocate(4).putInt(-1).array());\n return;\n }\n byte[] bytes = value.getBytes(StandardCharsets.UTF_8);\n digest.update(ByteBuffer.allocate(4).putInt(bytes.length).array());\n digest.update(bytes);\n }\n }\n}\n" }, { "path": "backend/historical-bootstrap-tool/src/main/java/hu/gombos/flightspotter/historicalbootstrap/BootstrapSourceManifestIO.java", "encoding": "UTF-8", "sha256": "fa1c5b637eb140bac63c7c8f9c7d7088764041cff39d507ab421f29ca76dab37", "content": "package hu.gombos.flightspotter.historicalbootstrap;\n\nimport com.fasterxml.jackson.annotation.JsonProperty;\nimport tools.jackson.databind.ObjectMapper;\nimport tools.jackson.databind.json.JsonMapper;\n\nimport java.nio.file.Path;\nimport java.time.LocalDate;\nimport java.util.ArrayList;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\n\n/** Reads a persisted P7.1 manifest and reconstructs the canonical in-memory model, re-verifying its SHA-256. */\nfinal class BootstrapSourceManifestIO {\n private BootstrapSourceManifestIO() {\n }\n\n static BootstrapSourceManifest read(Path path) throws Exception {\n if (path == null) throw new IllegalArgumentException(\"manifest path is required\");\n ObjectMapper mapper = JsonMapper.builder().build();\n ManifestDto value = mapper.readValue(path.toFile(), ManifestDto.class);\n if (value.schemaVersion() != 1 || !\"ADSB_LOL\".equals(value.provider())) {\n throw new IllegalArgumentException(\"unsupported historical bootstrap manifest identity\");\n }\n\n DatasetWindow window = window(value);\n PreflightPolicy policy = new PreflightPolicy(value.minimumArchiveBytes(), value.maximumArchiveBytes());\n List days = new ArrayList<>();\n for (DayDto day : value.days()) {\n LocalDate sourceDate = LocalDate.parse(day.sourceDate());\n DatasetWindow.DateRole role = DatasetWindow.DateRole.valueOf(day.role());\n if (role != window.roleFor(sourceDate)) {\n throw new IllegalArgumentException(\"persisted manifest day role does not match window: \" + sourceDate);\n }\n\n List candidates = new ArrayList<>();\n Map byId = new HashMap<>();\n for (ReleaseDto release : day.candidateReleases()) {\n List assets = release.assets().stream()\n .map(asset -> new SourceAsset(asset.assetId(), asset.name(), asset.byteSize(), asset.sha256()))\n .toList();\n SourceRelease candidate = new SourceRelease(\n release.repository(),\n release.releaseId(),\n release.releaseTag(),\n sourceDate,\n release.releaseDraft(),\n release.releasePrerelease(),\n ReleaseVariant.valueOf(release.releaseVariant()),\n assets\n );\n if (byId.put(candidate.releaseId(), candidate) != null) {\n throw new IllegalArgumentException(\"duplicate release ID in persisted manifest day: \" + sourceDate);\n }\n candidates.add(candidate);\n }\n\n SourceRelease selectedRelease = null;\n ResolvedArchiveFamily selectedFamily = null;\n if (day.selectedReleaseId() != null) {\n selectedRelease = byId.get(day.selectedReleaseId());\n if (selectedRelease == null) {\n throw new IllegalArgumentException(\"selected release missing from persisted candidate snapshot: \" + sourceDate);\n }\n if (day.selectedArchiveFamily() == null || day.selectedArchiveBytes() == null) {\n throw new IllegalArgumentException(\"selected archive family metadata is incomplete: \" + sourceDate);\n }\n Map selectedAssetsById = new HashMap<>();\n for (SourceAsset asset : selectedRelease.assets()) selectedAssetsById.put(asset.assetId(), asset);\n List selectedAssets = day.selectedAssetIds().stream().map(assetId -> {\n SourceAsset asset = selectedAssetsById.get(assetId);\n if (asset == null) {\n throw new IllegalArgumentException(\"selected asset is absent from candidate release: \" + assetId);\n }\n return asset;\n }).toList();\n selectedFamily = new ResolvedArchiveFamily(\n ArchiveFamilyKind.valueOf(day.selectedArchiveFamily()), selectedAssets);\n if (selectedFamily.totalBytes() != day.selectedArchiveBytes()) {\n throw new IllegalArgumentException(\"selected archive byte total changed: \" + sourceDate);\n }\n } else if (!day.selectedAssetIds().isEmpty() || day.selectedArchiveFamily() != null || day.selectedArchiveBytes() != null) {\n throw new IllegalArgumentException(\"persisted manifest has partial selected source identity: \" + sourceDate);\n }\n\n days.add(new ResolvedSourceDay(\n sourceDate,\n role,\n SourcePreflightStatus.valueOf(day.sourcePreflightStatus()),\n candidates,\n selectedRelease,\n selectedFamily,\n day.overrideReason()\n ));\n }\n\n return new BootstrapSourceManifest(\n value.schemaVersion(), value.provider(), window, policy, days, value.manifestSha256());\n }\n\n private static DatasetWindow window(ManifestDto value) {\n DatasetWindow.DatasetPurpose purpose = DatasetWindow.DatasetPurpose.valueOf(value.purpose());\n LocalDate from = LocalDate.parse(value.fromDate());\n LocalDate to = LocalDate.parse(value.toDate());\n if (purpose == DatasetWindow.DatasetPurpose.QUALIFICATION_EXPERIMENT) {\n return DatasetWindow.qualification(\n from,\n LocalDate.parse(required(value.learningEnd(), \"learningEnd\")),\n LocalDate.parse(required(value.embargoDate(), \"embargoDate\")),\n LocalDate.parse(required(value.holdoutStart(), \"holdoutStart\")),\n to\n );\n }\n LocalDate asOf = LocalDate.parse(required(value.asOfDate(), \"asOfDate\"));\n return new DatasetWindow(\n DatasetWindow.DatasetPurpose.BOOTSTRAP_SCORING,\n from,\n to,\n asOf,\n value.lookbackDays(),\n null,\n null,\n null\n );\n }\n\n private static String required(String value, String name) {\n if (value == null || value.isBlank()) throw new IllegalArgumentException(\"missing manifest field \" + name);\n return value;\n }\n\n private record ManifestDto(\n int schemaVersion,\n String provider,\n String manifestSha256,\n String purpose,\n String fromDate,\n String toDate,\n String asOfDate,\n Integer lookbackDays,\n String learningEnd,\n String embargoDate,\n String holdoutStart,\n Long minimumArchiveBytes,\n Long maximumArchiveBytes,\n List days\n ) {\n ManifestDto {\n if (days == null) days = List.of();\n }\n }\n\n private record DayDto(\n String sourceDate,\n String role,\n @JsonProperty(\"sourcePreflightStatus\") String sourcePreflightStatus,\n String overrideReason,\n List candidateReleases,\n Long selectedReleaseId,\n String selectedArchiveFamily,\n Long selectedArchiveBytes,\n List selectedAssetIds\n ) {\n DayDto {\n if (candidateReleases == null) candidateReleases = List.of();\n if (selectedAssetIds == null) selectedAssetIds = List.of();\n }\n }\n\n private record ReleaseDto(\n String repository,\n long releaseId,\n String releaseTag,\n String releaseVariant,\n boolean releaseDraft,\n boolean releasePrerelease,\n List assets\n ) {\n ReleaseDto {\n if (assets == null) assets = List.of();\n }\n }\n\n private record AssetDto(long assetId, String name, long byteSize, String sha256) {\n }\n}\n" }, { "path": "backend/historical-bootstrap-tool/src/main/java/hu/gombos/flightspotter/historicalbootstrap/BoundedTraceArchiveReader.java", "encoding": "UTF-8", "sha256": "684f23996382c6a3051e8c2aa11165e8995551cf43edcffe8c1d44863f595231", "content": "package hu.gombos.flightspotter.historicalbootstrap;\n\nimport org.apache.commons.compress.archivers.tar.TarArchiveEntry;\nimport org.apache.commons.compress.archivers.tar.TarArchiveInputStream;\n\nimport java.io.ByteArrayOutputStream;\nimport java.io.IOException;\nimport java.io.InputStream;\nimport java.util.regex.Pattern;\n\n/**\n * Streaming tar scanner that never extracts source files to the filesystem. Only regular gzip\n * JSON members matching readsb's globe-history trace layout are exposed to the trace parser.\n *\n *

readsb writes gzip-compressed payloads using a {@code .json} filename, for example\n * {@code YYYY/MM/DD/traces/3c/trace_full_3c66b0.json}. The compression is a content property;\n * the archived filename does not carry a {@code .gz} suffix.

\n */\nfinal class BoundedTraceArchiveReader {\n private static final Pattern TRACE_BUCKET = Pattern.compile(\"[0-9a-fA-F]{2}\");\n private static final Pattern TRACE_FILE = Pattern.compile(\"trace_full_~?[0-9a-fA-F]{6}\\\\.json\");\n\n private final ArchiveSafetyLimits limits;\n\n BoundedTraceArchiveReader(ArchiveSafetyLimits limits) {\n this.limits = limits;\n }\n\n ReadStats read(InputStream tarInput, TraceEntryConsumer consumer) throws Exception {\n if (tarInput == null || consumer == null) {\n throw new IllegalArgumentException(\"tarInput and consumer are required\");\n }\n long compressedTraceBytes = 0L;\n int entryCount = 0;\n int traceCount = 0;\n try (TarArchiveInputStream tar = new TarArchiveInputStream(tarInput)) {\n TarArchiveEntry entry;\n while ((entry = tar.getNextEntry()) != null) {\n entryCount = Math.addExact(entryCount, 1);\n if (entryCount > limits.maxTarEntries()) {\n throw new IOException(\"tar entry count exceeds configured hard bound\");\n }\n String name = safeName(entry.getName());\n if (entry.isDirectory()) {\n continue;\n }\n if (!entry.isFile()) {\n throw new IOException(\"unsupported special tar member: \" + name);\n }\n if (!isTraceMember(name)) {\n continue;\n }\n long size = entry.getSize();\n if (size < 1L || size > limits.maxCompressedTraceBytes()) {\n throw new IOException(\"trace member compressed size outside hard bound: \" + name);\n }\n byte[] compressed = readEntry(tar, size, limits.maxCompressedTraceBytes());\n compressedTraceBytes = Math.addExact(compressedTraceBytes, compressed.length);\n traceCount = Math.addExact(traceCount, 1);\n try {\n consumer.accept(name, compressed);\n } catch (IOException failure) {\n // Keep runtime/programming failures untouched, but attach deterministic archive context to\n // checked trace-processing failures so a long source-day run can classify the exact member.\n throw new TraceEntryProcessingException(name, entryCount, traceCount, failure);\n }\n }\n }\n return new ReadStats(entryCount, traceCount, compressedTraceBytes);\n }\n\n private static byte[] readEntry(InputStream input, long declaredSize, long maxBytes) throws IOException {\n ByteArrayOutputStream output = new ByteArrayOutputStream((int) Math.min(declaredSize, 1_048_576L));\n byte[] buffer = new byte[16 * 1024];\n long total = 0L;\n int read;\n while ((read = input.read(buffer)) != -1) {\n total = Math.addExact(total, read);\n if (total > maxBytes) {\n throw new IOException(\"trace member exceeded compressed byte bound while streaming\");\n }\n output.write(buffer, 0, read);\n }\n if (total != declaredSize) {\n throw new IOException(\"trace member size differs from tar header: declared=\" + declaredSize + \" actual=\" + total);\n }\n return output.toByteArray();\n }\n\n private static boolean isTraceMember(String name) {\n String[] parts = name.split(\"/\");\n for (int index = 0; index + 2 < parts.length; index++) {\n if (!\"traces\".equals(parts[index])) {\n continue;\n }\n if (index + 3 != parts.length) {\n return false;\n }\n return TRACE_BUCKET.matcher(parts[index + 1]).matches()\n && TRACE_FILE.matcher(parts[index + 2]).matches();\n }\n return false;\n }\n\n private static String safeName(String name) throws IOException {\n if (name == null || name.isBlank()) {\n throw new IOException(\"tar member has empty name\");\n }\n String normalized = name.replace('\\\\', '/');\n if (normalized.startsWith(\"/\")) {\n throw new IOException(\"absolute tar member path is not allowed\");\n }\n for (String part : normalized.split(\"/\")) {\n if (\"..\".equals(part)) {\n throw new IOException(\"parent traversal tar member is not allowed\");\n }\n }\n return normalized;\n }\n\n @FunctionalInterface\n interface TraceEntryConsumer {\n void accept(String archivePath, byte[] compressedTrace) throws Exception;\n }\n\n record ReadStats(int tarEntryCount, int traceEntryCount, long compressedTraceBytes) {\n }\n\n static final class TraceEntryProcessingException extends IOException {\n private final String archivePath;\n private final int tarEntryOrdinal;\n private final int traceEntryOrdinal;\n\n TraceEntryProcessingException(\n String archivePath,\n int tarEntryOrdinal,\n int traceEntryOrdinal,\n IOException cause\n ) {\n super(\"trace entry processing failed: archivePath=\" + archivePath\n + \" tarEntryOrdinal=\" + tarEntryOrdinal\n + \" traceEntryOrdinal=\" + traceEntryOrdinal, cause);\n this.archivePath = archivePath;\n this.tarEntryOrdinal = tarEntryOrdinal;\n this.traceEntryOrdinal = traceEntryOrdinal;\n }\n\n String archivePath() {\n return archivePath;\n }\n\n int tarEntryOrdinal() {\n return tarEntryOrdinal;\n }\n\n int traceEntryOrdinal() {\n return traceEntryOrdinal;\n }\n }\n}\n" }, { "path": "backend/historical-bootstrap-tool/src/main/java/hu/gombos/flightspotter/historicalbootstrap/CanonicalSourceResolver.java", "encoding": "UTF-8", "sha256": "0625d0cd6f80ad4c0575d0e2870a4383b2ae73a5624cc8b8067ff16c7ebb7265", "content": "package hu.gombos.flightspotter.historicalbootstrap;\n\nimport java.time.LocalDate;\nimport java.util.ArrayList;\nimport java.util.Comparator;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Optional;\nimport java.util.Set;\n\nfinal class CanonicalSourceResolver {\n private final ArchiveFamilyResolver familyResolver = new ArchiveFamilyResolver();\n\n ResolvedSourceDay resolve(\n DatasetWindow window,\n LocalDate sourceDate,\n List discovered,\n Optional override,\n PreflightPolicy policy\n ) {\n DatasetWindow.DateRole role = window.roleFor(sourceDate);\n List releases = discovered.stream()\n .filter(release -> release.sourceDate().equals(sourceDate))\n .sorted(Comparator.comparingLong(SourceRelease::releaseId))\n .toList();\n\n if (override.isPresent()) {\n return resolveOverride(sourceDate, role, releases, override.get(), policy);\n }\n if (releases.isEmpty()) {\n return result(sourceDate, role, SourcePreflightStatus.NO_RELEASE, releases, null, null, null);\n }\n\n List evaluations = releases.stream()\n .map(release -> evaluateRelease(release, policy, false))\n .toList();\n\n ResolvedSourceDay selected = selectLargestHealthy(\n sourceDate, role, releases, evaluations, ReleaseVariant.PROD);\n if (selected != null) return selected;\n selected = selectLargestHealthy(\n sourceDate, role, releases, evaluations, ReleaseVariant.STAGING);\n if (selected != null) return selected;\n\n List coherentProd = coherent(evaluations, ReleaseVariant.PROD);\n if (coherentProd.size() > 1) {\n return result(sourceDate, role, SourcePreflightStatus.MULTIPLE_RELEASE_CANDIDATES,\n releases, null, null, null);\n }\n List coherentStaging = coherent(evaluations, ReleaseVariant.STAGING);\n if (coherentStaging.size() > 1) {\n return result(sourceDate, role, SourcePreflightStatus.MULTIPLE_RELEASE_CANDIDATES,\n releases, null, null, null);\n }\n if (coherentProd.size() == 1) return coherentProd.getFirst().toResolved(sourceDate, role, releases, null);\n if (coherentStaging.size() == 1) return coherentStaging.getFirst().toResolved(sourceDate, role, releases, null);\n\n if (evaluations.stream().allMatch(value -> value.release().variant() == ReleaseVariant.MLAT_ONLY)) {\n return result(sourceDate, role, SourcePreflightStatus.MLAT_ONLY_RELEASE, releases, null, null, null);\n }\n\n EvaluatedRelease failure = evaluations.stream()\n .sorted(Comparator.comparingInt(this::failurePriority)\n .thenComparingLong(value -> value.release().releaseId()))\n .findFirst()\n .orElseThrow();\n return failure.toResolved(sourceDate, role, releases, null);\n }\n\n private ResolvedSourceDay resolveOverride(\n LocalDate sourceDate,\n DatasetWindow.DateRole role,\n List releases,\n ManualSourceOverride override,\n PreflightPolicy policy\n ) {\n if (!override.sourceDate().equals(sourceDate)) {\n throw new IllegalArgumentException(\"override sourceDate does not match resolved date\");\n }\n if (override.exclude()) {\n return result(sourceDate, role, SourcePreflightStatus.MANUAL_EXCLUSION,\n releases, null, null, override.reason());\n }\n\n SourceRelease release = releases.stream()\n .filter(value -> value.releaseId() == override.releaseId())\n .findFirst()\n .orElse(null);\n if (release == null) {\n return result(sourceDate, role, SourcePreflightStatus.OVERRIDE_INVALID,\n releases, null, null, override.reason());\n }\n if ((release.draft() || release.prerelease()) && !override.allowDraftOrPrerelease()) {\n return result(sourceDate, role, SourcePreflightStatus.UNSUPPORTED_RELEASE_STATE,\n releases, null, null, override.reason());\n }\n if (release.variant() == ReleaseVariant.MLAT_ONLY || release.variant() == ReleaseVariant.OTHER) {\n return result(sourceDate, role,\n release.variant() == ReleaseVariant.MLAT_ONLY\n ? SourcePreflightStatus.MLAT_ONLY_RELEASE\n : SourcePreflightStatus.OVERRIDE_INVALID,\n releases, null, null, override.reason());\n }\n\n Set wanted = new HashSet<>(override.assetIds());\n List selectedAssets = release.assets().stream()\n .filter(asset -> wanted.contains(asset.assetId()))\n .toList();\n if (selectedAssets.size() != wanted.size()) {\n return result(sourceDate, role, SourcePreflightStatus.OVERRIDE_INVALID,\n releases, null, null, override.reason());\n }\n SourceRelease selectedRelease = new SourceRelease(\n release.repository(), release.releaseId(), release.tag(), release.sourceDate(),\n release.draft(), release.prerelease(), release.variant(), selectedAssets\n );\n EvaluatedRelease evaluated = evaluateRelease(selectedRelease, policy, override.allowDraftOrPrerelease());\n if (evaluated.family() != null && evaluated.family().assets().size() != selectedAssets.size()) {\n return result(sourceDate, role, SourcePreflightStatus.OVERRIDE_INVALID,\n releases, null, null, override.reason());\n }\n return evaluated.toResolved(sourceDate, role, releases, override.reason());\n }\n\n /**\n * ADSB.lol may publish multiple normal releases for one source day (for example prod-0/prod-1).\n * Resolve each release to one coherent archive family first, then choose the unique largest\n * healthy canonical family inside the preferred variant. This deliberately differs from\n * summing every .tar* asset in a release: competing unsplit/split families were already\n * rejected by {@link ArchiveFamilyResolver} before their byte counts can participate here.\n */\n private ResolvedSourceDay selectLargestHealthy(\n LocalDate sourceDate,\n DatasetWindow.DateRole role,\n List releases,\n List evaluations,\n ReleaseVariant variant\n ) {\n List healthy = evaluations.stream()\n .filter(value -> value.release().variant() == variant)\n .filter(value -> value.status() == SourcePreflightStatus.PREFLIGHT_OK)\n .toList();\n if (healthy.isEmpty()) return null;\n\n long largestBytes = healthy.stream()\n .mapToLong(value -> value.family().totalBytes())\n .max()\n .orElseThrow();\n List largest = healthy.stream()\n .filter(value -> value.family().totalBytes() == largestBytes)\n .toList();\n if (largest.size() != 1) {\n return result(sourceDate, role, SourcePreflightStatus.MULTIPLE_RELEASE_CANDIDATES,\n releases, null, null, null);\n }\n return largest.getFirst().toResolved(sourceDate, role, releases, null);\n }\n\n private List coherent(List values, ReleaseVariant variant) {\n List result = new ArrayList<>();\n for (EvaluatedRelease value : values) {\n if (value.release().variant() == variant && value.family() != null) result.add(value);\n }\n return result;\n }\n\n private EvaluatedRelease evaluateRelease(SourceRelease release, PreflightPolicy policy, boolean allowReleaseState) {\n if ((release.draft() || release.prerelease()) && !allowReleaseState) {\n return new EvaluatedRelease(release, SourcePreflightStatus.UNSUPPORTED_RELEASE_STATE, null);\n }\n if (release.variant() == ReleaseVariant.MLAT_ONLY) {\n return new EvaluatedRelease(release, SourcePreflightStatus.MLAT_ONLY_RELEASE, null);\n }\n if (release.variant() == ReleaseVariant.OTHER) {\n return new EvaluatedRelease(release, SourcePreflightStatus.UNSUPPORTED_RELEASE_STATE, null);\n }\n\n ArchiveFamilyResolver.FamilyResolution family = familyResolver.resolve(release.assets());\n if (!family.ok()) return new EvaluatedRelease(release, family.status(), null);\n if (family.family().assets().stream().anyMatch(asset -> asset.sha256() == null)) {\n return new EvaluatedRelease(release, SourcePreflightStatus.UPSTREAM_DIGEST_MISSING, family.family());\n }\n if (!policy.volumeAllowed(family.family().totalBytes())) {\n return new EvaluatedRelease(release, SourcePreflightStatus.SOURCE_VOLUME_ANOMALY, family.family());\n }\n return new EvaluatedRelease(release, SourcePreflightStatus.PREFLIGHT_OK, family.family());\n }\n\n private int failurePriority(EvaluatedRelease value) {\n return switch (value.status()) {\n case ASSET_FAMILY_AMBIGUOUS -> 0;\n case SPLIT_SEQUENCE_INCOMPLETE -> 1;\n case UPSTREAM_DIGEST_MISSING -> 2;\n case SOURCE_VOLUME_ANOMALY -> 3;\n case MLAT_ONLY_RELEASE -> 4;\n case UNSUPPORTED_RELEASE_STATE -> 5;\n case NO_ARCHIVE_ASSETS -> 6;\n default -> 7;\n };\n }\n\n private ResolvedSourceDay result(\n LocalDate sourceDate,\n DatasetWindow.DateRole role,\n SourcePreflightStatus status,\n List candidates,\n SourceRelease selectedRelease,\n ResolvedArchiveFamily selectedFamily,\n String overrideReason\n ) {\n return new ResolvedSourceDay(sourceDate, role, status, candidates, selectedRelease, selectedFamily, overrideReason);\n }\n\n private record EvaluatedRelease(\n SourceRelease release,\n SourcePreflightStatus status,\n ResolvedArchiveFamily family\n ) {\n ResolvedSourceDay toResolved(\n LocalDate sourceDate,\n DatasetWindow.DateRole role,\n List candidates,\n String overrideReason\n ) {\n return new ResolvedSourceDay(sourceDate, role, status, candidates,\n family == null ? null : release,\n family,\n overrideReason);\n }\n }\n}\n" }, { "path": "backend/historical-bootstrap-tool/src/main/java/hu/gombos/flightspotter/historicalbootstrap/DatasetWindow.java", "encoding": "UTF-8", "sha256": "eb66fe07c04ff3d60053898775ec41ebcce006273c1ba0ebd4e5376634119261", "content": "package hu.gombos.flightspotter.historicalbootstrap;\n\nimport java.time.LocalDate;\nimport java.time.temporal.ChronoUnit;\nimport java.util.Objects;\n\npublic record DatasetWindow(\n DatasetPurpose purpose,\n LocalDate fromDate,\n LocalDate toDate,\n LocalDate asOfDate,\n Integer lookbackDays,\n LocalDate learningEnd,\n LocalDate embargoDate,\n LocalDate holdoutStart\n) {\n public DatasetWindow {\n Objects.requireNonNull(purpose, \"purpose\");\n Objects.requireNonNull(fromDate, \"fromDate\");\n Objects.requireNonNull(toDate, \"toDate\");\n if (toDate.isBefore(fromDate)) {\n throw new IllegalArgumentException(\"toDate must be on or after fromDate\");\n }\n if (purpose == DatasetPurpose.BOOTSTRAP_SCORING) {\n if (learningEnd != null || embargoDate != null || holdoutStart != null) {\n throw new IllegalArgumentException(\"scoring windows cannot contain qualification roles\");\n }\n if (asOfDate == null || !asOfDate.equals(toDate)) {\n throw new IllegalArgumentException(\"scoring asOfDate must equal toDate\");\n }\n if (lookbackDays != null && lookbackDays != calendarDayCount(fromDate, toDate)) {\n throw new IllegalArgumentException(\"lookbackDays must equal inclusive calendar-day count\");\n }\n } else {\n Objects.requireNonNull(learningEnd, \"learningEnd\");\n Objects.requireNonNull(embargoDate, \"embargoDate\");\n Objects.requireNonNull(holdoutStart, \"holdoutStart\");\n if (learningEnd.isBefore(fromDate)\n || !embargoDate.equals(learningEnd.plusDays(1))\n || !holdoutStart.equals(embargoDate.plusDays(1))\n || toDate.isBefore(holdoutStart)) {\n throw new IllegalArgumentException(\"qualification window roles must be contiguous and ordered\");\n }\n if (asOfDate != null || lookbackDays != null) {\n throw new IllegalArgumentException(\"qualification windows do not use rolling as-of metadata\");\n }\n }\n }\n\n public static DatasetWindow scoringLookback(LocalDate asOfDate, int lookbackDays) {\n Objects.requireNonNull(asOfDate, \"asOfDate\");\n if (lookbackDays < 1 || lookbackDays > 730) {\n throw new IllegalArgumentException(\"lookbackDays must be between 1 and 730\");\n }\n return new DatasetWindow(\n DatasetPurpose.BOOTSTRAP_SCORING,\n asOfDate.minusDays(lookbackDays - 1L),\n asOfDate,\n asOfDate,\n lookbackDays,\n null,\n null,\n null\n );\n }\n\n public static DatasetWindow fixedScoring(LocalDate fromDate, LocalDate toDate) {\n return new DatasetWindow(\n DatasetPurpose.BOOTSTRAP_SCORING,\n fromDate,\n toDate,\n toDate,\n calendarDayCount(fromDate, toDate),\n null,\n null,\n null\n );\n }\n\n public static DatasetWindow qualification(\n LocalDate fromDate,\n LocalDate learningEnd,\n LocalDate embargoDate,\n LocalDate holdoutStart,\n LocalDate toDate\n ) {\n return new DatasetWindow(\n DatasetPurpose.QUALIFICATION_EXPERIMENT,\n fromDate,\n toDate,\n null,\n null,\n learningEnd,\n embargoDate,\n holdoutStart\n );\n }\n\n public DateRole roleFor(LocalDate sourceDate) {\n Objects.requireNonNull(sourceDate, \"sourceDate\");\n if (sourceDate.isBefore(fromDate) || sourceDate.isAfter(toDate)) {\n throw new IllegalArgumentException(\"sourceDate is outside the dataset window\");\n }\n if (purpose == DatasetPurpose.BOOTSTRAP_SCORING) {\n return DateRole.SCORING;\n }\n if (!sourceDate.isAfter(learningEnd)) {\n return DateRole.LEARNING;\n }\n if (sourceDate.equals(embargoDate)) {\n return DateRole.EMBARGO;\n }\n return DateRole.HOLDOUT;\n }\n\n public int calendarDayCount() {\n return calendarDayCount(fromDate, toDate);\n }\n\n private static int calendarDayCount(LocalDate fromDate, LocalDate toDate) {\n Objects.requireNonNull(fromDate, \"fromDate\");\n Objects.requireNonNull(toDate, \"toDate\");\n long count = ChronoUnit.DAYS.between(fromDate, toDate) + 1L;\n if (count < 1L || count > Integer.MAX_VALUE) {\n throw new IllegalArgumentException(\"invalid calendar-day count\");\n }\n return Math.toIntExact(count);\n }\n\n public enum DatasetPurpose {\n QUALIFICATION_EXPERIMENT,\n BOOTSTRAP_SCORING\n }\n\n public enum DateRole {\n LEARNING,\n EMBARGO,\n HOLDOUT,\n SCORING\n }\n}\n" }, { "path": "backend/historical-bootstrap-tool/src/main/java/hu/gombos/flightspotter/historicalbootstrap/DigestCachingGateBAssetSource.java", "encoding": "UTF-8", "sha256": "bdcd4ada93e0ea1f798180a5e9702f3205855463aa819479f906fdd2503cda24", "content": "package hu.gombos.flightspotter.historicalbootstrap;\nimport java.io.*;import java.nio.file.*;import java.security.*;import java.util.*;import java.util.HexFormat;\n/** Bounded content-addressed cache for immutable historical assets. */\nfinal class DigestCachingGateBAssetSource implements GateBAssetSource {\n private static final int BUFFER=1024*1024; private final GateBAssetSource delegate;private final Path root;private final long maximumBytes;\n DigestCachingGateBAssetSource(GateBAssetSource delegate,Path root,long maximumBytes)throws IOException{this.delegate=Objects.requireNonNull(delegate);this.root=Objects.requireNonNull(root).toAbsolutePath().normalize();if(maximumBytes<1)throw new IllegalArgumentException(\"historical asset cache maximumBytes must be positive\");this.maximumBytes=maximumBytes;Files.createDirectories(this.root);}\n @Override public synchronized InputStream open(SourceRelease release,SourceAsset asset)throws IOException,InterruptedException{\n if(asset==null||asset.sha256()==null)throw new IOException(\"historical asset cache requires immutable upstream SHA-256\");if(asset.byteSize()>maximumBytes)throw new IOException(\"single historical asset exceeds configured cache capacity\");\n Path cached=root.resolve(asset.sha256()+\".asset\");if(Files.isRegularFile(cached)){if(valid(cached,asset)){Files.setLastModifiedTime(cached,java.nio.file.attribute.FileTime.fromMillis(System.currentTimeMillis()));return Files.newInputStream(cached);}Files.deleteIfExists(cached);}\n evict(asset.byteSize(),null);Path part=root.resolve(asset.sha256()+\".part\");Files.deleteIfExists(part);MessageDigest d=sha();long copied=0;\n try(InputStream in=delegate.open(release,asset);OutputStream out=Files.newOutputStream(part,StandardOpenOption.CREATE_NEW)){byte[] buf=new byte[BUFFER];int n;while((n=in.read(buf))!=-1){copied=Math.addExact(copied,n);if(copied>asset.byteSize())throw new ImmutableHistoricalSourceException(\"IMMUTABLE_ASSET_SIZE_MISMATCH\",\"historical asset exceeded immutable byte size\");out.write(buf,0,n);d.update(buf,0,n);}}\n catch(IOException|InterruptedException e){Files.deleteIfExists(part);throw e;}\n if(copied!=asset.byteSize()||!HexFormat.of().formatHex(d.digest()).equals(asset.sha256())){Files.deleteIfExists(part);throw new ImmutableHistoricalSourceException(\"IMMUTABLE_ASSET_VERIFICATION_FAILED\",\"historical asset cache fill failed immutable size/SHA-256 verification\");}\n try{Files.move(part,cached,StandardCopyOption.ATOMIC_MOVE);}catch(AtomicMoveNotSupportedException e){Files.move(part,cached,StandardCopyOption.REPLACE_EXISTING);}evict(0,cached);return Files.newInputStream(cached);\n }\n private boolean valid(Path p,SourceAsset a)throws IOException{if(Files.size(p)!=a.byteSize())return false;MessageDigest d=sha();try(InputStream in=Files.newInputStream(p)){byte[] b=new byte[BUFFER];int n;while((n=in.read(b))!=-1)d.update(b,0,n);}return HexFormat.of().formatHex(d.digest()).equals(a.sha256());}\n private void evict(long incoming,Path protect)throws IOException{long total=bytes();if(total+incoming<=maximumBytes)return;List candidates;try(var st=Files.list(root)){candidates=st.filter(Files::isRegularFile).filter(p->!p.getFileName().toString().endsWith(\".part\")).filter(p->protect==null||!p.equals(protect)).sorted(Comparator.comparingLong(DigestCachingGateBAssetSource::mtime).thenComparing(p->p.getFileName().toString())).toList();}for(Path p:candidates){total-=Files.size(p);Files.deleteIfExists(p);if(total+incoming<=maximumBytes)return;}if(total+incoming>maximumBytes)throw new IOException(\"historical asset cache cannot satisfy bounded capacity\");}\n private long bytes()throws IOException{long t=0;try(var st=Files.list(root)){for(Path p:st.filter(Files::isRegularFile).toList())t=Math.addExact(t,Files.size(p));}return t;} private static long mtime(Path p){try{return Files.getLastModifiedTime(p).toMillis();}catch(IOException e){return Long.MIN_VALUE;}}\n private static MessageDigest sha(){try{return MessageDigest.getInstance(\"SHA-256\");}catch(NoSuchAlgorithmException e){throw new IllegalStateException(e);}}\n}" }, { "path": "backend/historical-bootstrap-tool/src/main/java/hu/gombos/flightspotter/historicalbootstrap/GateBAssetSource.java", "encoding": "UTF-8", "sha256": "8c261757bd92808f365646eee84d0e39c88120a512f8b239187ca2784efa6c56", "content": "package hu.gombos.flightspotter.historicalbootstrap;\n\nimport java.io.FilterInputStream;\nimport java.io.IOException;\nimport java.io.InputStream;\nimport java.net.URI;\nimport java.net.http.HttpClient;\nimport java.net.http.HttpRequest;\nimport java.net.http.HttpResponse;\nimport java.time.Duration;\n\ninterface GateBAssetSource {\n InputStream open(SourceRelease release, SourceAsset asset) throws IOException, InterruptedException;\n}\n\n/**\n * Public ADSB.lol release-asset transport for later owner-approved Gate B execution. Construction\n * performs no network I/O. Authentication is deliberately unsupported: the historical source is\n * public and no GitHub credential may be forwarded across the release-asset CDN redirect chain.\n *\n * Retry count is owned by the bounded Gate B execution budget. This transport therefore performs\n * exactly one HTTP request per open() call and does not implement any hidden retry loop of its own.\n */\nfinal class GitHubReleaseAssetSource implements GateBAssetSource {\n private final HttpClient client;\n private final URI apiBase;\n private final Duration requestTimeout;\n\n GitHubReleaseAssetSource(Duration requestTimeout) {\n this(HttpClient.newBuilder()\n .connectTimeout(Duration.ofSeconds(15))\n .followRedirects(HttpClient.Redirect.ALWAYS)\n .build(),\n URI.create(\"https://api.github.com\"),\n requestTimeout);\n }\n\n GitHubReleaseAssetSource(HttpClient client, URI apiBase, Duration requestTimeout) {\n if (client == null || apiBase == null || requestTimeout == null\n || requestTimeout.isZero() || requestTimeout.isNegative()\n || requestTimeout.compareTo(Duration.ofHours(4)) > 0) {\n throw new IllegalArgumentException(\"invalid GitHub release asset source configuration\");\n }\n if (!\"https\".equalsIgnoreCase(apiBase.getScheme()) && !isLoopbackHttp(apiBase)) {\n throw new IllegalArgumentException(\"GitHub release asset source requires HTTPS except local fixture servers\");\n }\n this.client = client;\n this.apiBase = apiBase;\n this.requestTimeout = requestTimeout;\n }\n\n @Override\n public InputStream open(SourceRelease release, SourceAsset asset) throws IOException, InterruptedException {\n if (release == null || asset == null) throw new IllegalArgumentException(\"release and asset are required\");\n if (!release.repository().matches(\"adsblol/globe_history_\\\\d{4}\")) {\n throw new IllegalArgumentException(\"Gate B real transport is restricted to ADSB.lol historical repositories\");\n }\n URI uri = URI.create(apiBase.toString() + \"/repos/\" + release.repository()\n + \"/releases/assets/\" + asset.assetId());\n HttpRequest request = HttpRequest.newBuilder(uri)\n .timeout(requestTimeout)\n .header(\"Accept\", \"application/octet-stream\")\n .header(\"X-GitHub-Api-Version\", \"2022-11-28\")\n .header(\"User-Agent\", \"FlightTrove-P7-Gate-B\")\n .GET()\n .build();\n\n final HttpResponse response;\n try {\n response = client.send(request, HttpResponse.BodyHandlers.ofInputStream());\n } catch (IOException transportFailure) {\n throw new HistoricalSourceTransportException(\n \"GitHub release asset transport failed: assetId=\" + asset.assetId(), transportFailure);\n }\n if (response.statusCode() != 200) {\n try (InputStream body = response.body()) {\n // close response body before surfacing the bounded source failure\n }\n throw new HistoricalSourceTransportException(\n \"GitHub release asset download failed: status=\" + response.statusCode()\n + \" assetId=\" + asset.assetId());\n }\n return classifyTransportStream(response.body(), asset.assetId());\n }\n\n static InputStream classifyTransportStream(InputStream body, long assetId) {\n if (body == null) throw new IllegalArgumentException(\"response body is required\");\n return new FilterInputStream(body) {\n @Override\n public int read() throws IOException {\n try {\n return super.read();\n } catch (HistoricalSourceTransportException classified) {\n throw classified;\n } catch (IOException transportFailure) {\n throw classified(transportFailure);\n }\n }\n\n @Override\n public int read(byte[] buffer, int offset, int length) throws IOException {\n try {\n return super.read(buffer, offset, length);\n } catch (HistoricalSourceTransportException classified) {\n throw classified;\n } catch (IOException transportFailure) {\n throw classified(transportFailure);\n }\n }\n\n @Override\n public void close() throws IOException {\n try {\n super.close();\n } catch (HistoricalSourceTransportException classified) {\n throw classified;\n } catch (IOException transportFailure) {\n throw classified(transportFailure);\n }\n }\n\n private HistoricalSourceTransportException classified(IOException failure) {\n return new HistoricalSourceTransportException(\n \"GitHub release asset response stream failed: assetId=\" + assetId, failure);\n }\n };\n }\n\n private static boolean isLoopbackHttp(URI uri) {\n String host = uri.getHost();\n return \"http\".equalsIgnoreCase(uri.getScheme())\n && (\"localhost\".equalsIgnoreCase(host) || \"127.0.0.1\".equals(host) || \"::1\".equals(host));\n }\n}\n" }, { "path": "backend/historical-bootstrap-tool/src/main/java/hu/gombos/flightspotter/historicalbootstrap/GateBExecution.java", "encoding": "UTF-8", "sha256": "3ff692ac3bb1fb1aa94c1d1f3eb6cbd7838c63b40b8000c34b0aee2570467b53", "content": "package hu.gombos.flightspotter.historicalbootstrap;\n\nimport java.nio.ByteBuffer;\nimport java.nio.charset.StandardCharsets;\nimport java.nio.file.Path;\nimport java.security.MessageDigest;\nimport java.security.NoSuchAlgorithmException;\nimport java.time.LocalDate;\nimport java.util.HexFormat;\nimport java.util.LinkedHashMap;\nimport java.util.Map;\nimport java.util.Objects;\nimport java.util.TreeMap;\nimport java.util.UUID;\n\nrecord GateBContentPolicy(\n int maximumTraceParseFailures,\n long minimumAcceptedPoints\n) {\n GateBContentPolicy {\n if (maximumTraceParseFailures < 0 || minimumAcceptedPoints < 1L) {\n throw new IllegalArgumentException(\"invalid Gate B content policy\");\n }\n }\n}\n\nrecord GateBExecutionBudget(\n long maximumTransferBytes,\n long scratchReserveBytes,\n int maximumAssetAttempts,\n GateBContentPolicy contentPolicy\n) {\n GateBExecutionBudget {\n if (maximumTransferBytes < 1L\n || scratchReserveBytes < 0L\n || maximumAssetAttempts < 1 || maximumAssetAttempts > 3\n || contentPolicy == null) {\n throw new IllegalArgumentException(\"invalid Gate B execution budget\");\n }\n }\n}\n\nrecord GateBExecutionPlan(\n UUID datasetId,\n String semanticFingerprint,\n String manifestSha256,\n String dayManifestSha256,\n String coverageVersion,\n String coverageHash,\n LocalDate sourceDate,\n DatasetWindow.DateRole role,\n SourceRelease selectedRelease,\n ResolvedArchiveFamily selectedFamily,\n Path scratchRoot,\n Path stateDatabase,\n GateBExecutionBudget budget\n) {\n GateBExecutionPlan {\n Objects.requireNonNull(datasetId, \"datasetId\");\n if (semanticFingerprint == null || !semanticFingerprint.matches(\"[0-9a-f]{64}\")) {\n throw new IllegalArgumentException(\"semanticFingerprint must be lowercase SHA-256\");\n }\n if (manifestSha256 == null || !manifestSha256.matches(\"[0-9a-f]{64}\")\n || dayManifestSha256 == null || !dayManifestSha256.matches(\"[0-9a-f]{64}\")\n || coverageHash == null || !coverageHash.matches(\"[0-9a-f]{64}\")) {\n throw new IllegalArgumentException(\"manifest/coverage hashes must be lowercase SHA-256\");\n }\n if (coverageVersion == null || coverageVersion.isBlank()) {\n throw new IllegalArgumentException(\"coverageVersion is required\");\n }\n Objects.requireNonNull(sourceDate, \"sourceDate\");\n Objects.requireNonNull(role, \"role\");\n Objects.requireNonNull(selectedRelease, \"selectedRelease\");\n Objects.requireNonNull(selectedFamily, \"selectedFamily\");\n Objects.requireNonNull(scratchRoot, \"scratchRoot\");\n Objects.requireNonNull(stateDatabase, \"stateDatabase\");\n Objects.requireNonNull(budget, \"budget\");\n scratchRoot = scratchRoot.toAbsolutePath().normalize();\n stateDatabase = stateDatabase.toAbsolutePath().normalize();\n if (stateDatabase.startsWith(scratchRoot)) {\n throw new IllegalArgumentException(\"compact state database must live outside raw scratch\");\n }\n if (!selectedRelease.sourceDate().equals(sourceDate)) {\n throw new IllegalArgumentException(\"selected release date does not match execution day\");\n }\n if (selectedFamily.totalBytes() > budget.maximumTransferBytes()) {\n throw new IllegalArgumentException(\"selected source bytes exceed explicit transfer budget\");\n }\n }\n\n static GateBExecutionPlan create(\n BootstrapSourceManifest manifest,\n LocalDate sourceDate,\n UUID datasetId,\n BootstrapCoverageSpec coverage,\n Path scratchRoot,\n Path stateDatabase,\n GateBExecutionBudget budget\n ) {\n Objects.requireNonNull(manifest, \"manifest\");\n Objects.requireNonNull(coverage, \"coverage\");\n ResolvedSourceDay day = manifest.days().stream()\n .filter(value -> value.sourceDate().equals(sourceDate))\n .findFirst()\n .orElseThrow(() -> new IllegalArgumentException(\"sourceDate is outside manifest\"));\n if (day.preflightStatus() != SourcePreflightStatus.PREFLIGHT_OK\n || day.selectedRelease() == null || day.selectedFamily() == null) {\n throw new IllegalArgumentException(\"source-day execution requires one PREFLIGHT_OK selected source day\");\n }\n String fingerprint = GateBSemanticFingerprint.hash(manifest, coverage, budget.contentPolicy());\n return new GateBExecutionPlan(\n datasetId,\n fingerprint,\n manifest.manifestSha256(),\n manifest.dayManifestSha256(sourceDate),\n coverage.version(),\n coverage.hash(),\n sourceDate,\n day.role(),\n day.selectedRelease(),\n day.selectedFamily(),\n scratchRoot,\n stateDatabase,\n budget\n );\n }\n\n LocalDayCommitStore.DayIdentity dayIdentity() {\n return new LocalDayCommitStore.DayIdentity(datasetId, sourceDate, dayManifestSha256, semanticFingerprint);\n }\n\n long selectedSourceBytes() {\n return selectedFamily.totalBytes();\n }\n}\n\nrecord GateBQualificationRecord(\n UUID datasetId,\n LocalDate sourceDate,\n String manifestSha256,\n String dayManifestSha256,\n String semanticFingerprint,\n String coverageVersion,\n String coverageHash,\n String parserVersion,\n String eligibilityVersion,\n String segmentationVersion,\n String dayDeltaVersion,\n long selectedReleaseId,\n String selectedReleaseTag,\n String selectedArchiveFamily,\n long maximumTransferBytes,\n long scratchReserveBytes,\n int maximumAssetAttempts,\n int maximumTraceParseFailures,\n long minimumAcceptedPoints,\n long selectedSourceBytes,\n long transferBytes,\n int selectedAssetCount,\n long scratchUsableBeforeBytes,\n long scratchRequiredBytes,\n long peakRawScratchBytes,\n long rawScratchBytesAfterCleanup,\n long compactStateBytes,\n int tarEntryCount,\n int traceEntryCount,\n int parsedTraceCount,\n int traceParseFailureCount,\n long malformedPointCount,\n long acceptedPointCount,\n long rejectedPointCount,\n Map rejectionCounts,\n long finalizedSegmentCount,\n long outOfOrderRejectedCount,\n int outgoingCarryCount,\n int patternCount,\n int hourBinCount,\n int holdoutOutcomeCount,\n int excludedSegmentCount,\n String dayDeltaChecksum,\n long wallMillis,\n long currentThreadCpuMillis,\n long maxObservedHeapBytes,\n boolean rawScratchCleanupVerified\n) {\n GateBQualificationRecord {\n Objects.requireNonNull(datasetId, \"datasetId\");\n Objects.requireNonNull(sourceDate, \"sourceDate\");\n if (manifestSha256 == null || !manifestSha256.matches(\"[0-9a-f]{64}\")\n || dayManifestSha256 == null || !dayManifestSha256.matches(\"[0-9a-f]{64}\")\n || semanticFingerprint == null || !semanticFingerprint.matches(\"[0-9a-f]{64}\")\n || coverageHash == null || !coverageHash.matches(\"[0-9a-f]{64}\")\n || dayDeltaChecksum == null || !dayDeltaChecksum.matches(\"[0-9a-f]{64}\")) {\n throw new IllegalArgumentException(\"Gate B record contains invalid SHA-256 identity\");\n }\n if (coverageVersion == null || coverageVersion.isBlank()\n || parserVersion == null || parserVersion.isBlank()\n || eligibilityVersion == null || eligibilityVersion.isBlank()\n || segmentationVersion == null || segmentationVersion.isBlank()\n || dayDeltaVersion == null || dayDeltaVersion.isBlank()\n || selectedReleaseTag == null || selectedReleaseTag.isBlank()\n || selectedArchiveFamily == null || selectedArchiveFamily.isBlank()) {\n throw new IllegalArgumentException(\"Gate B record contains missing version/source identity\");\n }\n rejectionCounts = Map.copyOf(new TreeMap<>(rejectionCounts == null ? Map.of() : rejectionCounts));\n if (selectedReleaseId < 1L || maximumTransferBytes < 1L || scratchReserveBytes < 0L\n || maximumAssetAttempts < 1 || maximumTraceParseFailures < 0 || minimumAcceptedPoints < 1L\n || selectedSourceBytes < 1L || transferBytes < selectedSourceBytes || selectedAssetCount < 1\n || scratchUsableBeforeBytes < 0L || scratchRequiredBytes < selectedSourceBytes\n || peakRawScratchBytes < selectedSourceBytes || rawScratchBytesAfterCleanup < 0L\n || compactStateBytes < 0L || tarEntryCount < 0 || traceEntryCount < 0 || parsedTraceCount < 0\n || traceParseFailureCount < 0 || malformedPointCount < 0L || acceptedPointCount < 0L\n || rejectedPointCount < 0L || finalizedSegmentCount < 0L || outOfOrderRejectedCount < 0L\n || outgoingCarryCount < 0 || patternCount < 0 || hourBinCount < 0 || holdoutOutcomeCount < 0\n || excludedSegmentCount < 0 || wallMillis < 0L || currentThreadCpuMillis < -1L\n || maxObservedHeapBytes < 0L) {\n throw new IllegalArgumentException(\"Gate B record contains invalid measurement\");\n }\n }\n\n Map toJsonModel() {\n Map value = new LinkedHashMap<>();\n value.put(\"schemaVersion\", 1);\n value.put(\"gate\", \"P7_GATE_B_ONE_DAY\");\n value.put(\"datasetId\", datasetId.toString());\n value.put(\"sourceDate\", sourceDate.toString());\n value.put(\"manifestSha256\", manifestSha256);\n value.put(\"dayManifestSha256\", dayManifestSha256);\n value.put(\"semanticFingerprint\", semanticFingerprint);\n value.put(\"coverageVersion\", coverageVersion);\n value.put(\"coverageHash\", coverageHash);\n value.put(\"parserVersion\", parserVersion);\n value.put(\"archiveSafetyProfile\", ArchiveSafetyLimits.defaults().semanticFingerprintComponent());\n value.put(\"eligibilityVersion\", eligibilityVersion);\n value.put(\"segmentationVersion\", segmentationVersion);\n value.put(\"dayDeltaVersion\", dayDeltaVersion);\n value.put(\"corridorModelVersion\", HistoricalCorridorModel.VERSION);\n value.put(\"selectedReleaseId\", selectedReleaseId);\n value.put(\"selectedReleaseTag\", selectedReleaseTag);\n value.put(\"selectedArchiveFamily\", selectedArchiveFamily);\n value.put(\"maximumTransferBytes\", maximumTransferBytes);\n value.put(\"scratchReserveBytes\", scratchReserveBytes);\n value.put(\"maximumAssetAttempts\", maximumAssetAttempts);\n value.put(\"maximumTraceParseFailures\", maximumTraceParseFailures);\n value.put(\"minimumAcceptedPoints\", minimumAcceptedPoints);\n value.put(\"selectedSourceBytes\", selectedSourceBytes);\n value.put(\"transferBytes\", transferBytes);\n value.put(\"selectedAssetCount\", selectedAssetCount);\n value.put(\"scratchUsableBeforeBytes\", scratchUsableBeforeBytes);\n value.put(\"scratchRequiredBytes\", scratchRequiredBytes);\n value.put(\"peakRawScratchBytes\", peakRawScratchBytes);\n value.put(\"rawScratchBytesAfterCleanup\", rawScratchBytesAfterCleanup);\n value.put(\"compactStateBytes\", compactStateBytes);\n value.put(\"tarEntryCount\", tarEntryCount);\n value.put(\"traceEntryCount\", traceEntryCount);\n value.put(\"parsedTraceCount\", parsedTraceCount);\n value.put(\"traceParseFailureCount\", traceParseFailureCount);\n value.put(\"malformedPointCount\", malformedPointCount);\n value.put(\"acceptedPointCount\", acceptedPointCount);\n value.put(\"rejectedPointCount\", rejectedPointCount);\n value.put(\"rejectionCounts\", rejectionCounts);\n value.put(\"finalizedSegmentCount\", finalizedSegmentCount);\n value.put(\"outOfOrderRejectedCount\", outOfOrderRejectedCount);\n value.put(\"outgoingCarryCount\", outgoingCarryCount);\n value.put(\"patternCount\", patternCount);\n value.put(\"hourBinCount\", hourBinCount);\n value.put(\"holdoutOutcomeCount\", holdoutOutcomeCount);\n value.put(\"excludedSegmentCount\", excludedSegmentCount);\n value.put(\"dayDeltaChecksum\", dayDeltaChecksum);\n value.put(\"wallMillis\", wallMillis);\n value.put(\"currentThreadCpuMillis\", currentThreadCpuMillis);\n value.put(\"maxObservedHeapBytes\", maxObservedHeapBytes);\n value.put(\"rawScratchCleanupVerified\", rawScratchCleanupVerified);\n return value;\n }\n}\n\nfinal class GateBSemanticFingerprint {\n static final String VERSION = \"p7-gate-b-semantic-v3-corridor\";\n static final String PARSER_VERSION = \"readsb-trace-v2-globe-history-json\";\n static final String DAY_DELTA_VERSION = HistoricalDayDelta.VERSION;\n\n private GateBSemanticFingerprint() {\n }\n\n static String hash(\n BootstrapSourceManifest manifest,\n BootstrapCoverageSpec coverage,\n GateBContentPolicy contentPolicy\n ) {\n return hash(manifest, coverage, contentPolicy, ArchiveSafetyLimits.defaults());\n }\n\n static String hash(\n BootstrapSourceManifest manifest,\n BootstrapCoverageSpec coverage,\n GateBContentPolicy contentPolicy,\n ArchiveSafetyLimits archiveLimits\n ) {\n if (manifest == null || coverage == null || contentPolicy == null || archiveLimits == null) {\n throw new IllegalArgumentException(\"manifest, coverage, contentPolicy and archiveLimits are required\");\n }\n MessageDigest digest = sha256();\n put(digest, VERSION);\n put(digest, manifest.manifestSha256());\n put(digest, coverage.version());\n put(digest, coverage.hash());\n put(digest, PARSER_VERSION);\n put(digest, archiveLimits.semanticFingerprintComponent());\n put(digest, HistoricalEligibilityV1.VERSION);\n put(digest, HistoricalSegmenterV1.VERSION);\n put(digest, HistoricalCorridorModel.VERSION);\n put(digest, DAY_DELTA_VERSION);\n put(digest, Integer.toString(contentPolicy.maximumTraceParseFailures()));\n put(digest, Long.toString(contentPolicy.minimumAcceptedPoints()));\n return HexFormat.of().formatHex(digest.digest());\n }\n\n private static MessageDigest sha256() {\n try {\n return MessageDigest.getInstance(\"SHA-256\");\n } catch (NoSuchAlgorithmException exception) {\n throw new IllegalStateException(\"SHA-256 unavailable\", exception);\n }\n }\n\n private static void put(MessageDigest digest, String value) {\n byte[] bytes = value.getBytes(StandardCharsets.UTF_8);\n digest.update(ByteBuffer.allocate(4).putInt(bytes.length).array());\n digest.update(bytes);\n }\n}\n" }, { "path": "backend/historical-bootstrap-tool/src/main/java/hu/gombos/flightspotter/historicalbootstrap/GateBQualificationRecordWriter.java", "encoding": "UTF-8", "sha256": "f1a14fe3060a4b3fe9a89fddcf7e6cf341fe148e14157814547aeb055e1bba06", "content": "package hu.gombos.flightspotter.historicalbootstrap;\n\nimport tools.jackson.databind.ObjectMapper;\nimport tools.jackson.databind.json.JsonMapper;\n\nimport java.io.IOException;\nimport java.nio.file.Files;\nimport java.nio.file.Path;\nimport java.nio.file.StandardCopyOption;\nimport java.nio.file.StandardOpenOption;\n\n/** Atomic owner-readable JSON output for a completed Gate B fixture/real qualification run. */\nfinal class GateBQualificationRecordWriter {\n private final ObjectMapper mapper = JsonMapper.builder().build();\n\n void write(Path target, GateBQualificationRecord record) throws IOException {\n if (target == null || record == null) throw new IllegalArgumentException(\"target and record are required\");\n Path absolute = target.toAbsolutePath().normalize();\n Path parent = absolute.getParent();\n if (parent == null) throw new IllegalArgumentException(\"qualification record target requires a parent directory\");\n Files.createDirectories(parent);\n Path temporary = parent.resolve(absolute.getFileName().toString() + \".tmp\");\n byte[] json = mapper.writerWithDefaultPrettyPrinter().writeValueAsBytes(record.toJsonModel());\n Files.write(temporary, json, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE);\n try {\n Files.move(temporary, absolute, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);\n } catch (java.nio.file.AtomicMoveNotSupportedException ignored) {\n Files.move(temporary, absolute, StandardCopyOption.REPLACE_EXISTING);\n }\n }\n}\n" }, { "path": "backend/historical-bootstrap-tool/src/main/java/hu/gombos/flightspotter/historicalbootstrap/GateBSourceDayRunner.java", "encoding": "UTF-8", "sha256": "db8241fa57e30cb57b48e88ede759c5a0c4dbfe1069a5c55815d80b5aa203416", "content": "package hu.gombos.flightspotter.historicalbootstrap;\n\nimport java.io.BufferedInputStream;\nimport java.io.IOException;\nimport java.io.InputStream;\nimport java.io.OutputStream;\nimport java.lang.management.ManagementFactory;\nimport java.lang.management.MemoryMXBean;\nimport java.lang.management.ThreadMXBean;\nimport java.nio.file.FileStore;\nimport java.nio.file.Files;\nimport java.nio.file.Path;\nimport java.nio.file.StandardCopyOption;\nimport java.nio.file.StandardOpenOption;\nimport java.security.MessageDigest;\nimport java.security.NoSuchAlgorithmException;\nimport java.time.Instant;\nimport java.time.LocalDate;\nimport java.time.ZoneOffset;\nimport java.util.ArrayList;\nimport java.util.EnumMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Objects;\nimport java.util.TreeMap;\n\n/** Bounded one-source-day execution using the already-qualified P7.2 derivation core. */\nfinal class GateBSourceDayRunner {\n private static final int COPY_BUFFER_BYTES = 1024 * 1024;\n private static final long DOWNLOAD_PROGRESS_BYTES = 256L * 1024L * 1024L;\n private static final int TRACE_PROGRESS_INTERVAL = 1_000;\n\n private final ArchiveSafetyLimits archiveLimits;\n private final GateBAssetSource assetSource;\n\n GateBSourceDayRunner(ArchiveSafetyLimits archiveLimits, GateBAssetSource assetSource) {\n if (archiveLimits == null || assetSource == null) {\n throw new IllegalArgumentException(\"archiveLimits and assetSource are required\");\n }\n this.archiveLimits = archiveLimits;\n this.assetSource = assetSource;\n }\n\n GateBQualificationRecord run(GateBExecutionPlan plan, BootstrapCoverageSpec coverage) throws Exception {\n return run(plan, coverage, BeforeCommitObserver.none());\n }\n\n GateBQualificationRecord run(\n GateBExecutionPlan plan,\n BootstrapCoverageSpec coverage,\n BeforeCommitObserver beforeCommitObserver\n ) throws Exception {\n return runInternal(plan, coverage, 0, true,\n beforeCommitObserver == null ? BeforeCommitObserver.none() : beforeCommitObserver);\n }\n\n GateBQualificationRecord runContinuation(\n GateBExecutionPlan plan,\n BootstrapCoverageSpec coverage,\n int expectedCommittedDays\n ) throws Exception {\n return runContinuation(plan, coverage, expectedCommittedDays, BeforeCommitObserver.none());\n }\n\n GateBQualificationRecord runContinuation(\n GateBExecutionPlan plan,\n BootstrapCoverageSpec coverage,\n int expectedCommittedDays,\n BeforeCommitObserver beforeCommitObserver\n ) throws Exception {\n if (expectedCommittedDays < 0) throw new IllegalArgumentException(\"expectedCommittedDays cannot be negative\");\n return runInternal(plan, coverage, expectedCommittedDays, false,\n beforeCommitObserver == null ? BeforeCommitObserver.none() : beforeCommitObserver);\n }\n\n private GateBQualificationRecord runInternal(\n GateBExecutionPlan plan,\n BootstrapCoverageSpec coverage,\n int expectedCommittedDays,\n boolean requireEmptyGateBState,\n BeforeCommitObserver beforeCommitObserver\n ) throws Exception {\n if (plan == null || coverage == null) throw new IllegalArgumentException(\"plan and coverage are required\");\n Objects.requireNonNull(beforeCommitObserver, \"beforeCommitObserver\");\n if (!plan.coverageVersion().equals(coverage.version()) || !plan.coverageHash().equals(coverage.hash())) {\n throw new IllegalArgumentException(\"execution coverage does not match frozen source-day plan\");\n }\n if (plan.selectedFamily().assets().stream().anyMatch(asset -> asset.sha256() == null)) {\n throw new IllegalArgumentException(\"source-day runner refuses selected assets without upstream SHA-256\");\n }\n\n String gateLabel = requireEmptyGateBState ? \"Gate B\" : \"Historical batch\";\n long wallStart = System.nanoTime();\n ThreadMXBean threadBean = ManagementFactory.getThreadMXBean();\n long cpuStart = currentThreadCpuTime(threadBean);\n MemoryProbe memory = new MemoryProbe(gateLabel);\n memory.log(\"phase=PRECHECK sourceDate=\" + plan.sourceDate()\n + \" role=\" + plan.role()\n + \" selectedBytes=\" + plan.selectedSourceBytes()\n + \" expectedCommittedDays=\" + expectedCommittedDays);\n\n Files.createDirectories(plan.scratchRoot());\n String runPrefix = requireEmptyGateBState ? \"gate-b-\" : \"historical-batch-\";\n Path runDir = plan.scratchRoot().resolve(runPrefix + plan.datasetId() + \"-\" + plan.sourceDate());\n if (Files.exists(runDir)) throw new IllegalStateException(gateLabel + \" raw scratch run directory already exists\");\n\n FileStore fileStore = Files.getFileStore(plan.scratchRoot());\n long usableScratch = fileStore.getUsableSpace();\n long scratchRequired = Math.addExact(plan.selectedSourceBytes(), plan.budget().scratchReserveBytes());\n if (usableScratch < scratchRequired) {\n throw new IllegalStateException(\"insufficient raw scratch: usable=\" + usableScratch\n + \" required=\" + scratchRequired);\n }\n Files.createDirectory(runDir);\n memory.log(\"phase=SCRATCH_READY usableBytes=\" + usableScratch + \" requiredBytes=\" + scratchRequired);\n\n boolean committed = false;\n try (LocalDayCommitStore store = LocalDayCommitStore.open(plan.stateDatabase(), archiveLimits)) {\n if (store.isCommitted(plan.dayIdentity())) {\n throw new IllegalStateException(\"source day is already committed; FULL_DAY_REBUILD is prohibited\");\n }\n LocalDayCommitStore.ExecutionState before = store.executionState(plan.datasetId());\n validateStartingState(before, expectedCommittedDays, requireEmptyGateBState);\n\n MaterializedAssets materialized = materialize(plan, runDir, memory);\n memory.log(\"phase=DOWNLOAD_VERIFY_COMPLETE transferBytes=\" + materialized.transferBytes()\n + \" verifiedAssets=\" + materialized.paths().size());\n\n DerivationResult derived = derive(plan, coverage, materialized.paths(), before.carry(), memory);\n DerivationMetrics metrics = derived.metrics();\n String contentFailureReason = terminalContentFailureReason(\n metrics.archiveStats.traceEntryCount(),\n metrics.acceptedPointCount,\n plan.budget().contentPolicy().minimumAcceptedPoints());\n if (contentFailureReason != null) {\n throw new ContentQualityFailure(\n contentFailureReason,\n materialized.transferBytes(),\n materialized.paths().size(),\n metrics.archiveStats.tarEntryCount(),\n metrics.archiveStats.traceEntryCount(),\n metrics.parsedTraceCount,\n metrics.traceParseFailureCount,\n metrics.malformedPointCount,\n metrics.acceptedPointCount,\n metrics.rejectedPointCount,\n metrics.finalizedSegmentCount,\n metrics.outOfOrderRejectedCount);\n }\n PreparedDayMetrics prepared = new PreparedDayMetrics(\n plan.datasetId(),\n plan.sourceDate(),\n plan.role(),\n plan.manifestSha256(),\n plan.dayManifestSha256(),\n plan.semanticFingerprint(),\n plan.coverageVersion(),\n plan.coverageHash(),\n LocalDayCommitStore.EvidenceQualityStatus.QUALIFIED,\n null,\n materialized.transferBytes(),\n plan.selectedFamily().assets().size(),\n metrics.archiveStats.tarEntryCount(),\n metrics.archiveStats.traceEntryCount(),\n metrics.parsedTraceCount,\n metrics.traceParseFailureCount,\n metrics.malformedPointCount,\n metrics.acceptedPointCount,\n metrics.rejectedPointCount,\n metrics.finalizedSegmentCount,\n metrics.outOfOrderRejectedCount,\n derived.delta().excludedSegmentCount(),\n derived.outgoingCarry().size(),\n derived.delta().checksum(),\n Instant.now()\n );\n // This callback is deliberately before the compact commit. Gate D uses it to atomically\n // persist staging metrics first, closing the hard-crash window between day commit and report output.\n beforeCommitObserver.beforeCommit(prepared);\n\n memory.log(\"phase=COMMIT_START accepted=\" + metrics.acceptedPointCount\n + \" finalizedSegments=\" + metrics.finalizedSegmentCount);\n store.apply(\n LocalDayCommitStore.ReplayMode.FULL_DAY_REBUILD,\n plan.dayIdentity(),\n derived.delta(),\n derived.outgoingCarry(),\n LocalDayCommitStore.EvidenceQualityStatus.QUALIFIED,\n LocalDayCommitStore.FailureInjector.none()\n );\n committed = true;\n LocalDayCommitStore.ExecutionState after = store.executionState(plan.datasetId());\n if (after.committedDayCount() != expectedCommittedDays + 1) {\n throw new IllegalStateException(\"source-day commit did not advance committed prefix by exactly one day\");\n }\n memory.log(\"phase=COMMIT_COMPLETE patterns=\" + after.patternCount()\n + \" hourBins=\" + after.hourBinCount()\n + \" holdoutOutcomes=\" + after.holdoutOutcomeCount()\n + \" outgoingCarry=\" + derived.outgoingCarry().size());\n\n long compactStateBytes = compactStateBytes(plan.stateDatabase());\n memory.log(\"phase=CLEANUP_START compactStateBytes=\" + compactStateBytes);\n cleanupTree(runDir);\n long rawAfterCleanup = Files.exists(runDir) ? directoryBytes(runDir) : 0L;\n if (rawAfterCleanup != 0L || Files.exists(runDir)) {\n throw new IllegalStateException(gateLabel + \" raw scratch cleanup did not return run directory to 0 B\");\n }\n memory.log(\"phase=CLEANUP_COMPLETE rawScratchBytes=0\");\n\n long wallMillis = nanosToMillis(System.nanoTime() - wallStart);\n long cpuEnd = currentThreadCpuTime(threadBean);\n long cpuMillis = cpuStart < 0L || cpuEnd < 0L ? -1L : nanosToMillis(Math.max(0L, cpuEnd - cpuStart));\n GateBContentPolicy content = plan.budget().contentPolicy();\n return new GateBQualificationRecord(\n plan.datasetId(),\n plan.sourceDate(),\n plan.manifestSha256(),\n plan.dayManifestSha256(),\n plan.semanticFingerprint(),\n plan.coverageVersion(),\n plan.coverageHash(),\n GateBSemanticFingerprint.PARSER_VERSION,\n HistoricalEligibilityV1.VERSION,\n HistoricalSegmenterV1.VERSION,\n GateBSemanticFingerprint.DAY_DELTA_VERSION,\n plan.selectedRelease().releaseId(),\n plan.selectedRelease().tag(),\n plan.selectedFamily().kind().name(),\n plan.budget().maximumTransferBytes(),\n plan.budget().scratchReserveBytes(),\n plan.budget().maximumAssetAttempts(),\n content.maximumTraceParseFailures(),\n content.minimumAcceptedPoints(),\n plan.selectedSourceBytes(),\n materialized.transferBytes(),\n plan.selectedFamily().assets().size(),\n usableScratch,\n scratchRequired,\n materialized.peakRawScratchBytes(),\n rawAfterCleanup,\n compactStateBytes,\n metrics.archiveStats.tarEntryCount(),\n metrics.archiveStats.traceEntryCount(),\n metrics.parsedTraceCount,\n metrics.traceParseFailureCount,\n metrics.malformedPointCount,\n metrics.acceptedPointCount,\n metrics.rejectedPointCount,\n metrics.rejectionCountsAsStrings(),\n metrics.finalizedSegmentCount,\n metrics.outOfOrderRejectedCount,\n derived.outgoingCarry().size(),\n after.patternCount(),\n after.hourBinCount(),\n after.holdoutOutcomeCount(),\n derived.delta().excludedSegmentCount(),\n derived.delta().checksum(),\n wallMillis,\n cpuMillis,\n memory.maxObservedHeapBytes,\n true\n );\n } catch (Exception | Error failure) {\n try {\n cleanupTree(runDir);\n } catch (Exception cleanupFailure) {\n failure.addSuppressed(cleanupFailure);\n }\n if (committed) {\n // The local day marker remains authoritative. A caller must not rerun FULL_DAY_REBUILD.\n }\n throw failure;\n }\n }\n\n private static void validateStartingState(\n LocalDayCommitStore.ExecutionState before,\n int expectedCommittedDays,\n boolean requireEmptyGateBState\n ) {\n if (before.committedDayCount() != expectedCommittedDays) {\n throw new IllegalStateException(\"compact state committed-day count does not match expected prefix\");\n }\n boolean hasEvidence = before.patternCount() > 0 || before.hourBinCount() > 0\n || before.holdoutOutcomeCount() > 0 || !before.carry().isEmpty();\n if (requireEmptyGateBState && (expectedCommittedDays != 0 || hasEvidence)) {\n throw new IllegalStateException(\"Gate B requires an empty compact dataset state\");\n }\n if (!requireEmptyGateBState && expectedCommittedDays == 0 && hasEvidence) {\n throw new IllegalStateException(\"historical batch zero-day prefix requires empty compact evidence state\");\n }\n }\n\n private MaterializedAssets materialize(GateBExecutionPlan plan, Path runDir, MemoryProbe memory) throws Exception {\n List verified = new ArrayList<>();\n long transferred = 0L;\n long verifiedBytes = 0L;\n long peakRawScratch = 0L;\n int index = 0;\n int assetCount = plan.selectedFamily().assets().size();\n\n for (SourceAsset asset : plan.selectedFamily().assets()) {\n index = Math.addExact(index, 1);\n IOException lastFailure = null;\n boolean complete = false;\n for (int attempt = 1; attempt <= plan.budget().maximumAssetAttempts(); attempt++) {\n Path partial = runDir.resolve(String.format(\"asset-%03d-%d.part\", index, asset.assetId()));\n Path destination = runDir.resolve(String.format(\"asset-%03d-%d.verified\", index, asset.assetId()));\n Files.deleteIfExists(partial);\n Files.deleteIfExists(destination);\n long attemptBytes = 0L;\n long nextProgressBytes = DOWNLOAD_PROGRESS_BYTES;\n MessageDigest digest = sha256();\n memory.log(\"phase=DOWNLOAD_START asset=\" + index + \"/\" + assetCount\n + \" assetId=\" + asset.assetId() + \" attempt=\" + attempt\n + \" expectedBytes=\" + asset.byteSize());\n try (InputStream input = new BufferedInputStream(assetSource.open(plan.selectedRelease(), asset), COPY_BUFFER_BYTES);\n OutputStream output = Files.newOutputStream(partial, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) {\n byte[] buffer = new byte[COPY_BUFFER_BYTES];\n int read;\n while ((read = input.read(buffer)) != -1) {\n transferred = Math.addExact(transferred, read);\n attemptBytes = Math.addExact(attemptBytes, read);\n if (transferred > plan.budget().maximumTransferBytes()) {\n throw new IOException(\"source-day transfer budget exceeded\");\n }\n if (attemptBytes > asset.byteSize()) {\n throw new ImmutableHistoricalSourceException(\n \"IMMUTABLE_ASSET_SIZE_MISMATCH\",\n \"asset exceeded immutable expected byte size: \" + asset.assetId());\n }\n output.write(buffer, 0, read);\n digest.update(buffer, 0, read);\n peakRawScratch = Math.max(peakRawScratch, Math.addExact(verifiedBytes, attemptBytes));\n if (attemptBytes >= nextProgressBytes) {\n memory.log(\"phase=DOWNLOAD asset=\" + index + \"/\" + assetCount\n + \" assetId=\" + asset.assetId()\n + \" assetBytes=\" + attemptBytes + \"/\" + asset.byteSize()\n + \" transferBytes=\" + transferred + \"/\" + plan.budget().maximumTransferBytes());\n while (nextProgressBytes <= attemptBytes) {\n nextProgressBytes = Math.addExact(nextProgressBytes, DOWNLOAD_PROGRESS_BYTES);\n }\n }\n }\n } catch (InterruptedException interrupted) {\n Files.deleteIfExists(partial);\n Thread.currentThread().interrupt();\n throw interrupted;\n } catch (IOException failure) {\n Files.deleteIfExists(partial);\n lastFailure = failure;\n memory.log(\"phase=DOWNLOAD_FAILED asset=\" + index + \"/\" + assetCount\n + \" assetId=\" + asset.assetId() + \" attempt=\" + attempt\n + \" bytes=\" + attemptBytes);\n if (failure instanceof ImmutableHistoricalSourceException\n || attempt == plan.budget().maximumAssetAttempts()) {\n throw failure;\n }\n continue;\n }\n\n String actualDigest = java.util.HexFormat.of().formatHex(digest.digest());\n if (attemptBytes != asset.byteSize() || !actualDigest.equals(asset.sha256())) {\n Files.deleteIfExists(partial);\n lastFailure = new ImmutableHistoricalSourceException(\n \"IMMUTABLE_ASSET_VERIFICATION_FAILED\",\n \"asset size/SHA-256 mismatch: assetId=\" + asset.assetId());\n memory.log(\"phase=VERIFY_FAILED asset=\" + index + \"/\" + assetCount\n + \" assetId=\" + asset.assetId() + \" bytes=\" + attemptBytes);\n throw lastFailure;\n }\n moveVerified(partial, destination);\n verified.add(destination);\n verifiedBytes = Math.addExact(verifiedBytes, attemptBytes);\n memory.log(\"phase=VERIFY_COMPLETE asset=\" + index + \"/\" + assetCount\n + \" assetId=\" + asset.assetId() + \" verifiedBytes=\" + attemptBytes);\n complete = true;\n break;\n }\n if (!complete) throw lastFailure == null ? new IOException(\"asset materialization failed\") : lastFailure;\n }\n\n if (verifiedBytes != plan.selectedSourceBytes()) {\n throw new ImmutableHistoricalSourceException(\n \"IMMUTABLE_ASSET_FAMILY_SIZE_MISMATCH\",\n \"verified asset family bytes do not match immutable selected total\");\n }\n return new MaterializedAssets(List.copyOf(verified), transferred, Math.max(peakRawScratch, verifiedBytes));\n }\n\n private DerivationResult derive(\n GateBExecutionPlan plan,\n BootstrapCoverageSpec coverage,\n List verifiedAssets,\n Map incomingCarries,\n MemoryProbe memory\n ) throws Exception {\n ReadsbTraceParser parser = new ReadsbTraceParser(archiveLimits);\n HistoricalEligibilityV1 eligibility = new HistoricalEligibilityV1();\n HistoricalSegmenterV1 segmenter = new HistoricalSegmenterV1();\n HistoricalDayDelta.Builder deltaBuilder = HistoricalDayDelta.builder(plan.role());\n TreeMap carries = new TreeMap<>(\n incomingCarries == null ? Map.of() : incomingCarries);\n if (carries.size() > archiveLimits.maxCarryAircraft()) {\n throw new IOException(\"incoming carry aircraft count exceeded hard bound\");\n }\n DerivationMetrics metrics = new DerivationMetrics();\n Instant dayStart = plan.sourceDate().atStartOfDay(ZoneOffset.UTC).toInstant();\n Instant carryUncertaintyEnd = dayStart.plus(HistoricalSegmenterV1.carryHorizon());\n memory.log(\"phase=DERIVE_START verifiedAssets=\" + verifiedAssets.size()\n + \" incomingCarry=\" + carries.size());\n\n BoundedTraceArchiveReader.ReadStats archiveStats;\n try (InputStream archive = new AssetFamilyInputStream(verifiedAssets)) {\n archiveStats = new BoundedTraceArchiveReader(archiveLimits).read(archive, (name, compressedTrace) -> {\n ReadsbTrace trace;\n try {\n trace = parser.parseGzip(compressedTrace);\n } catch (IOException parseFailure) {\n metrics.traceParseFailureCount = Math.addExact(metrics.traceParseFailureCount, 1);\n if (metrics.traceParseFailureCount > plan.budget().contentPolicy().maximumTraceParseFailures()) {\n throw new IOException(\"trace parse failure count exceeded explicit source-day maximum\", parseFailure);\n }\n maybeLogDerivationProgress(metrics, carries.size(), memory);\n return;\n }\n metrics.parsedTraceCount = Math.addExact(metrics.parsedTraceCount, 1);\n metrics.malformedPointCount = Math.addExact(metrics.malformedPointCount, trace.malformedPointCount());\n\n List accepted = new ArrayList<>();\n for (ReadsbTracePoint point : trace.points()) {\n HistoricalEligibilityV1.Decision decision = eligibility.evaluate(trace, point, plan.sourceDate(), coverage);\n if (decision.accepted()) {\n accepted.add(decision.point());\n metrics.acceptedPointCount = Math.addExact(metrics.acceptedPointCount, 1L);\n } else {\n metrics.rejectedPointCount = Math.addExact(metrics.rejectedPointCount, 1L);\n metrics.rejectionCounts.merge(decision.rejectionReason(), 1L, Math::addExact);\n }\n }\n if (accepted.isEmpty()) {\n maybeLogDerivationProgress(metrics, carries.size(), memory);\n return;\n }\n\n HistoricalSegmentCarry incoming = carries.remove(trace.icao24());\n boolean boundaryTruncated = incoming == null\n && accepted.getFirst().observedAt().isBefore(carryUncertaintyEnd);\n HistoricalSegmenterV1.Result segmented = segmenter.process(incoming, accepted, boundaryTruncated);\n metrics.outOfOrderRejectedCount = Math.addExact(\n metrics.outOfOrderRejectedCount, segmented.outOfOrderRejected());\n for (HistoricalSegment segment : segmented.finalizedSegments()) {\n deltaBuilder.accept(segment);\n metrics.finalizedSegmentCount = Math.addExact(metrics.finalizedSegmentCount, 1L);\n }\n if (segmented.outgoingCarry() != null) {\n carries.put(trace.icao24(), segmented.outgoingCarry());\n if (carries.size() > archiveLimits.maxCarryAircraft()) {\n throw new IOException(\"carry aircraft count exceeded hard bound\");\n }\n }\n maybeLogDerivationProgress(metrics, carries.size(), memory);\n });\n }\n metrics.archiveStats = archiveStats;\n memory.log(\"phase=DERIVE_ARCHIVE_COMPLETE tarEntries=\" + archiveStats.tarEntryCount()\n + \" traces=\" + archiveStats.traceEntryCount()\n + \" parsed=\" + metrics.parsedTraceCount\n + \" accepted=\" + metrics.acceptedPointCount\n + \" rejected=\" + metrics.rejectedPointCount);\n\n Instant dayEnd = plan.sourceDate().plusDays(1).atStartOfDay(ZoneOffset.UTC).toInstant();\n TreeMap outgoing = new TreeMap<>();\n for (Map.Entry entry : carries.entrySet()) {\n HistoricalSegment quiet = segmenter.finalizeQuiet(entry.getValue(), dayEnd);\n if (quiet != null) {\n deltaBuilder.accept(quiet);\n metrics.finalizedSegmentCount = Math.addExact(metrics.finalizedSegmentCount, 1L);\n } else {\n outgoing.put(entry.getKey(), entry.getValue());\n }\n }\n if (outgoing.size() > archiveLimits.maxCarryAircraft()) {\n throw new IOException(\"outgoing carry count exceeded hard bound\");\n }\n HistoricalDayDelta delta = deltaBuilder.seal();\n memory.log(\"phase=DERIVE_COMPLETE patterns=\" + delta.patterns().size()\n + \" hourBins=\" + delta.hourBins().size()\n + \" holdoutOutcomes=\" + delta.holdoutOutcomes().size()\n + \" finalizedSegments=\" + metrics.finalizedSegmentCount\n + \" outgoingCarry=\" + outgoing.size());\n return new DerivationResult(delta, Map.copyOf(outgoing), metrics);\n }\n\n private static void maybeLogDerivationProgress(\n DerivationMetrics metrics,\n int carryCount,\n MemoryProbe memory\n ) {\n int processed = Math.addExact(metrics.parsedTraceCount, metrics.traceParseFailureCount);\n memory.sample();\n if (processed > 0 && processed % TRACE_PROGRESS_INTERVAL == 0) {\n memory.log(\"phase=DERIVE traces=\" + processed\n + \" parsed=\" + metrics.parsedTraceCount\n + \" parseFailures=\" + metrics.traceParseFailureCount\n + \" accepted=\" + metrics.acceptedPointCount\n + \" rejected=\" + metrics.rejectedPointCount\n + \" finalizedSegments=\" + metrics.finalizedSegmentCount\n + \" carry=\" + carryCount);\n }\n }\n\n static String terminalContentFailureReason(\n int traceEntryCount,\n long acceptedPointCount,\n long minimumAcceptedPoints\n ) {\n if (traceEntryCount < 0 || acceptedPointCount < 0L || minimumAcceptedPoints < 1L) {\n throw new IllegalArgumentException(\"invalid source-day content quality counters\");\n }\n if (traceEntryCount == 0) {\n return \"TRACE_SET_MISSING\";\n }\n if (acceptedPointCount < minimumAcceptedPoints) {\n return \"COVERAGE_EVIDENCE_SPARSE\";\n }\n return null;\n }\n\n private static void moveVerified(Path partial, Path destination) throws IOException {\n try {\n Files.move(partial, destination, StandardCopyOption.ATOMIC_MOVE);\n } catch (java.nio.file.AtomicMoveNotSupportedException ignored) {\n Files.move(partial, destination);\n }\n }\n\n private static MessageDigest sha256() {\n try {\n return MessageDigest.getInstance(\"SHA-256\");\n } catch (NoSuchAlgorithmException exception) {\n throw new IllegalStateException(\"SHA-256 unavailable\", exception);\n }\n }\n\n private static long compactStateBytes(Path stateDatabase) throws IOException {\n Path parent = stateDatabase.toAbsolutePath().normalize().getParent();\n if (parent == null || !Files.exists(parent)) return 0L;\n String prefix = stateDatabase.getFileName().toString() + \".\";\n long total = 0L;\n try (var stream = Files.list(parent)) {\n for (Path path : stream.filter(Files::isRegularFile)\n .filter(path -> path.getFileName().toString().startsWith(prefix)).toList()) {\n total = Math.addExact(total, Files.size(path));\n }\n }\n return total;\n }\n\n private static long directoryBytes(Path directory) throws IOException {\n if (!Files.exists(directory)) return 0L;\n long total = 0L;\n try (var stream = Files.walk(directory)) {\n for (Path path : stream.filter(Files::isRegularFile).toList()) {\n total = Math.addExact(total, Files.size(path));\n }\n }\n return total;\n }\n\n private static void cleanupTree(Path root) throws IOException {\n if (root == null || !Files.exists(root)) return;\n try (var stream = Files.walk(root)) {\n List paths = stream.sorted(java.util.Comparator.reverseOrder()).toList();\n IOException failure = null;\n for (Path path : paths) {\n try {\n Files.deleteIfExists(path);\n } catch (IOException exception) {\n if (failure == null) failure = exception;\n else failure.addSuppressed(exception);\n }\n }\n if (failure != null) throw failure;\n }\n }\n\n private static long currentThreadCpuTime(ThreadMXBean bean) {\n if (bean == null || !bean.isCurrentThreadCpuTimeSupported()) return -1L;\n try {\n if (!bean.isThreadCpuTimeEnabled()) bean.setThreadCpuTimeEnabled(true);\n return bean.getCurrentThreadCpuTime();\n } catch (UnsupportedOperationException | SecurityException ignored) {\n return -1L;\n }\n }\n\n private static long nanosToMillis(long nanos) {\n return nanos < 0L ? -1L : nanos / 1_000_000L;\n }\n\n static final class ContentQualityFailure extends IllegalStateException {\n private final String reasonCode;\n private final long transferBytes;\n private final int verifiedAssetCount;\n private final int tarEntryCount;\n private final int traceEntryCount;\n private final int parsedTraceCount;\n private final int traceParseFailureCount;\n private final long malformedPointCount;\n private final long acceptedPointCount;\n private final long rejectedPointCount;\n private final long finalizedSegmentCount;\n private final long outOfOrderRejectedCount;\n\n ContentQualityFailure(\n String reasonCode,\n long transferBytes,\n int verifiedAssetCount,\n int tarEntryCount,\n int traceEntryCount,\n int parsedTraceCount,\n int traceParseFailureCount,\n long malformedPointCount,\n long acceptedPointCount,\n long rejectedPointCount,\n long finalizedSegmentCount,\n long outOfOrderRejectedCount\n ) {\n super(\"source-day terminal content-quality policy failed: \" + reasonCode);\n if (!\"TRACE_SET_MISSING\".equals(reasonCode)\n && !\"COVERAGE_EVIDENCE_SPARSE\".equals(reasonCode)) {\n throw new IllegalArgumentException(\"unsupported terminal content-quality reason \" + reasonCode);\n }\n if (transferBytes < 0L || verifiedAssetCount < 0 || tarEntryCount < 0 || traceEntryCount < 0\n || parsedTraceCount < 0 || traceParseFailureCount < 0 || malformedPointCount < 0L\n || acceptedPointCount < 0L || rejectedPointCount < 0L || finalizedSegmentCount < 0L\n || outOfOrderRejectedCount < 0L) {\n throw new IllegalArgumentException(\"invalid terminal content-quality counters\");\n }\n this.reasonCode = reasonCode;\n this.transferBytes = transferBytes;\n this.verifiedAssetCount = verifiedAssetCount;\n this.tarEntryCount = tarEntryCount;\n this.traceEntryCount = traceEntryCount;\n this.parsedTraceCount = parsedTraceCount;\n this.traceParseFailureCount = traceParseFailureCount;\n this.malformedPointCount = malformedPointCount;\n this.acceptedPointCount = acceptedPointCount;\n this.rejectedPointCount = rejectedPointCount;\n this.finalizedSegmentCount = finalizedSegmentCount;\n this.outOfOrderRejectedCount = outOfOrderRejectedCount;\n }\n\n String reasonCode() { return reasonCode; }\n long transferBytes() { return transferBytes; }\n int verifiedAssetCount() { return verifiedAssetCount; }\n int tarEntryCount() { return tarEntryCount; }\n int traceEntryCount() { return traceEntryCount; }\n int parsedTraceCount() { return parsedTraceCount; }\n int traceParseFailureCount() { return traceParseFailureCount; }\n long malformedPointCount() { return malformedPointCount; }\n long acceptedPointCount() { return acceptedPointCount; }\n long rejectedPointCount() { return rejectedPointCount; }\n long finalizedSegmentCount() { return finalizedSegmentCount; }\n long outOfOrderRejectedCount() { return outOfOrderRejectedCount; }\n }\n\n @FunctionalInterface\n interface BeforeCommitObserver {\n void beforeCommit(PreparedDayMetrics metrics) throws Exception;\n\n static BeforeCommitObserver none() {\n return ignored -> { };\n }\n }\n\n record PreparedDayMetrics(\n java.util.UUID datasetId,\n LocalDate sourceDate,\n DatasetWindow.DateRole role,\n String manifestSha256,\n String dayManifestSha256,\n String semanticFingerprint,\n String coverageVersion,\n String coverageHash,\n LocalDayCommitStore.EvidenceQualityStatus quality,\n String reasonCode,\n long transferBytes,\n int verifiedAssetCount,\n int tarEntryCount,\n int traceEntryCount,\n int parsedTraceCount,\n int traceParseFailureCount,\n long malformedPointCount,\n long acceptedPointCount,\n long rejectedPointCount,\n long finalizedSegmentCount,\n long outOfOrderRejectedCount,\n int excludedSegmentCount,\n int outgoingCarryCount,\n String dayDeltaChecksum,\n Instant preparedAt\n ) {\n PreparedDayMetrics {\n Objects.requireNonNull(datasetId, \"datasetId\");\n Objects.requireNonNull(sourceDate, \"sourceDate\");\n Objects.requireNonNull(role, \"role\");\n Objects.requireNonNull(quality, \"quality\");\n Objects.requireNonNull(preparedAt, \"preparedAt\");\n requireHash(manifestSha256, \"manifestSha256\");\n requireHash(dayManifestSha256, \"dayManifestSha256\");\n requireHash(semanticFingerprint, \"semanticFingerprint\");\n requireHash(coverageHash, \"coverageHash\");\n requireHash(dayDeltaChecksum, \"dayDeltaChecksum\");\n if (coverageVersion == null || coverageVersion.isBlank()\n || transferBytes < 0L || verifiedAssetCount < 0 || tarEntryCount < 0 || traceEntryCount < 0\n || parsedTraceCount < 0 || traceParseFailureCount < 0 || malformedPointCount < 0L\n || acceptedPointCount < 0L || rejectedPointCount < 0L || finalizedSegmentCount < 0L\n || outOfOrderRejectedCount < 0L || excludedSegmentCount < 0 || outgoingCarryCount < 0) {\n throw new IllegalArgumentException(\"invalid prepared day metrics\");\n }\n if (reasonCode != null && !reasonCode.matches(\"[A-Z0-9_]{1,64}\")) {\n throw new IllegalArgumentException(\"reasonCode must be a finite uppercase identifier\");\n }\n }\n\n private static void requireHash(String value, String name) {\n if (value == null || !value.matches(\"[0-9a-f]{64}\")) {\n throw new IllegalArgumentException(name + \" must be lowercase SHA-256\");\n }\n }\n }\n\n private record MaterializedAssets(List paths, long transferBytes, long peakRawScratchBytes) {\n }\n\n private record DerivationResult(\n HistoricalDayDelta delta,\n Map outgoingCarry,\n DerivationMetrics metrics\n ) {\n }\n\n private static final class DerivationMetrics {\n private BoundedTraceArchiveReader.ReadStats archiveStats = new BoundedTraceArchiveReader.ReadStats(0, 0, 0L);\n private int parsedTraceCount;\n private int traceParseFailureCount;\n private long malformedPointCount;\n private long acceptedPointCount;\n private long rejectedPointCount;\n private long finalizedSegmentCount;\n private long outOfOrderRejectedCount;\n private final EnumMap rejectionCounts =\n new EnumMap<>(HistoricalEligibilityV1.RejectionReason.class);\n\n Map rejectionCountsAsStrings() {\n TreeMap result = new TreeMap<>();\n rejectionCounts.forEach((key, value) -> result.put(key.name(), value));\n return Map.copyOf(result);\n }\n }\n\n private static final class MemoryProbe {\n private static final long MEBIBYTE = 1024L * 1024L;\n private final MemoryMXBean bean = ManagementFactory.getMemoryMXBean();\n private final String gateLabel;\n private long maxObservedHeapBytes;\n\n MemoryProbe(String gateLabel) {\n this.gateLabel = gateLabel;\n }\n\n void sample() {\n long used = bean.getHeapMemoryUsage().getUsed();\n if (used > maxObservedHeapBytes) maxObservedHeapBytes = used;\n }\n\n void log(String fields) {\n long used = bean.getHeapMemoryUsage().getUsed();\n long committed = bean.getHeapMemoryUsage().getCommitted();\n long maximum = bean.getHeapMemoryUsage().getMax();\n if (used > maxObservedHeapBytes) maxObservedHeapBytes = used;\n System.out.println(\"P7 \" + gateLabel + \" progress \" + fields\n + \" heapUsedMiB=\" + toMiB(used)\n + \" heapCommittedMiB=\" + toMiB(committed)\n + \" heapMaxMiB=\" + (maximum < 0L ? -1L : toMiB(maximum))\n + \" peakHeapMiB=\" + toMiB(maxObservedHeapBytes));\n }\n\n private static long toMiB(long bytes) {\n return bytes / MEBIBYTE;\n }\n }\n\n /** Opens split assets lazily so a 676-part family does not consume hundreds of file descriptors. */\n private static final class AssetFamilyInputStream extends InputStream {\n private final List paths;\n private int index;\n private InputStream current;\n\n AssetFamilyInputStream(List paths) {\n if (paths == null || paths.isEmpty()) throw new IllegalArgumentException(\"verified asset paths are required\");\n this.paths = List.copyOf(paths);\n }\n\n @Override\n public int read() throws IOException {\n byte[] single = new byte[1];\n int read = read(single, 0, 1);\n return read < 0 ? -1 : single[0] & 0xff;\n }\n\n @Override\n public int read(byte[] buffer, int offset, int length) throws IOException {\n if (length == 0) return 0;\n while (true) {\n if (current == null && !openNext()) return -1;\n int read = current.read(buffer, offset, length);\n if (read >= 0) return read;\n current.close();\n current = null;\n }\n }\n\n private boolean openNext() throws IOException {\n if (index >= paths.size()) return false;\n current = new BufferedInputStream(Files.newInputStream(paths.get(index++)), COPY_BUFFER_BYTES);\n return true;\n }\n\n @Override\n public void close() throws IOException {\n if (current != null) current.close();\n current = null;\n index = paths.size();\n }\n }\n}\n" }, { "path": "backend/historical-bootstrap-tool/src/main/java/hu/gombos/flightspotter/historicalbootstrap/GateDPreparedDayMetricsStore.java", "encoding": "UTF-8", "sha256": "3d2fb5db2d60c6d69d3da4de83d27f5792a7a5dd1782d7acd012997b0abc7ee1", "content": "package hu.gombos.flightspotter.historicalbootstrap;\n\nimport tools.jackson.databind.ObjectMapper;\nimport tools.jackson.databind.json.JsonMapper;\n\nimport java.io.IOException;\nimport java.nio.file.Files;\nimport java.nio.file.Path;\nimport java.nio.file.StandardCopyOption;\nimport java.nio.file.StandardOpenOption;\nimport java.time.Instant;\nimport java.time.LocalDate;\nimport java.util.LinkedHashMap;\nimport java.util.Map;\nimport java.util.Objects;\nimport java.util.UUID;\n\n/**\n * Durable pre-commit audit/staging evidence for the laptop Gate D build.\n *\n * A payload source day writes this file after source verification + derivation but before the compact H2 day commit.\n * If the host dies after the H2 commit but before any human-readable final report is written, this artifact already\n * contains the terminal per-day counters required by the later V69 staging request. An uncommitted day may safely\n * overwrite its prepared artifact on explicit rebuild because the H2 committed marker remains authoritative.\n */\nfinal class GateDPreparedDayMetricsStore {\n static final int SCHEMA_VERSION = 1;\n\n private final Path directory;\n private final ObjectMapper mapper = JsonMapper.builder().build();\n\n GateDPreparedDayMetricsStore(Path directory) throws IOException {\n if (directory == null) throw new IllegalArgumentException(\"prepared metrics directory is required\");\n this.directory = directory.toAbsolutePath().normalize();\n Files.createDirectories(this.directory);\n }\n\n Path directory() {\n return directory;\n }\n\n Path pathFor(LocalDate sourceDate) {\n Objects.requireNonNull(sourceDate, \"sourceDate\");\n return directory.resolve(sourceDate + \".prepared.json\");\n }\n\n void write(GateBSourceDayRunner.PreparedDayMetrics metrics) throws IOException {\n Objects.requireNonNull(metrics, \"metrics\");\n Path target = pathFor(metrics.sourceDate());\n Path temporary = target.resolveSibling(target.getFileName().toString() + \".tmp\");\n byte[] json = mapper.writerWithDefaultPrettyPrinter().writeValueAsBytes(toJson(metrics));\n Files.write(temporary, json, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING,\n StandardOpenOption.WRITE);\n try {\n Files.move(temporary, target, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);\n } catch (java.nio.file.AtomicMoveNotSupportedException ignored) {\n Files.move(temporary, target, StandardCopyOption.REPLACE_EXISTING);\n }\n }\n\n boolean exists(LocalDate sourceDate) {\n return Files.isRegularFile(pathFor(sourceDate));\n }\n\n void delete(LocalDate sourceDate) throws IOException {\n Files.deleteIfExists(pathFor(sourceDate));\n }\n\n GateBSourceDayRunner.PreparedDayMetrics read(LocalDate sourceDate) throws Exception {\n Path path = pathFor(sourceDate);\n if (!Files.isRegularFile(path)) {\n throw new IllegalStateException(\"prepared day metrics are missing: \" + path);\n }\n Dto value = mapper.readValue(path.toFile(), Dto.class);\n if (value.schemaVersion() != SCHEMA_VERSION) {\n throw new IllegalStateException(\"unsupported prepared day metrics schema: \" + value.schemaVersion());\n }\n GateBSourceDayRunner.PreparedDayMetrics metrics = new GateBSourceDayRunner.PreparedDayMetrics(\n UUID.fromString(value.datasetId()),\n LocalDate.parse(value.sourceDate()),\n DatasetWindow.DateRole.valueOf(value.role()),\n value.manifestSha256(),\n value.dayManifestSha256(),\n value.semanticFingerprint(),\n value.coverageVersion(),\n value.coverageHash(),\n LocalDayCommitStore.EvidenceQualityStatus.valueOf(value.quality()),\n value.reasonCode(),\n value.transferBytes(),\n value.verifiedAssetCount(),\n value.tarEntryCount(),\n value.traceEntryCount(),\n value.parsedTraceCount(),\n value.traceParseFailureCount(),\n value.malformedPointCount(),\n value.acceptedPointCount(),\n value.rejectedPointCount(),\n value.finalizedSegmentCount(),\n value.outOfOrderRejectedCount(),\n value.excludedSegmentCount(),\n value.outgoingCarryCount(),\n value.dayDeltaChecksum(),\n Instant.parse(value.preparedAt())\n );\n if (!metrics.sourceDate().equals(sourceDate)) {\n throw new IllegalStateException(\"prepared metrics source date does not match filename\");\n }\n return metrics;\n }\n\n GateBSourceDayRunner.PreparedDayMetrics requireMatches(\n LocalDate sourceDate,\n UUID datasetId,\n String manifestSha256,\n String dayManifestSha256,\n String semanticFingerprint,\n String coverageVersion,\n String coverageHash\n ) throws Exception {\n GateBSourceDayRunner.PreparedDayMetrics metrics = read(sourceDate);\n if (!metrics.datasetId().equals(datasetId)\n || !metrics.manifestSha256().equals(manifestSha256)\n || !metrics.dayManifestSha256().equals(dayManifestSha256)\n || !metrics.semanticFingerprint().equals(semanticFingerprint)\n || !metrics.coverageVersion().equals(coverageVersion)\n || !metrics.coverageHash().equals(coverageHash)) {\n throw new IllegalStateException(\"prepared day metrics immutable identity mismatch at \" + sourceDate);\n }\n return metrics;\n }\n\n private static Map toJson(GateBSourceDayRunner.PreparedDayMetrics value) {\n Map map = new LinkedHashMap<>();\n map.put(\"schemaVersion\", SCHEMA_VERSION);\n map.put(\"datasetId\", value.datasetId().toString());\n map.put(\"sourceDate\", value.sourceDate().toString());\n map.put(\"role\", value.role().name());\n map.put(\"manifestSha256\", value.manifestSha256());\n map.put(\"dayManifestSha256\", value.dayManifestSha256());\n map.put(\"semanticFingerprint\", value.semanticFingerprint());\n map.put(\"coverageVersion\", value.coverageVersion());\n map.put(\"coverageHash\", value.coverageHash());\n map.put(\"quality\", value.quality().name());\n map.put(\"reasonCode\", value.reasonCode());\n map.put(\"transferBytes\", value.transferBytes());\n map.put(\"verifiedAssetCount\", value.verifiedAssetCount());\n map.put(\"tarEntryCount\", value.tarEntryCount());\n map.put(\"traceEntryCount\", value.traceEntryCount());\n map.put(\"parsedTraceCount\", value.parsedTraceCount());\n map.put(\"traceParseFailureCount\", value.traceParseFailureCount());\n map.put(\"malformedPointCount\", value.malformedPointCount());\n map.put(\"acceptedPointCount\", value.acceptedPointCount());\n map.put(\"rejectedPointCount\", value.rejectedPointCount());\n map.put(\"finalizedSegmentCount\", value.finalizedSegmentCount());\n map.put(\"outOfOrderRejectedCount\", value.outOfOrderRejectedCount());\n map.put(\"excludedSegmentCount\", value.excludedSegmentCount());\n map.put(\"outgoingCarryCount\", value.outgoingCarryCount());\n map.put(\"dayDeltaChecksum\", value.dayDeltaChecksum());\n map.put(\"preparedAt\", value.preparedAt().toString());\n return map;\n }\n\n private record Dto(\n int schemaVersion,\n String datasetId,\n String sourceDate,\n String role,\n String manifestSha256,\n String dayManifestSha256,\n String semanticFingerprint,\n String coverageVersion,\n String coverageHash,\n String quality,\n String reasonCode,\n long transferBytes,\n int verifiedAssetCount,\n int tarEntryCount,\n int traceEntryCount,\n int parsedTraceCount,\n int traceParseFailureCount,\n long malformedPointCount,\n long acceptedPointCount,\n long rejectedPointCount,\n long finalizedSegmentCount,\n long outOfOrderRejectedCount,\n int excludedSegmentCount,\n int outgoingCarryCount,\n String dayDeltaChecksum,\n String preparedAt\n ) {\n }\n}\n" }, { "path": "backend/historical-bootstrap-tool/src/main/java/hu/gombos/flightspotter/historicalbootstrap/GitHubReleaseDiscoveryClient.java", "encoding": "UTF-8", "sha256": "10ab2419f848e45c9089314726b3962cf25d121d8565cfab34fc569c1c50bbb9", "content": "package hu.gombos.flightspotter.historicalbootstrap;\n\nimport com.fasterxml.jackson.annotation.JsonProperty;\nimport tools.jackson.databind.DeserializationFeature;\nimport tools.jackson.databind.ObjectMapper;\nimport tools.jackson.databind.json.JsonMapper;\n\nimport java.io.IOException;\nimport java.net.URI;\nimport java.net.http.HttpClient;\nimport java.net.http.HttpRequest;\nimport java.net.http.HttpResponse;\nimport java.time.Duration;\nimport java.time.LocalDate;\nimport java.time.format.DateTimeFormatter;\nimport java.time.format.DateTimeParseException;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.Locale;\nimport java.util.Optional;\nimport java.util.regex.Matcher;\nimport java.util.regex.Pattern;\n\nfinal class GitHubReleaseDiscoveryClient {\n private static final int PAGE_SIZE = 100;\n private static final int MAX_PAGES_PER_YEAR = 20;\n private static final Pattern TAG_DATE = Pattern.compile(\"^v?(\\\\d{4}\\\\.\\\\d{2}\\\\.\\\\d{2})-.*$\");\n private static final DateTimeFormatter TAG_DATE_FORMAT = DateTimeFormatter.ofPattern(\"yyyy.MM.dd\", Locale.ROOT);\n\n private final HttpClient httpClient;\n private final ObjectMapper mapper;\n private final String token;\n private final URI apiBase;\n\n GitHubReleaseDiscoveryClient(String token) {\n this(HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(15)).build(), token,\n URI.create(\"https://api.github.com\"));\n }\n\n GitHubReleaseDiscoveryClient(HttpClient httpClient, String token, URI apiBase) {\n this.httpClient = httpClient;\n this.token = token == null || token.isBlank() ? null : token.trim();\n this.apiBase = apiBase;\n this.mapper = JsonMapper.builder()\n .disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES)\n .build();\n }\n\n List discover(DatasetWindow window) throws Exception {\n List releases = new ArrayList<>();\n for (int year = window.fromDate().getYear(); year <= window.toDate().getYear(); year++) {\n String repository = \"adsblol/globe_history_\" + year;\n for (int page = 1; page <= MAX_PAGES_PER_YEAR; page++) {\n URI uri = URI.create(apiBase + \"/repos/\" + repository + \"/releases?per_page=\"\n + PAGE_SIZE + \"&page=\" + page);\n String body = get(uri);\n List parsed = parsePage(repository, body, window);\n releases.addAll(parsed);\n ReleaseDto[] raw = mapper.readValue(body, ReleaseDto[].class);\n if (raw.length < PAGE_SIZE) break;\n if (page == MAX_PAGES_PER_YEAR) {\n throw new IOException(\"release pagination exceeded hard page bound for \" + repository);\n }\n }\n }\n return releases.stream()\n .sorted(java.util.Comparator.comparing(SourceRelease::sourceDate)\n .thenComparingLong(SourceRelease::releaseId))\n .toList();\n }\n\n List parsePage(String repository, String json, DatasetWindow window) throws Exception {\n ReleaseDto[] values = mapper.readValue(json, ReleaseDto[].class);\n List result = new ArrayList<>();\n for (ReleaseDto value : values) {\n Optional sourceDate = sourceDate(value.tagName());\n if (sourceDate.isEmpty()\n || sourceDate.get().isBefore(window.fromDate())\n || sourceDate.get().isAfter(window.toDate())) {\n continue;\n }\n List assets = value.assets() == null ? List.of() : value.assets().stream()\n .map(asset -> new SourceAsset(asset.id(), asset.name(), asset.size(), asset.digest()))\n .toList();\n result.add(new SourceRelease(\n repository,\n value.id(),\n value.tagName(),\n sourceDate.get(),\n value.draft(),\n value.prerelease(),\n ReleaseVariant.fromTag(value.tagName()),\n assets\n ));\n }\n return List.copyOf(result);\n }\n\n private String get(URI uri) throws IOException, InterruptedException {\n IOException lastFailure = null;\n for (int attempt = 1; attempt <= 3; attempt++) {\n HttpRequest.Builder builder = HttpRequest.newBuilder(uri)\n .timeout(Duration.ofSeconds(30))\n .header(\"Accept\", \"application/vnd.github+json\")\n .header(\"X-GitHub-Api-Version\", \"2022-11-28\")\n .header(\"User-Agent\", \"FlightTrove-P7-Historical-Bootstrap\")\n .GET();\n if (token != null) builder.header(\"Authorization\", \"Bearer \" + token);\n\n HttpResponse response = httpClient.send(builder.build(), HttpResponse.BodyHandlers.ofString());\n if (response.statusCode() == 200) return response.body();\n String rateRemaining = response.headers().firstValue(\"X-RateLimit-Remaining\").orElse(\"unknown\");\n String message = \"GitHub release discovery failed: status=\" + response.statusCode()\n + \" uri=\" + uri + \" rateRemaining=\" + rateRemaining;\n if (response.statusCode() >= 500 && attempt < 3) {\n lastFailure = new IOException(message);\n Thread.sleep(250L * attempt);\n continue;\n }\n throw new IOException(message);\n }\n throw lastFailure == null ? new IOException(\"GitHub release discovery failed\") : lastFailure;\n }\n\n private Optional sourceDate(String tag) {\n if (tag == null) return Optional.empty();\n Matcher matcher = TAG_DATE.matcher(tag);\n if (!matcher.matches()) return Optional.empty();\n try {\n return Optional.of(LocalDate.parse(matcher.group(1), TAG_DATE_FORMAT));\n } catch (DateTimeParseException ignored) {\n return Optional.empty();\n }\n }\n\n private record ReleaseDto(\n long id,\n @JsonProperty(\"tag_name\") String tagName,\n boolean draft,\n boolean prerelease,\n List assets\n ) {\n }\n\n private record AssetDto(long id, String name, long size, String digest) {\n }\n}\n" }, { "path": "backend/historical-bootstrap-tool/src/main/java/hu/gombos/flightspotter/historicalbootstrap/HistoricalBootstrapGateBMain.java", "encoding": "UTF-8", "sha256": "418c149eb108cf33d422b6bfac34a249a84f61d7fe822c132d6e7d26715e07a7", "content": "package hu.gombos.flightspotter.historicalbootstrap;\n\nimport hu.gombos.flightspotter.common.geo.H3Service;\nimport hu.gombos.flightspotter.common.geo.UberH3Service;\n\nimport java.nio.charset.StandardCharsets;\nimport java.nio.file.Files;\nimport java.nio.file.Path;\nimport java.time.Duration;\nimport java.time.LocalDate;\nimport java.util.HashMap;\nimport java.util.LinkedHashSet;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.UUID;\n\n/** Explicit, owner-invoked CLI for a single bounded real-source Gate B qualification day. */\npublic final class HistoricalBootstrapGateBMain {\n private HistoricalBootstrapGateBMain() {\n }\n\n public static void main(String[] rawArgs) throws Exception {\n CliArguments args = CliArguments.parse(rawArgs);\n BootstrapSourceManifest manifest = BootstrapSourceManifestIO.read(args.manifestFile());\n BootstrapCoverageSpec coverage = coverage(args, new UberH3Service());\n GateBContentPolicy contentPolicy = new GateBContentPolicy(\n args.maximumTraceParseFailures(), args.minimumAcceptedPoints());\n GateBExecutionBudget budget = new GateBExecutionBudget(\n args.maximumTransferBytes(),\n args.scratchReserveBytes(),\n args.maximumAssetAttempts(),\n contentPolicy\n );\n\n UUID datasetId = args.datasetId() == null\n ? deterministicDatasetId(manifest, args.sourceDate(), coverage, contentPolicy)\n : args.datasetId();\n GateBExecutionPlan plan = GateBExecutionPlan.create(\n manifest,\n args.sourceDate(),\n datasetId,\n coverage,\n args.scratchRoot(),\n args.stateDatabase(),\n budget\n );\n requireRepresentativeRole(plan.role());\n\n System.out.println(\"P7 Gate B one-day execution authorized by explicit CLI invocation\");\n System.out.println(\"datasetId=\" + datasetId);\n System.out.println(\"sourceDate=\" + plan.sourceDate());\n System.out.println(\"selectedRelease=\" + plan.selectedRelease().releaseId() + \" \" + plan.selectedRelease().tag());\n System.out.println(\"selectedSourceBytes=\" + plan.selectedSourceBytes());\n System.out.println(\"maximumTransferBytes=\" + budget.maximumTransferBytes());\n System.out.println(\"scratchRoot=\" + plan.scratchRoot());\n System.out.println(\"stateDatabase=\" + plan.stateDatabase());\n System.out.println(\"coverage=\" + coverage.version() + \" cells=\" + coverage.allowedH3R5().size()\n + \" hash=\" + coverage.hash());\n System.out.println(\"semanticFingerprint=\" + plan.semanticFingerprint());\n\n GateBSourceDayRunner runner = new GateBSourceDayRunner(\n ArchiveSafetyLimits.defaults(),\n new GitHubReleaseAssetSource(Duration.ofMinutes(args.requestTimeoutMinutes()))\n );\n GateBQualificationRecord record = runner.run(plan, coverage);\n new GateBQualificationRecordWriter().write(args.reportFile(), record);\n System.out.println(\"P7 Gate B one-day execution complete\");\n System.out.println(\"acceptedPointCount=\" + record.acceptedPointCount());\n System.out.println(\"patternCount=\" + record.patternCount());\n System.out.println(\"rawScratchBytesAfterCleanup=\" + record.rawScratchBytesAfterCleanup());\n System.out.println(\"report=\" + args.reportFile().toAbsolutePath().normalize());\n }\n\n static void requireRepresentativeRole(DatasetWindow.DateRole role) {\n if (role == DatasetWindow.DateRole.EMBARGO || role == DatasetWindow.DateRole.HOLDOUT) {\n throw new IllegalArgumentException(\"Gate B representative day must be LEARNING or SCORING\");\n }\n }\n\n static BootstrapCoverageSpec coverage(CliArguments args, H3Service h3) throws Exception {\n if (args.coverageCellsFile() != null) {\n Set cells = new LinkedHashSet<>();\n for (String raw : Files.readAllLines(args.coverageCellsFile(), StandardCharsets.UTF_8)) {\n String line = raw.trim().toLowerCase(java.util.Locale.ROOT);\n if (line.isEmpty() || line.startsWith(\"#\")) continue;\n if (!line.matches(\"85[0-9a-f]{13}\")) {\n throw new IllegalArgumentException(\"coverage file contains non-R5 H3 cell: \" + raw);\n }\n cells.add(line);\n }\n return new BootstrapCoverageSpec(args.coverageVersion(), cells, h3);\n }\n String centerCell = h3.latLonToCell(args.coverageCenterLat(), args.coverageCenterLon(), 5);\n return new BootstrapCoverageSpec(\n args.coverageVersion(), h3.gridDisk(centerCell, args.coverageGridK()), h3);\n }\n\n static UUID deterministicDatasetId(\n BootstrapSourceManifest manifest,\n LocalDate sourceDate,\n BootstrapCoverageSpec coverage,\n GateBContentPolicy contentPolicy\n ) {\n String fingerprint = GateBSemanticFingerprint.hash(manifest, coverage, contentPolicy);\n String canonical = \"p7-gate-b-one-day-v1|\" + sourceDate + \"|\" + fingerprint;\n return UUID.nameUUIDFromBytes(canonical.getBytes(StandardCharsets.UTF_8));\n }\n\n record CliArguments(\n Path manifestFile,\n LocalDate sourceDate,\n String coverageVersion,\n Path coverageCellsFile,\n Double coverageCenterLat,\n Double coverageCenterLon,\n Integer coverageGridK,\n Path scratchRoot,\n Path stateDatabase,\n Path reportFile,\n long maximumTransferBytes,\n long scratchReserveBytes,\n int maximumAssetAttempts,\n int maximumTraceParseFailures,\n long minimumAcceptedPoints,\n long requestTimeoutMinutes,\n UUID datasetId\n ) {\n private static final Set KNOWN = Set.of(\n \"--manifest\", \"--source-date\", \"--coverage-version\", \"--coverage-cells\",\n \"--coverage-center-lat\", \"--coverage-center-lon\", \"--coverage-grid-k\",\n \"--scratch\", \"--state-db\", \"--report\", \"--max-transfer-bytes\", \"--scratch-reserve-bytes\",\n \"--max-asset-attempts\", \"--max-trace-parse-failures\", \"--min-accepted-points\",\n \"--request-timeout-minutes\", \"--dataset-id\"\n );\n\n CliArguments {\n if (manifestFile == null || sourceDate == null || coverageVersion == null || coverageVersion.isBlank()\n || scratchRoot == null || stateDatabase == null || reportFile == null\n || maximumTransferBytes < 1L || scratchReserveBytes < 0L\n || maximumAssetAttempts < 1 || maximumAssetAttempts > 3\n || maximumTraceParseFailures < 0 || minimumAcceptedPoints < 1L\n || requestTimeoutMinutes < 1L || requestTimeoutMinutes > 240L) {\n throw new IllegalArgumentException(\"invalid Gate B CLI arguments\");\n }\n boolean fileMode = coverageCellsFile != null;\n boolean diskMode = coverageCenterLat != null || coverageCenterLon != null || coverageGridK != null;\n if (fileMode == diskMode) {\n throw new IllegalArgumentException(\"choose exactly one coverage mode: --coverage-cells OR center+grid-k\");\n }\n if (diskMode) {\n if (coverageCenterLat == null || coverageCenterLon == null || coverageGridK == null\n || !Double.isFinite(coverageCenterLat) || coverageCenterLat < -90d || coverageCenterLat > 90d\n || !Double.isFinite(coverageCenterLon) || coverageCenterLon < -180d || coverageCenterLon > 180d\n || coverageGridK < 0 || coverageGridK > 100) {\n throw new IllegalArgumentException(\"invalid deterministic H3 disk coverage\");\n }\n }\n }\n\n static CliArguments parse(String[] args) {\n Map values = new HashMap<>();\n for (int index = 0; index < args.length; index++) {\n String key = args[index];\n if (!key.startsWith(\"--\") || index + 1 >= args.length || !KNOWN.contains(key)) {\n throw usage(\"invalid or missing option value near: \" + key);\n }\n if (values.put(key, args[++index]) != null) throw usage(\"duplicate option: \" + key);\n }\n\n Path cells = path(values.get(\"--coverage-cells\"));\n Double lat = decimal(values.get(\"--coverage-center-lat\"));\n Double lon = decimal(values.get(\"--coverage-center-lon\"));\n Integer gridK = integer(values.get(\"--coverage-grid-k\"));\n return new CliArguments(\n Path.of(required(values, \"--manifest\")),\n LocalDate.parse(required(values, \"--source-date\")),\n required(values, \"--coverage-version\"),\n cells,\n lat,\n lon,\n gridK,\n Path.of(required(values, \"--scratch\")),\n Path.of(required(values, \"--state-db\")),\n Path.of(required(values, \"--report\")),\n Long.parseLong(required(values, \"--max-transfer-bytes\")),\n Long.parseLong(required(values, \"--scratch-reserve-bytes\")),\n Integer.parseInt(values.getOrDefault(\"--max-asset-attempts\", \"1\")),\n Integer.parseInt(values.getOrDefault(\"--max-trace-parse-failures\", \"0\")),\n Long.parseLong(values.getOrDefault(\"--min-accepted-points\", \"1\")),\n Long.parseLong(values.getOrDefault(\"--request-timeout-minutes\", \"120\")),\n values.containsKey(\"--dataset-id\") ? UUID.fromString(values.get(\"--dataset-id\")) : null\n );\n }\n\n private static String required(Map values, String key) {\n String value = values.get(key);\n if (value == null || value.isBlank()) throw usage(\"missing required option \" + key);\n return value;\n }\n\n private static Path path(String value) { return value == null ? null : Path.of(value); }\n private static Double decimal(String value) { return value == null ? null : Double.parseDouble(value); }\n private static Integer integer(String value) { return value == null ? null : Integer.parseInt(value); }\n\n private static IllegalArgumentException usage(String message) {\n return new IllegalArgumentException(message + \"\\n\"\n + \"required: --manifest FILE --source-date YYYY-MM-DD --coverage-version VERSION \"\n + \"(--coverage-cells FILE | --coverage-center-lat LAT --coverage-center-lon LON --coverage-grid-k K) \"\n + \"--scratch DIR --state-db PATH --report FILE --max-transfer-bytes N --scratch-reserve-bytes N\\n\"\n + \"optional: --max-asset-attempts 1..3 --max-trace-parse-failures N --min-accepted-points N \"\n + \"--request-timeout-minutes 1..240 --dataset-id UUID\");\n }\n }\n}\n" }, { "path": "backend/historical-bootstrap-tool/src/main/java/hu/gombos/flightspotter/historicalbootstrap/HistoricalBootstrapPostgresPersistence.java", "encoding": "UTF-8", "sha256": "ea7898bc443c758af218d44e9b25b8b1bfd87dcd51940f6d363f5032feaadd2d", "content": "package hu.gombos.flightspotter.historicalbootstrap;\n\nimport javax.sql.DataSource;\nimport java.nio.ByteBuffer;\nimport java.nio.charset.StandardCharsets;\nimport java.nio.file.Path;\nimport java.security.MessageDigest;\nimport java.security.NoSuchAlgorithmException;\nimport java.sql.Connection;\nimport java.sql.Date;\nimport java.sql.DriverManager;\nimport java.sql.PreparedStatement;\nimport java.sql.ResultSet;\nimport java.sql.SQLException;\nimport java.sql.Timestamp;\nimport java.time.Instant;\nimport java.time.LocalDate;\nimport java.util.ArrayList;\nimport java.util.Comparator;\nimport java.util.HexFormat;\nimport java.util.LinkedHashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Objects;\nimport java.util.TreeMap;\nimport java.util.UUID;\n\n/** P7 bridge from qualified compact H2 state into immutable PostgreSQL bootstrap snapshots. */\nfinal class HistoricalBootstrapPostgresPersistence {\n static final String IMPORTER_VERSION = \"p7-postgres-loader-v2-packed-corridors\";\n static final String LICENSE_IDENTIFIER = \"ODBL-1.0\";\n\n private HistoricalBootstrapPostgresPersistence() {\n }\n\n static ExportedSnapshot exportLocalSnapshot(Path stateDatabase, UUID datasetId) throws Exception {\n Objects.requireNonNull(stateDatabase, \"stateDatabase\");\n Objects.requireNonNull(datasetId, \"datasetId\");\n Path absolute = stateDatabase.toAbsolutePath().normalize();\n try (Connection connection = DriverManager.getConnection(\n \"jdbc:h2:file:\" + absolute + \";AUTO_SERVER=FALSE;DB_CLOSE_ON_EXIT=FALSE\")) {\n String dataset = datasetId.toString();\n List days = readDays(connection, dataset);\n TreeMap patterns = readPatterns(connection, dataset);\n TreeMap hourBins = readHourBins(connection, dataset);\n TreeMap corridors = readCorridors(connection, dataset);\n List holdout = readHoldout(connection, dataset);\n return new ExportedSnapshot(datasetId, days, patterns, hourBins, corridors, holdout);\n }\n }\n\n static LoadResult load(DataSource dataSource, LoadRequest request) throws Exception {\n Objects.requireNonNull(dataSource, \"dataSource\");\n validate(request);\n try (Connection connection = dataSource.getConnection()) {\n boolean originalAutoCommit = connection.getAutoCommit();\n connection.setAutoCommit(false);\n try {\n rejectExistingDataset(connection, request.datasetId());\n insertDataset(connection, request);\n transition(connection, request.datasetId(), \"MANIFEST_LOCKED\", null);\n transition(connection, request.datasetId(), \"DERIVING\", null);\n insertDaysAndAssets(connection, request);\n Map patternIds = insertPatterns(connection, request);\n insertHourBins(connection, request, patternIds);\n insertCorridors(connection, request);\n insertHoldout(connection, request);\n transition(connection, request.datasetId(), \"STAGED\", null);\n validateStaged(connection, request);\n transition(connection, request.datasetId(), \"READY\", request.loadedAt());\n validateReady(connection, request);\n connection.commit();\n return new LoadResult(\n request.datasetId(),\n request.snapshot().days().size(),\n request.snapshot().patterns().size(),\n request.snapshot().hourBins().size(),\n request.snapshot().corridorAggregates().size(),\n request.snapshot().holdout().size(),\n request.loadedAt());\n } catch (Exception exception) {\n connection.rollback();\n throw exception;\n } finally {\n connection.setAutoCommit(originalAutoCommit);\n }\n }\n }\n\n private static List readDays(Connection connection, String dataset) throws SQLException {\n List days = new ArrayList<>();\n try (PreparedStatement statement = connection.prepareStatement(\"\"\"\n select source_date,day_manifest_hash,semantic_fingerprint,delta_checksum,evidence_quality\n from p7_local_committed_days\n where dataset_id=? order by source_date\n \"\"\")) {\n statement.setString(1, dataset);\n try (ResultSet rs = statement.executeQuery()) {\n while (rs.next()) {\n days.add(new ExportedDay(\n rs.getDate(1).toLocalDate(), rs.getString(2), rs.getString(3), rs.getString(4),\n LocalDayCommitStore.EvidenceQualityStatus.valueOf(rs.getString(5))));\n }\n }\n }\n return List.copyOf(days);\n }\n\n private static TreeMap readPatterns(\n Connection connection,\n String dataset\n ) throws SQLException {\n TreeMap patterns = new TreeMap<>();\n try (PreparedStatement statement = connection.prepareStatement(\"\"\"\n select pattern_kind,subject_key,start_h3_r5,end_h3_r5,evidence_count,clean_evidence_count,\n first_occurrence_ms,last_occurrence_ms,minimum_altitude_ft,maximum_altitude_ft,\n mean_altitude_sum_ft,altitude_evidence_count,duration_seconds_sum,observation_count_sum,source_counts\n from p7_local_patterns where dataset_id=?\n order by pattern_kind,subject_key,start_h3_r5,end_h3_r5\n \"\"\")) {\n statement.setString(1, dataset);\n try (ResultSet rs = statement.executeQuery()) {\n while (rs.next()) {\n HistoricalPatternKey key = new HistoricalPatternKey(\n HistoricalPatternKind.valueOf(rs.getString(1)), rs.getString(2), rs.getString(3), rs.getString(4));\n patterns.put(key, new HistoricalPatternStats(\n rs.getLong(5), rs.getLong(6), Instant.ofEpochMilli(rs.getLong(7)), Instant.ofEpochMilli(rs.getLong(8)),\n rs.getInt(9), rs.getInt(10), rs.getLong(11), rs.getLong(12), rs.getLong(13), rs.getLong(14),\n decodeSourceCounts(rs.getString(15))));\n }\n }\n }\n return patterns;\n }\n\n private static TreeMap readHourBins(Connection connection, String dataset)\n throws SQLException {\n TreeMap hourBins = new TreeMap<>();\n try (PreparedStatement statement = connection.prepareStatement(\"\"\"\n select pattern_kind,subject_key,start_h3_r5,end_h3_r5,hour_of_week,evidence_count\n from p7_local_hour_bins where dataset_id=?\n order by pattern_kind,subject_key,start_h3_r5,end_h3_r5,hour_of_week\n \"\"\")) {\n statement.setString(1, dataset);\n try (ResultSet rs = statement.executeQuery()) {\n while (rs.next()) {\n HistoricalPatternKey key = new HistoricalPatternKey(\n HistoricalPatternKind.valueOf(rs.getString(1)), rs.getString(2), rs.getString(3), rs.getString(4));\n hourBins.put(new HistoricalPatternHourKey(key, rs.getShort(5)), rs.getLong(6));\n }\n }\n }\n return hourBins;\n }\n\n private static TreeMap readCorridors(\n Connection connection,\n String dataset\n ) throws SQLException {\n TreeMap corridors = new TreeMap<>();\n try (PreparedStatement statement = connection.prepareStatement(\"\"\"\n select start_h3_r5,end_h3_r5,corridor_h3_r7,traversal_count,progress_permille_sum,\n minimum_altitude_ft,maximum_altitude_ft,altitude_sum_ft,altitude_evidence_count\n from p7_local_corridor_aggregates where dataset_id=?\n order by start_h3_r5,end_h3_r5,corridor_h3_r7\n \"\"\")) {\n statement.setString(1, dataset);\n try (ResultSet rs = statement.executeQuery()) {\n while (rs.next()) {\n HistoricalRouteCorridorKey key = new HistoricalRouteCorridorKey(\n rs.getString(1), rs.getString(2), rs.getString(3));\n corridors.put(key, new HistoricalRouteCorridorStats(\n rs.getLong(4), rs.getLong(5), rs.getInt(6), rs.getInt(7), rs.getLong(8), rs.getLong(9)));\n }\n }\n }\n return corridors;\n }\n\n private static List readHoldout(Connection connection, String dataset) throws SQLException {\n List holdout = new ArrayList<>();\n try (PreparedStatement statement = connection.prepareStatement(\"\"\"\n select source_date,segment_id,icao24,callsign,first_observed_ms,last_observed_ms,start_h3_r5,end_h3_r5,\n minimum_altitude_ft,maximum_altitude_ft,mean_altitude_ft,altitude_evidence_count,\n duration_seconds,observation_count,source_counts,corridor_overflow\n from p7_local_holdout where dataset_id=? order by source_date,segment_id\n \"\"\")) {\n statement.setString(1, dataset);\n try (ResultSet rs = statement.executeQuery()) {\n while (rs.next()) {\n String segmentId = rs.getString(2);\n boolean overflow = rs.getBoolean(16);\n HistoricalHoldoutOutcome outcome = new HistoricalHoldoutOutcome(\n segmentId, rs.getString(3), rs.getString(4), Instant.ofEpochMilli(rs.getLong(5)),\n Instant.ofEpochMilli(rs.getLong(6)), rs.getString(7), rs.getString(8), rs.getInt(9),\n rs.getInt(10), rs.getInt(11), rs.getLong(12), rs.getLong(13), rs.getLong(14),\n decodeSourceCounts(rs.getString(15)),\n readHoldoutCorridor(connection, dataset, segmentId), overflow);\n holdout.add(new ExportedHoldout(rs.getDate(1).toLocalDate(), outcome));\n }\n }\n }\n return List.copyOf(holdout);\n }\n\n private static List readHoldoutCorridor(\n Connection connection,\n String dataset,\n String segmentId\n ) throws SQLException {\n List cells = new ArrayList<>();\n try (PreparedStatement statement = connection.prepareStatement(\"\"\"\n select sequence_index,corridor_h3_r7,progress_permille,minimum_altitude_ft,maximum_altitude_ft,\n altitude_sum_ft,altitude_evidence_count\n from p7_local_holdout_corridor\n where dataset_id=? and segment_id=? order by sequence_index\n \"\"\")) {\n statement.setString(1, dataset);\n statement.setString(2, segmentId);\n try (ResultSet rs = statement.executeQuery()) {\n while (rs.next()) {\n cells.add(new HistoricalCorridorCellEvidence(\n rs.getInt(1), rs.getString(2), rs.getInt(3), rs.getInt(4), rs.getInt(5),\n rs.getLong(6), rs.getLong(7)));\n }\n }\n }\n return List.copyOf(cells);\n }\n\n private static void validate(LoadRequest request) {\n Objects.requireNonNull(request, \"request\");\n Objects.requireNonNull(request.datasetId(), \"datasetId\");\n Objects.requireNonNull(request.manifest(), \"manifest\");\n Objects.requireNonNull(request.snapshot(), \"snapshot\");\n Objects.requireNonNull(request.loadedAt(), \"loadedAt\");\n requireHash(request.semanticFingerprint(), \"semanticFingerprint\");\n requireText(request.coverageVersion(), 80, \"coverageVersion\");\n requireHash(request.coverageHash(), \"coverageHash\");\n requireText(request.derivationVersion(), 80, \"derivationVersion\");\n if (!request.snapshot().datasetId().equals(request.datasetId())) {\n throw new IllegalArgumentException(\"local snapshot dataset ID does not match load request\");\n }\n if (request.snapshot().days().size() != request.manifest().days().size()) {\n throw new IllegalArgumentException(\"local committed-day count does not match immutable manifest\");\n }\n if (request.dayMetrics() == null || request.dayMetrics().size() != request.manifest().days().size()) {\n throw new IllegalArgumentException(\"one terminal metrics row is required per manifest day\");\n }\n Map exportedDays = new TreeMap<>();\n for (ExportedDay day : request.snapshot().days()) {\n if (exportedDays.put(day.sourceDate(), day) != null) {\n throw new IllegalArgumentException(\"duplicate local committed source day\");\n }\n requireHash(day.dayManifestSha256(), \"dayManifestSha256\");\n requireHash(day.semanticFingerprint(), \"day semanticFingerprint\");\n requireHash(day.dayDeltaChecksum(), \"dayDeltaChecksum\");\n if (!day.semanticFingerprint().equals(request.semanticFingerprint())) {\n throw new IllegalArgumentException(\"local day semantic fingerprint differs from dataset generation\");\n }\n }\n for (ResolvedSourceDay day : request.manifest().days()) {\n ExportedDay local = exportedDays.get(day.sourceDate());\n if (local == null || !local.dayManifestSha256().equals(request.manifest().dayManifestSha256(day.sourceDate()))) {\n throw new IllegalArgumentException(\"local committed day does not match immutable manifest at \" + day.sourceDate());\n }\n DayLoadMetrics metrics = request.dayMetrics().get(day.sourceDate());\n if (metrics == null || !metrics.sourceDate().equals(day.sourceDate())) {\n throw new IllegalArgumentException(\"missing terminal day metrics at \" + day.sourceDate());\n }\n if (local.quality() == LocalDayCommitStore.EvidenceQualityStatus.QUALIFIED && metrics.reasonCode() != null) {\n throw new IllegalArgumentException(\"QUALIFIED day cannot carry a terminal failure reason\");\n }\n if (local.quality() != LocalDayCommitStore.EvidenceQualityStatus.QUALIFIED && metrics.reasonCode() == null) {\n throw new IllegalArgumentException(\"DEGRADED/GAP day requires a finite terminal reason\");\n }\n }\n for (Map.Entry entry\n : request.snapshot().corridorAggregates().entrySet()) {\n HistoricalRouteCorridorKey corridor = entry.getKey();\n HistoricalPatternKey route = new HistoricalPatternKey(\n HistoricalPatternKind.ROUTE, \"-\", corridor.startH3R5(), corridor.endH3R5());\n HistoricalPatternStats routeStats = request.snapshot().patterns().get(route);\n if (routeStats == null) {\n throw new IllegalArgumentException(\"corridor aggregate is missing corresponding coarse ROUTE evidence\");\n }\n if (entry.getValue().traversalCount() > routeStats.evidenceCount()) {\n throw new IllegalArgumentException(\"corridor traversal count exceeds corresponding ROUTE evidence\");\n }\n }\n if (!request.snapshot().holdout().isEmpty()) {\n Objects.requireNonNull(request.holdoutDeleteAfter(), \"holdoutDeleteAfter is required when holdout rows exist\");\n for (ExportedHoldout row : request.snapshot().holdout()) {\n if (!request.holdoutDeleteAfter().isAfter(row.outcome().lastObservedAt())) {\n throw new IllegalArgumentException(\"holdoutDeleteAfter must be after every holdout observation\");\n }\n if (request.manifest().window().roleFor(row.sourceDate()) != DatasetWindow.DateRole.HOLDOUT) {\n throw new IllegalArgumentException(\"local holdout row is not attached to a HOLDOUT source day\");\n }\n }\n }\n if (request.scoringEligibleFrom() != null && request.scoringExpiresAt() != null\n && !request.scoringExpiresAt().isAfter(request.scoringEligibleFrom())) {\n throw new IllegalArgumentException(\"scoring expiry must be after eligibility start\");\n }\n sourceRepository(request.manifest());\n sourceYear(request.manifest());\n }\n\n private static void rejectExistingDataset(Connection connection, UUID datasetId) throws SQLException {\n try (PreparedStatement statement = connection.prepareStatement(\n \"select state from historical_bootstrap_datasets where dataset_id=?\")) {\n statement.setObject(1, datasetId);\n try (ResultSet rs = statement.executeQuery()) {\n if (rs.next()) {\n throw new IllegalStateException(\"historical bootstrap dataset ID already exists and is immutable: \"\n + datasetId + \" state=\" + rs.getString(1));\n }\n }\n }\n }\n\n private static void insertDataset(Connection connection, LoadRequest request) throws SQLException {\n DatasetWindow window = request.manifest().window();\n try (PreparedStatement statement = connection.prepareStatement(\"\"\"\n insert into historical_bootstrap_datasets(\n dataset_id,dataset_purpose,provider,source_repository,source_year,license_identifier,\n from_date,to_date,as_of_date,lookback_days,learning_end,embargo_date,holdout_start,\n importer_version,parser_version,archive_safety_profile,eligibility_version,segmentation_version,\n derivation_version,semantic_fingerprint,coverage_version,coverage_hash,manifest_sha256,state,\n scoring_eligible_from,scoring_expires_at,created_at)\n values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)\n \"\"\")) {\n int index = 1;\n statement.setObject(index++, request.datasetId());\n statement.setString(index++, window.purpose().name());\n statement.setString(index++, request.manifest().provider());\n statement.setString(index++, sourceRepository(request.manifest()));\n statement.setShort(index++, (short) sourceYear(request.manifest()));\n statement.setString(index++, LICENSE_IDENTIFIER);\n statement.setDate(index++, Date.valueOf(window.fromDate()));\n statement.setDate(index++, Date.valueOf(window.toDate()));\n setDate(statement, index++, window.asOfDate());\n if (window.lookbackDays() == null) statement.setObject(index++, null); else statement.setInt(index++, window.lookbackDays());\n setDate(statement, index++, window.learningEnd());\n setDate(statement, index++, window.embargoDate());\n setDate(statement, index++, window.holdoutStart());\n statement.setString(index++, IMPORTER_VERSION);\n statement.setString(index++, GateBSemanticFingerprint.PARSER_VERSION);\n statement.setString(index++, ArchiveSafetyLimits.defaults().semanticFingerprintComponent());\n statement.setString(index++, HistoricalEligibilityV1.VERSION);\n statement.setString(index++, HistoricalSegmenterV1.VERSION);\n statement.setString(index++, request.derivationVersion());\n statement.setString(index++, request.semanticFingerprint());\n statement.setString(index++, request.coverageVersion());\n statement.setString(index++, request.coverageHash());\n statement.setString(index++, request.manifest().manifestSha256());\n statement.setString(index++, \"PLANNED\");\n setTimestamp(statement, index++, request.scoringEligibleFrom());\n setTimestamp(statement, index++, request.scoringExpiresAt());\n statement.setTimestamp(index, Timestamp.from(request.loadedAt()));\n statement.executeUpdate();\n }\n }\n\n private static void insertDaysAndAssets(Connection connection, LoadRequest request) throws SQLException {\n Map exported = request.snapshot().days().stream()\n .collect(java.util.stream.Collectors.toMap(ExportedDay::sourceDate, value -> value));\n for (ResolvedSourceDay manifestDay : request.manifest().days()) {\n ExportedDay local = exported.get(manifestDay.sourceDate());\n DayLoadMetrics metrics = request.dayMetrics().get(manifestDay.sourceDate());\n SourceRelease release = manifestDay.selectedRelease();\n ResolvedArchiveFamily family = manifestDay.selectedFamily();\n try (PreparedStatement statement = connection.prepareStatement(\"\"\"\n insert into historical_bootstrap_days(\n dataset_id,source_date,experiment_role,selected_release_id,selected_release_tag,\n selected_release_variant,terminal_quality,reason_code,selected_archive_family,selected_bytes,\n parsed_trace_count,trace_parse_failure_count,malformed_point_count,accepted_point_count,\n rejected_point_count,finalized_segment_count,day_manifest_sha256,day_delta_checksum,qualified_at)\n values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)\n \"\"\")) {\n statement.setObject(1, request.datasetId());\n statement.setDate(2, Date.valueOf(manifestDay.sourceDate()));\n statement.setString(3, manifestDay.role().name());\n if (release == null) {\n statement.setObject(4, null); statement.setString(5, null); statement.setString(6, null);\n } else {\n statement.setLong(4, release.releaseId()); statement.setString(5, release.tag());\n statement.setString(6, release.variant().name());\n }\n statement.setString(7, local.quality().name());\n statement.setString(8, metrics.reasonCode());\n statement.setString(9, family == null ? null : family.kind().name());\n statement.setLong(10, family == null ? 0L : family.totalBytes());\n statement.setInt(11, metrics.parsedTraceCount());\n statement.setInt(12, metrics.traceParseFailureCount());\n statement.setLong(13, metrics.malformedPointCount());\n statement.setLong(14, metrics.acceptedPointCount());\n statement.setLong(15, metrics.rejectedPointCount());\n statement.setLong(16, metrics.finalizedSegmentCount());\n statement.setString(17, local.dayManifestSha256());\n statement.setString(18, local.dayDeltaChecksum());\n statement.setTimestamp(19, Timestamp.from(metrics.qualifiedAt()));\n statement.executeUpdate();\n }\n if (family != null) {\n short ordinal = 0;\n for (SourceAsset asset : family.assets()) {\n if (asset.sha256() == null) throw new IllegalArgumentException(\"selected source asset is missing SHA-256\");\n try (PreparedStatement statement = connection.prepareStatement(\"\"\"\n insert into historical_bootstrap_assets(\n dataset_id,source_date,asset_ordinal,release_asset_id,filename,byte_size,\n upstream_sha256,verified_sha256,source_repository,release_tag,archive_family)\n values(?,?,?,?,?,?,?,?,?,?,?)\n \"\"\")) {\n statement.setObject(1, request.datasetId());\n statement.setDate(2, Date.valueOf(manifestDay.sourceDate()));\n statement.setShort(3, ordinal++);\n statement.setLong(4, asset.assetId());\n statement.setString(5, asset.name());\n statement.setLong(6, asset.byteSize());\n statement.setString(7, asset.sha256());\n statement.setString(8, asset.sha256());\n statement.setString(9, release.repository());\n statement.setString(10, release.tag());\n statement.setString(11, family.kind().name());\n statement.executeUpdate();\n }\n }\n }\n }\n }\n\n private static Map insertPatterns(Connection connection, LoadRequest request) throws SQLException {\n Map ids = new LinkedHashMap<>();\n for (Map.Entry entry : request.snapshot().patterns().entrySet()) {\n HistoricalPatternKey key = entry.getKey();\n HistoricalPatternStats stats = entry.getValue();\n HistoricalSourceCounts sources = stats.sourceCounts();\n UUID patternId = uuidV5(request.datasetId(), request.derivationVersion() + \"|\" + key.canonical());\n ids.put(key, patternId);\n try (PreparedStatement statement = connection.prepareStatement(\"\"\"\n insert into historical_bootstrap_pattern_aggregates(\n pattern_id,dataset_id,derivation_version,pattern_kind,subject_key,start_h3_r5,end_h3_r5,\n evidence_count,qualified_evidence_count,first_seen_at,last_seen_at,minimum_altitude_ft,\n maximum_altitude_ft,mean_altitude_sum_ft,altitude_evidence_count,duration_seconds_sum,\n observation_count_sum,adsb_icao_count,adsr_icao_count,tisb_icao_count,mlat_count,created_at)\n values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)\n \"\"\")) {\n int index = 1;\n statement.setObject(index++, patternId);\n statement.setObject(index++, request.datasetId());\n statement.setString(index++, request.derivationVersion());\n statement.setString(index++, key.kind().name());\n statement.setString(index++, key.subjectKey());\n statement.setString(index++, key.startH3R5());\n statement.setString(index++, key.endH3R5());\n statement.setLong(index++, stats.evidenceCount());\n statement.setLong(index++, stats.cleanEvidenceCount());\n statement.setTimestamp(index++, Timestamp.from(stats.firstOccurrenceAt()));\n statement.setTimestamp(index++, Timestamp.from(stats.lastOccurrenceAt()));\n statement.setInt(index++, stats.minimumAltitudeFt());\n statement.setInt(index++, stats.maximumAltitudeFt());\n statement.setLong(index++, stats.meanAltitudeSumFt());\n statement.setLong(index++, stats.altitudeEvidenceCount());\n statement.setLong(index++, stats.durationSecondsSum());\n statement.setLong(index++, stats.observationCountSum());\n statement.setLong(index++, sources.adsbIcao());\n statement.setLong(index++, sources.adsrIcao());\n statement.setLong(index++, sources.tisbIcao());\n statement.setLong(index++, sources.mlat());\n statement.setTimestamp(index, Timestamp.from(request.loadedAt()));\n statement.executeUpdate();\n }\n }\n return Map.copyOf(ids);\n }\n\n private static void insertHourBins(Connection connection, LoadRequest request, Map ids)\n throws SQLException {\n for (Map.Entry entry : request.snapshot().hourBins().entrySet()) {\n UUID patternId = ids.get(entry.getKey().patternKey());\n if (patternId == null) throw new IllegalStateException(\"hour bin references missing compact pattern\");\n try (PreparedStatement statement = connection.prepareStatement(\n \"insert into historical_bootstrap_hour_bins(pattern_id,hour_of_week,evidence_count) values(?,?,?)\")) {\n statement.setObject(1, patternId);\n statement.setShort(2, entry.getKey().hourOfWeek());\n statement.setLong(3, entry.getValue());\n statement.executeUpdate();\n }\n }\n }\n\n private static void insertCorridors(Connection connection, LoadRequest request) throws SQLException {\n HistoricalCorridorPackWriter.WriteResult result = HistoricalCorridorPackWriter.insertSnapshot(\n connection,\n request.datasetId(),\n request.loadedAt(),\n request.snapshot().corridorAggregates());\n if (result.cellCount() != request.snapshot().corridorAggregates().size()) {\n throw new IllegalStateException(\"compact corridor persistence lost snapshot cells\");\n }\n }\n\n private static void insertHoldout(Connection connection, LoadRequest request) throws SQLException {\n for (ExportedHoldout row : request.snapshot().holdout()) {\n HistoricalHoldoutOutcome outcome = row.outcome();\n HistoricalSourceCounts sources = outcome.sourceCounts();\n try (PreparedStatement statement = connection.prepareStatement(\"\"\"\n insert into historical_bootstrap_holdout_segments(\n dataset_id,source_date,segment_id,icao24,callsign,first_observed_at,last_observed_at,\n start_h3_r5,end_h3_r5,minimum_altitude_ft,maximum_altitude_ft,mean_altitude_ft,\n altitude_evidence_count,duration_seconds,observation_count,adsb_icao_count,adsr_icao_count,\n tisb_icao_count,mlat_count,delete_after,corridor_overflow)\n values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)\n \"\"\")) {\n int index = 1;\n statement.setObject(index++, request.datasetId());\n statement.setDate(index++, Date.valueOf(row.sourceDate()));\n statement.setString(index++, outcome.segmentId());\n statement.setString(index++, outcome.icao24());\n statement.setString(index++, outcome.callsign());\n statement.setTimestamp(index++, Timestamp.from(outcome.firstObservedAt()));\n statement.setTimestamp(index++, Timestamp.from(outcome.lastObservedAt()));\n statement.setString(index++, outcome.startH3R5());\n statement.setString(index++, outcome.endH3R5());\n statement.setInt(index++, outcome.minimumAltitudeFt());\n statement.setInt(index++, outcome.maximumAltitudeFt());\n statement.setInt(index++, outcome.meanAltitudeFt());\n statement.setLong(index++, outcome.altitudeEvidenceCount());\n statement.setLong(index++, outcome.durationSeconds());\n statement.setLong(index++, outcome.observationCount());\n statement.setLong(index++, sources.adsbIcao());\n statement.setLong(index++, sources.adsrIcao());\n statement.setLong(index++, sources.tisbIcao());\n statement.setLong(index++, sources.mlat());\n statement.setTimestamp(index++, Timestamp.from(request.holdoutDeleteAfter()));\n statement.setBoolean(index, outcome.corridorOverflow());\n statement.executeUpdate();\n }\n if (!outcome.corridorOverflow()) {\n try (PreparedStatement statement = connection.prepareStatement(\"\"\"\n insert into historical_bootstrap_holdout_corridor_cells(\n dataset_id,segment_id,sequence_index,corridor_h3_r7,progress_permille,\n minimum_altitude_ft,maximum_altitude_ft,altitude_sum_ft,altitude_evidence_count)\n values(?,?,?,?,?,?,?,?,?)\n \"\"\")) {\n for (HistoricalCorridorCellEvidence cell : outcome.corridorCells()) {\n statement.setObject(1, request.datasetId());\n statement.setString(2, outcome.segmentId());\n statement.setShort(3, (short) cell.sequenceIndex());\n statement.setString(4, cell.h3R7());\n statement.setShort(5, (short) cell.firstEntryProgressPermille());\n statement.setInt(6, cell.minimumAltitudeFt());\n statement.setInt(7, cell.maximumAltitudeFt());\n statement.setLong(8, cell.altitudeSumFt());\n statement.setLong(9, cell.altitudeEvidenceCount());\n statement.addBatch();\n }\n statement.executeBatch();\n }\n }\n }\n }\n\n private static void validateStaged(Connection connection, LoadRequest request) throws SQLException {\n assertCount(connection, \"historical_bootstrap_days\", request.datasetId(), request.snapshot().days().size());\n assertCount(connection, \"historical_bootstrap_pattern_aggregates\", request.datasetId(), request.snapshot().patterns().size());\n assertCorridorCellCount(\n connection, request.datasetId(), request.snapshot().corridorAggregates().size());\n assertCount(connection, \"historical_bootstrap_holdout_segments\", request.datasetId(), request.snapshot().holdout().size());\n long expectedHoldoutCells = request.snapshot().holdout().stream()\n .mapToLong(row -> row.outcome().corridorCells().size()).sum();\n assertCount(connection, \"historical_bootstrap_holdout_corridor_cells\", request.datasetId(), expectedHoldoutCells);\n\n long readyPatterns = scalar(connection,\n \"select count(*) from historical_bootstrap_ready_pattern_aggregates where dataset_id=?\", request.datasetId());\n long readyCorridors = scalar(connection,\n \"select count(*) from historical_bootstrap_ready_route_corridor_aggregates where dataset_id=?\", request.datasetId());\n if (readyPatterns != 0L || readyCorridors != 0L) {\n throw new IllegalStateException(\"STAGED bootstrap evidence became visible through READY seam\");\n }\n long persistedHourBins = scalar(connection, \"\"\"\n select count(*) from historical_bootstrap_hour_bins h\n join historical_bootstrap_pattern_aggregates p on p.pattern_id=h.pattern_id\n where p.dataset_id=?\n \"\"\", request.datasetId());\n if (persistedHourBins != request.snapshot().hourBins().size()) {\n throw new IllegalStateException(\"staged bootstrap hour-bin cardinality mismatch\");\n }\n }\n\n private static void validateReady(Connection connection, LoadRequest request) throws SQLException {\n long readyDatasets = scalar(connection,\n \"select count(*) from historical_bootstrap_ready_datasets where dataset_id=?\", request.datasetId());\n long readyPatterns = scalar(connection,\n \"select count(*) from historical_bootstrap_ready_pattern_aggregates where dataset_id=?\", request.datasetId());\n long readyHourBins = scalar(connection, \"\"\"\n select count(*) from historical_bootstrap_ready_hour_bins h\n join historical_bootstrap_pattern_aggregates p on p.pattern_id=h.pattern_id\n where p.dataset_id=?\n \"\"\", request.datasetId());\n long readyCorridors = scalar(connection,\n \"select count(*) from historical_bootstrap_ready_route_corridor_aggregates where dataset_id=?\", request.datasetId());\n if (readyDatasets != 1L || readyPatterns != request.snapshot().patterns().size()\n || readyHourBins != request.snapshot().hourBins().size()\n || readyCorridors != request.snapshot().corridorAggregates().size()) {\n throw new IllegalStateException(\"READY bootstrap read seam does not match validated snapshot\");\n }\n }\n\n private static void assertCorridorCellCount(Connection connection, UUID datasetId, long expected)\n throws SQLException {\n long actual = scalar(connection, \"\"\"\n select\n (select count(*) from historical_bootstrap_route_corridor_aggregates where dataset_id=?)\n + coalesce((\n select sum(cardinality(corridor_h3_r7))\n from historical_bootstrap_route_corridor_packs\n where dataset_id=?\n ),0)\n \"\"\", datasetId, datasetId);\n if (actual != expected) {\n throw new IllegalStateException(\n \"historical corridor cell cardinality mismatch: expected=\" + expected + \" actual=\" + actual);\n }\n }\n\n private static void assertCount(Connection connection, String table, UUID datasetId, long expected) throws SQLException {\n long actual = scalar(connection, \"select count(*) from \" + table + \" where dataset_id=?\", datasetId);\n if (actual != expected) {\n throw new IllegalStateException(table + \" cardinality mismatch: expected=\" + expected + \" actual=\" + actual);\n }\n }\n\n private static long scalar(Connection connection, String sql, UUID datasetId) throws SQLException {\n try (PreparedStatement statement = connection.prepareStatement(sql)) {\n if (datasetId != null) statement.setObject(1, datasetId);\n try (ResultSet rs = statement.executeQuery()) {\n if (!rs.next()) throw new IllegalStateException(\"scalar query returned no row\");\n return rs.getLong(1);\n }\n }\n }\n\n private static long scalar(Connection connection, String sql, UUID first, UUID second) throws SQLException {\n try (PreparedStatement statement = connection.prepareStatement(sql)) {\n statement.setObject(1, first);\n statement.setObject(2, second);\n try (ResultSet rs = statement.executeQuery()) {\n if (!rs.next()) throw new IllegalStateException(\"scalar query returned no row\");\n return rs.getLong(1);\n }\n }\n }\n\n private static void transition(Connection connection, UUID datasetId, String state, Instant readyAt) throws SQLException {\n String sql = readyAt == null\n ? \"update historical_bootstrap_datasets set state=? where dataset_id=?\"\n : \"update historical_bootstrap_datasets set state=?,ready_at=? where dataset_id=?\";\n try (PreparedStatement statement = connection.prepareStatement(sql)) {\n statement.setString(1, state);\n if (readyAt == null) {\n statement.setObject(2, datasetId);\n } else {\n statement.setTimestamp(2, Timestamp.from(readyAt));\n statement.setObject(3, datasetId);\n }\n if (statement.executeUpdate() != 1) {\n throw new IllegalStateException(\"dataset lifecycle transition failed: \" + state);\n }\n }\n }\n\n private static String sourceRepository(BootstrapSourceManifest manifest) {\n List repositories = manifest.days().stream()\n .flatMap(day -> day.candidateReleases().stream())\n .map(SourceRelease::repository).distinct().sorted().toList();\n if (repositories.size() != 1) {\n throw new IllegalArgumentException(\"initial P7 persistence requires exactly one frozen source repository\");\n }\n return repositories.getFirst();\n }\n\n private static int sourceYear(BootstrapSourceManifest manifest) {\n List years = manifest.days().stream().map(day -> day.sourceDate().getYear()).distinct().sorted().toList();\n if (years.size() != 1) {\n throw new IllegalArgumentException(\"initial P7 persistence requires one source year per dataset\");\n }\n return years.getFirst();\n }\n\n private static HistoricalSourceCounts decodeSourceCounts(String value) {\n if (value == null) throw new IllegalStateException(\"missing compact source counts\");\n String[] values = value.split(\",\", -1);\n if (values.length != 4) throw new IllegalStateException(\"invalid compact source-count encoding\");\n try {\n return new HistoricalSourceCounts(Long.parseLong(values[0]), Long.parseLong(values[1]),\n Long.parseLong(values[2]), Long.parseLong(values[3]));\n } catch (NumberFormatException exception) {\n throw new IllegalStateException(\"invalid compact source-count encoding\", exception);\n }\n }\n\n private static UUID uuidV5(UUID namespace, String name) {\n try {\n MessageDigest sha1 = MessageDigest.getInstance(\"SHA-1\");\n sha1.update(ByteBuffer.allocate(16).putLong(namespace.getMostSignificantBits())\n .putLong(namespace.getLeastSignificantBits()).array());\n byte[] bytes = sha1.digest(name.getBytes(StandardCharsets.UTF_8));\n bytes[6] = (byte) ((bytes[6] & 0x0f) | 0x50);\n bytes[8] = (byte) ((bytes[8] & 0x3f) | 0x80);\n ByteBuffer buffer = ByteBuffer.wrap(bytes);\n return new UUID(buffer.getLong(), buffer.getLong());\n } catch (NoSuchAlgorithmException exception) {\n throw new IllegalStateException(\"SHA-1 unavailable\", exception);\n }\n }\n\n private static void requireHash(String value, String name) {\n if (value == null || !value.matches(\"[0-9a-f]{64}\")) {\n throw new IllegalArgumentException(name + \" must be lowercase SHA-256\");\n }\n }\n\n private static void requireText(String value, int max, String name) {\n if (value == null || value.isBlank() || value.length() > max) {\n throw new IllegalArgumentException(name + \" is missing or unbounded\");\n }\n }\n\n private static void setDate(PreparedStatement statement, int index, LocalDate value) throws SQLException {\n if (value == null) statement.setDate(index, null); else statement.setDate(index, Date.valueOf(value));\n }\n\n private static void setTimestamp(PreparedStatement statement, int index, Instant value) throws SQLException {\n if (value == null) statement.setTimestamp(index, null); else statement.setTimestamp(index, Timestamp.from(value));\n }\n\n record ExportedDay(\n LocalDate sourceDate,\n String dayManifestSha256,\n String semanticFingerprint,\n String dayDeltaChecksum,\n LocalDayCommitStore.EvidenceQualityStatus quality\n ) {\n }\n\n record ExportedHoldout(LocalDate sourceDate, HistoricalHoldoutOutcome outcome) {\n }\n\n record ExportedSnapshot(\n UUID datasetId,\n List days,\n Map patterns,\n Map hourBins,\n Map corridorAggregates,\n List holdout\n ) {\n ExportedSnapshot {\n Objects.requireNonNull(datasetId, \"datasetId\");\n days = days == null ? List.of() : days.stream().sorted(Comparator.comparing(ExportedDay::sourceDate)).toList();\n patterns = Map.copyOf(new TreeMap<>(patterns == null ? Map.of() : patterns));\n hourBins = Map.copyOf(new TreeMap<>(hourBins == null ? Map.of() : hourBins));\n corridorAggregates = Map.copyOf(new TreeMap<>(corridorAggregates == null ? Map.of() : corridorAggregates));\n holdout = holdout == null ? List.of() : holdout.stream()\n .sorted(Comparator.comparing(ExportedHoldout::sourceDate)\n .thenComparing(row -> row.outcome().segmentId())).toList();\n }\n\n ExportedSnapshot(\n UUID datasetId,\n List days,\n Map patterns,\n Map hourBins,\n List holdout\n ) {\n this(datasetId, days, patterns, hourBins, Map.of(), holdout);\n }\n }\n\n record DayLoadMetrics(\n LocalDate sourceDate,\n int parsedTraceCount,\n int traceParseFailureCount,\n long malformedPointCount,\n long acceptedPointCount,\n long rejectedPointCount,\n long finalizedSegmentCount,\n String reasonCode,\n Instant qualifiedAt\n ) {\n DayLoadMetrics {\n Objects.requireNonNull(sourceDate, \"sourceDate\");\n Objects.requireNonNull(qualifiedAt, \"qualifiedAt\");\n if (parsedTraceCount < 0 || traceParseFailureCount < 0 || malformedPointCount < 0 || acceptedPointCount < 0\n || rejectedPointCount < 0 || finalizedSegmentCount < 0) {\n throw new IllegalArgumentException(\"day load metrics cannot be negative\");\n }\n if (reasonCode != null && !reasonCode.matches(\"[A-Z0-9_]{1,64}\")) {\n throw new IllegalArgumentException(\"reasonCode must be finite uppercase identifier\");\n }\n }\n }\n\n record LoadRequest(\n UUID datasetId,\n BootstrapSourceManifest manifest,\n String semanticFingerprint,\n String coverageVersion,\n String coverageHash,\n String derivationVersion,\n ExportedSnapshot snapshot,\n Map dayMetrics,\n Instant loadedAt,\n Instant holdoutDeleteAfter,\n Instant scoringEligibleFrom,\n Instant scoringExpiresAt\n ) {\n LoadRequest {\n dayMetrics = dayMetrics == null ? Map.of() : Map.copyOf(new TreeMap<>(dayMetrics));\n }\n }\n\n record LoadResult(\n UUID datasetId,\n int dayCount,\n int patternCount,\n int hourBinCount,\n int corridorCount,\n int holdoutCount,\n Instant readyAt\n ) {\n }\n}\n" }, { "path": "backend/historical-bootstrap-tool/src/main/java/hu/gombos/flightspotter/historicalbootstrap/HistoricalBootstrapPostgresStreamingPersistence.java", "encoding": "UTF-8", "sha256": "2815f8561c105042f3bd6a13512fe2fc31e934ac9c779492cffb3a12bd139a01", "content": "package hu.gombos.flightspotter.historicalbootstrap;\n\nimport javax.sql.DataSource;\nimport java.nio.ByteBuffer;\nimport java.nio.charset.StandardCharsets;\nimport java.nio.file.Path;\nimport java.security.MessageDigest;\nimport java.security.NoSuchAlgorithmException;\nimport java.sql.Connection;\nimport java.sql.Date;\nimport java.sql.DriverManager;\nimport java.sql.PreparedStatement;\nimport java.sql.ResultSet;\nimport java.sql.SQLException;\nimport java.sql.Timestamp;\nimport java.time.Instant;\nimport java.time.LocalDate;\nimport java.util.LinkedHashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Objects;\nimport java.util.TreeMap;\nimport java.util.UUID;\nimport java.util.regex.Pattern;\n\n/**\n * Bounded-memory H2 -> PostgreSQL staging seam for qualified P7 snapshots.\n *\n * This class intentionally does not call exportLocalSnapshot(): the real Gate D state is streamed\n * directly from H2 through bounded JDBC batches while preserving the V69/V70/V71 lifecycle and\n * validation invariants. The older object-snapshot seam remains for small integration fixtures.\n */\nfinal class HistoricalBootstrapPostgresStreamingPersistence {\n static final String IMPORTER_VERSION = \"p7-postgres-loader-v5-packed-corridors\";\n static final int BATCH_SIZE = 1_000;\n static final int SOURCE_FETCH_SIZE = 2_048;\n static final int HOLDOUT_SEGMENT_PAGE_SIZE = 128;\n private static final Pattern PERSISTABLE_CALLSIGN = Pattern.compile(\"[A-Z0-9][A-Z0-9-]{0,31}\");\n\n private HistoricalBootstrapPostgresStreamingPersistence() {\n }\n\n static LocalSummary qualifyLocal(StreamingLoadRequest request) throws Exception {\n validateRequestMetadata(request, false);\n try (Connection source = openSource(request.stateDatabase())) {\n return qualifySource(source, request, true);\n }\n }\n\n static StreamingLoadResult load(DataSource dataSource, StreamingLoadRequest request) throws Exception {\n Objects.requireNonNull(dataSource, \"dataSource\");\n validateRequestMetadata(request, true);\n\n try (Connection source = openSource(request.stateDatabase())) {\n LocalSummary summary = qualifySource(source, request, false);\n\n try (Connection target = dataSource.getConnection()) {\n boolean originalAutoCommit = target.getAutoCommit();\n boolean committed = false;\n boolean restoreAutoCommit = true;\n Throwable failure = null;\n target.setAutoCommit(false);\n try {\n rejectExistingDataset(target, request.datasetId());\n Map localDays = readLocalDays(source, request.datasetId().toString());\n\n insertDataset(target, request);\n transition(target, request.datasetId(), \"MANIFEST_LOCKED\", null);\n transition(target, request.datasetId(), \"DERIVING\", null);\n insertDaysAndAssets(target, request, localDays);\n long writtenPatterns = streamPatterns(source, target, request);\n long writtenHourBins = streamHourBins(source, target, request);\n HistoricalCorridorPackWriter.WriteResult writtenCorridors =\n streamCorridors(source, target, request);\n streamHoldoutSegments(source, target, request);\n streamHoldoutCorridorCells(source, target, request);\n\n transition(target, request.datasetId(), \"STAGED\", null);\n validateStaged(target, request.datasetId(), summary,\n new StreamedCounts(writtenPatterns, writtenHourBins,\n writtenCorridors.cellCount(), writtenCorridors.packCount()));\n transition(target, request.datasetId(), \"READY\", request.loadedAt());\n validateReady(target, request.datasetId(), summary);\n target.commit();\n committed = true;\n\n return new StreamingLoadResult(\n request.datasetId(),\n summary.dayCount(),\n summary.patternCount(),\n summary.hourBinCount(),\n summary.corridorCount(),\n summary.holdoutCount(),\n summary.holdoutCorridorCount(),\n request.loadedAt());\n } catch (Exception | Error caught) {\n failure = caught;\n if (!committed) {\n try {\n target.rollback();\n } catch (Exception | Error rollbackFailure) {\n restoreAutoCommit = false;\n caught.addSuppressed(rollbackFailure);\n try {\n target.close();\n } catch (Exception | Error closeFailure) {\n caught.addSuppressed(closeFailure);\n }\n }\n }\n throw caught;\n } finally {\n if (restoreAutoCommit) {\n try {\n target.setAutoCommit(originalAutoCommit);\n } catch (Exception | Error restoreFailure) {\n if (failure != null) {\n failure.addSuppressed(restoreFailure);\n } else {\n throw restoreFailure;\n }\n }\n }\n }\n }\n }\n }\n\n private static Connection openSource(Path stateDatabase) throws SQLException {\n Objects.requireNonNull(stateDatabase, \"stateDatabase\");\n Path absolute = stateDatabase.toAbsolutePath().normalize();\n Connection connection = DriverManager.getConnection(\n \"jdbc:h2:file:\" + absolute.toString().replace('\\\\', '/')\n + \";IFEXISTS=TRUE;ACCESS_MODE_DATA=r;DB_CLOSE_ON_EXIT=FALSE\");\n connection.setReadOnly(true);\n return connection;\n }\n\n private static void validateRequestMetadata(StreamingLoadRequest request, boolean requireRetention) {\n Objects.requireNonNull(request, \"request\");\n Objects.requireNonNull(request.stateDatabase(), \"stateDatabase\");\n Objects.requireNonNull(request.datasetId(), \"datasetId\");\n Objects.requireNonNull(request.manifest(), \"manifest\");\n Objects.requireNonNull(request.loadedAt(), \"loadedAt\");\n requireHash(request.semanticFingerprint(), \"semanticFingerprint\");\n requireText(request.coverageVersion(), 80, \"coverageVersion\");\n requireHash(request.coverageHash(), \"coverageHash\");\n requireText(request.derivationVersion(), 80, \"derivationVersion\");\n\n if (request.dayMetrics().size() != request.manifest().days().size()) {\n throw new IllegalArgumentException(\"one terminal metrics row is required per manifest day\");\n }\n if (request.scoringEligibleFrom() != null && request.scoringExpiresAt() != null\n && !request.scoringExpiresAt().isAfter(request.scoringEligibleFrom())) {\n throw new IllegalArgumentException(\"scoring expiry must be after eligibility start\");\n }\n sourceRepository(request.manifest());\n sourceYear(request.manifest());\n\n if (requireRetention && request.holdoutDeleteAfter() == null) {\n throw new IllegalArgumentException(\"holdoutDeleteAfter is required for a staging load\");\n }\n }\n\n private static LocalSummary qualifySource(\n Connection source,\n StreamingLoadRequest request,\n boolean scanEveryLargeRow\n ) throws Exception {\n String dataset = request.datasetId().toString();\n Map localDays = readLocalDays(source, dataset);\n\n if (localDays.size() != request.manifest().days().size()) {\n throw new IllegalArgumentException(\"local committed-day count does not match immutable manifest\");\n }\n\n for (ResolvedSourceDay manifestDay : request.manifest().days()) {\n LocalDay local = localDays.get(manifestDay.sourceDate());\n if (local == null) {\n throw new IllegalArgumentException(\"missing local committed day at \" + manifestDay.sourceDate());\n }\n if (!local.dayManifestSha256().equals(request.manifest().dayManifestSha256(manifestDay.sourceDate()))) {\n throw new IllegalArgumentException(\"local committed day does not match immutable manifest at \"\n + manifestDay.sourceDate());\n }\n if (!local.semanticFingerprint().equals(request.semanticFingerprint())) {\n throw new IllegalArgumentException(\"local day semantic fingerprint differs at \"\n + manifestDay.sourceDate());\n }\n requireHash(local.dayManifestSha256(), \"dayManifestSha256\");\n requireHash(local.semanticFingerprint(), \"day semanticFingerprint\");\n requireHash(local.dayDeltaChecksum(), \"dayDeltaChecksum\");\n\n HistoricalBootstrapPostgresPersistence.DayLoadMetrics metrics =\n request.dayMetrics().get(manifestDay.sourceDate());\n if (metrics == null || !metrics.sourceDate().equals(manifestDay.sourceDate())) {\n throw new IllegalArgumentException(\"missing terminal day metrics at \" + manifestDay.sourceDate());\n }\n if (local.quality() == LocalDayCommitStore.EvidenceQualityStatus.QUALIFIED\n && metrics.reasonCode() != null) {\n throw new IllegalArgumentException(\"QUALIFIED day cannot carry a terminal failure reason\");\n }\n if (local.quality() != LocalDayCommitStore.EvidenceQualityStatus.QUALIFIED\n && metrics.reasonCode() == null) {\n throw new IllegalArgumentException(\"DEGRADED/GAP day requires a finite terminal reason\");\n }\n }\n\n long carryCount = scalar(source,\n \"select count(*) from p7_local_carry where dataset_id=?\", dataset);\n if (carryCount != 0L) {\n throw new IllegalStateException(\"final local carry is not empty: \" + carryCount);\n }\n\n long patternCount = scalar(source,\n \"select count(*) from p7_local_patterns where dataset_id=?\", dataset);\n long hourBinCount = scalar(source,\n \"select count(*) from p7_local_hour_bins where dataset_id=?\", dataset);\n long corridorCount = scalar(source,\n \"select count(*) from p7_local_corridor_aggregates where dataset_id=?\", dataset);\n long holdoutCount = scalar(source,\n \"select count(*) from p7_local_holdout where dataset_id=?\", dataset);\n long holdoutCorridorCount = scalar(source,\n \"select count(*) from p7_local_holdout_corridor where dataset_id=?\", dataset);\n long normalizedHoldoutCallsignCount = countNonPersistableHoldoutCallsigns(source, dataset);\n\n long invalidCorridorCount = scalar(source, \"\"\"\n select count(*)\n from p7_local_corridor_aggregates c\n left join p7_local_patterns p\n on p.dataset_id=c.dataset_id\n and p.pattern_kind='ROUTE'\n and p.subject_key='-'\n and p.start_h3_r5=c.start_h3_r5\n and p.end_h3_r5=c.end_h3_r5\n where c.dataset_id=?\n and (p.evidence_count is null or c.traversal_count > p.evidence_count)\n \"\"\", dataset);\n if (invalidCorridorCount != 0L) {\n throw new IllegalStateException(\"local corridor/coarse-ROUTE invariant violations=\" + invalidCorridorCount);\n }\n\n long orphanHoldoutCells = scalar(source, \"\"\"\n select count(*)\n from p7_local_holdout_corridor c\n left join p7_local_holdout h\n on h.dataset_id=c.dataset_id and h.segment_id=c.segment_id\n where c.dataset_id=? and h.segment_id is null\n \"\"\", dataset);\n if (orphanHoldoutCells != 0L) {\n throw new IllegalStateException(\"orphan local HOLDOUT corridor cells=\" + orphanHoldoutCells);\n }\n\n long overflowHoldoutCells = scalar(source, \"\"\"\n select count(*)\n from p7_local_holdout_corridor c\n join p7_local_holdout h\n on h.dataset_id=c.dataset_id and h.segment_id=c.segment_id\n where c.dataset_id=? and h.corridor_overflow=true\n \"\"\", dataset);\n if (overflowHoldoutCells != 0L) {\n throw new IllegalStateException(\"overflow local HOLDOUT segments retained corridor cells=\"\n + overflowHoldoutCells);\n }\n\n long completeHoldoutWithoutCells = scalar(source, \"\"\"\n select count(*)\n from p7_local_holdout h\n where h.dataset_id=?\n and h.corridor_overflow=false\n and not exists (\n select 1 from p7_local_holdout_corridor c\n where c.dataset_id=h.dataset_id and c.segment_id=h.segment_id\n )\n \"\"\", dataset);\n if (completeHoldoutWithoutCells != 0L) {\n throw new IllegalStateException(\"non-overflow local HOLDOUT segments missing corridor cells=\"\n + completeHoldoutWithoutCells);\n }\n\n try (PreparedStatement statement = source.prepareStatement(\n \"select distinct source_date from p7_local_holdout where dataset_id=?\")) {\n statement.setString(1, dataset);\n try (ResultSet rs = statement.executeQuery()) {\n while (rs.next()) {\n LocalDate date = rs.getDate(1).toLocalDate();\n if (request.manifest().window().roleFor(date) != DatasetWindow.DateRole.HOLDOUT) {\n throw new IllegalStateException(\"local HOLDOUT row attached to non-HOLDOUT day \" + date);\n }\n }\n }\n }\n\n if (holdoutCount > 0L && request.holdoutDeleteAfter() != null) {\n long maxLastObservedMs = scalar(source,\n \"select coalesce(max(last_observed_ms),0) from p7_local_holdout where dataset_id=?\", dataset);\n if (request.holdoutDeleteAfter().toEpochMilli() <= maxLastObservedMs) {\n throw new IllegalArgumentException(\"holdoutDeleteAfter must be after every HOLDOUT observation\");\n }\n }\n\n HeapSampler heap = new HeapSampler();\n\n // Validate the 8.7M-cell HOLDOUT structure with bounded per-segment state instead of\n // a global GROUP BY. This runs for both qualification and actual staging prevalidation.\n long structurallyScannedHoldoutCells = scanHoldoutCorridorCells(source, dataset, heap);\n requireCount(\"HOLDOUT corridor structural scan\", holdoutCorridorCount, structurallyScannedHoldoutCells);\n\n if (scanEveryLargeRow) {\n long scannedPatterns = scanPatterns(source, dataset, heap);\n long scannedHourBins = scanHourBins(source, dataset, heap);\n long scannedCorridors = scanCorridors(source, dataset, heap);\n long scannedHoldout = scanHoldout(source, dataset, heap);\n\n requireCount(\"pattern scan\", patternCount, scannedPatterns);\n requireCount(\"hour-bin scan\", hourBinCount, scannedHourBins);\n requireCount(\"corridor scan\", corridorCount, scannedCorridors);\n requireCount(\"HOLDOUT scan\", holdoutCount, scannedHoldout);\n }\n\n return new LocalSummary(\n localDays.size(),\n patternCount,\n hourBinCount,\n corridorCount,\n holdoutCount,\n holdoutCorridorCount,\n normalizedHoldoutCallsignCount,\n carryCount,\n heap.peakBytes());\n }\n\n private static Map readLocalDays(Connection source, String dataset) throws SQLException {\n Map days = new TreeMap<>();\n try (PreparedStatement statement = source.prepareStatement(\"\"\"\n select source_date,day_manifest_hash,semantic_fingerprint,delta_checksum,evidence_quality\n from p7_local_committed_days\n where dataset_id=?\n order by source_date\n \"\"\")) {\n statement.setString(1, dataset);\n try (ResultSet rs = statement.executeQuery()) {\n while (rs.next()) {\n LocalDay row = new LocalDay(\n rs.getDate(1).toLocalDate(),\n rs.getString(2),\n rs.getString(3),\n rs.getString(4),\n LocalDayCommitStore.EvidenceQualityStatus.valueOf(rs.getString(5)));\n if (days.put(row.sourceDate(), row) != null) {\n throw new IllegalStateException(\"duplicate local committed source day \" + row.sourceDate());\n }\n }\n }\n }\n return Map.copyOf(days);\n }\n\n private static long scanPatterns(Connection source, String dataset, HeapSampler heap) throws SQLException {\n long count = 0L;\n try (PreparedStatement statement = source.prepareStatement(\"\"\"\n select pattern_kind,subject_key,start_h3_r5,end_h3_r5,evidence_count,clean_evidence_count,\n first_occurrence_ms,last_occurrence_ms,minimum_altitude_ft,maximum_altitude_ft,\n mean_altitude_sum_ft,altitude_evidence_count,duration_seconds_sum,observation_count_sum,source_counts\n from p7_local_patterns where dataset_id=?\n \"\"\")) {\n statement.setFetchSize(SOURCE_FETCH_SIZE);\n statement.setString(1, dataset);\n try (ResultSet rs = statement.executeQuery()) {\n while (rs.next()) {\n String kind = rs.getString(1);\n String subject = rs.getString(2);\n String start = rs.getString(3);\n String end = rs.getString(4);\n if (kind == null || subject == null || start == null || end == null\n || rs.getLong(5) < 1L || rs.getLong(6) < 0L\n || rs.getLong(14) < 1L || rs.getString(15) == null) {\n throw new IllegalStateException(\"invalid streamed local pattern row\");\n }\n count++;\n heap.sample(count);\n }\n }\n }\n return count;\n }\n\n private static long scanHourBins(Connection source, String dataset, HeapSampler heap) throws SQLException {\n long count = 0L;\n try (PreparedStatement statement = source.prepareStatement(\"\"\"\n select pattern_kind,subject_key,start_h3_r5,end_h3_r5,hour_of_week,evidence_count\n from p7_local_hour_bins where dataset_id=?\n \"\"\")) {\n statement.setFetchSize(SOURCE_FETCH_SIZE);\n statement.setString(1, dataset);\n try (ResultSet rs = statement.executeQuery()) {\n while (rs.next()) {\n short hour = rs.getShort(5);\n if (rs.getString(1) == null || rs.getString(2) == null || rs.getString(3) == null\n || rs.getString(4) == null || hour < 0 || hour > 167 || rs.getLong(6) < 1L) {\n throw new IllegalStateException(\"invalid streamed local hour-bin row\");\n }\n count++;\n heap.sample(count);\n }\n }\n }\n return count;\n }\n\n private static long scanCorridors(Connection source, String dataset, HeapSampler heap) throws SQLException {\n long count = 0L;\n try (PreparedStatement statement = source.prepareStatement(\"\"\"\n select start_h3_r5,end_h3_r5,corridor_h3_r7,traversal_count,progress_permille_sum,\n minimum_altitude_ft,maximum_altitude_ft,altitude_sum_ft,altitude_evidence_count\n from p7_local_corridor_aggregates where dataset_id=?\n \"\"\")) {\n statement.setFetchSize(SOURCE_FETCH_SIZE);\n statement.setString(1, dataset);\n try (ResultSet rs = statement.executeQuery()) {\n while (rs.next()) {\n if (rs.getString(1) == null || rs.getString(2) == null || rs.getString(3) == null\n || rs.getLong(4) < 1L || rs.getLong(9) < 1L) {\n throw new IllegalStateException(\"invalid streamed local corridor row\");\n }\n count++;\n heap.sample(count);\n }\n }\n }\n return count;\n }\n\n private static long countNonPersistableHoldoutCallsigns(Connection source, String dataset) throws SQLException {\n long count = 0L;\n try (PreparedStatement statement = source.prepareStatement(\n \"select callsign from p7_local_holdout where dataset_id=? and callsign is not null\")) {\n statement.setFetchSize(SOURCE_FETCH_SIZE);\n statement.setString(1, dataset);\n try (ResultSet rs = statement.executeQuery()) {\n while (rs.next()) {\n if (!isPersistableCallsign(rs.getString(1))) {\n count++;\n }\n }\n }\n }\n return count;\n }\n\n private static boolean isPersistableCallsign(String callsign) {\n return callsign != null && PERSISTABLE_CALLSIGN.matcher(callsign).matches();\n }\n\n private static String persistableCallsignOrNull(String callsign) {\n return isPersistableCallsign(callsign) ? callsign : null;\n }\n\n private static long scanHoldout(Connection source, String dataset, HeapSampler heap) throws SQLException {\n long count = 0L;\n try (PreparedStatement statement = source.prepareStatement(\"\"\"\n select source_date,segment_id,icao24,callsign,first_observed_ms,last_observed_ms,start_h3_r5,end_h3_r5,\n minimum_altitude_ft,maximum_altitude_ft,mean_altitude_ft,altitude_evidence_count,\n duration_seconds,observation_count,source_counts,corridor_overflow\n from p7_local_holdout where dataset_id=?\n \"\"\")) {\n statement.setFetchSize(SOURCE_FETCH_SIZE);\n statement.setString(1, dataset);\n try (ResultSet rs = statement.executeQuery()) {\n while (rs.next()) {\n if (rs.getDate(1) == null || rs.getString(2) == null || rs.getString(3) == null\n || rs.getLong(6) < rs.getLong(5) || rs.getString(7) == null || rs.getString(8) == null\n || rs.getLong(12) < 1L || rs.getLong(14) < 1L || rs.getString(15) == null) {\n throw new IllegalStateException(\"invalid streamed local HOLDOUT row\");\n }\n count++;\n heap.sample(count);\n }\n }\n }\n return count;\n }\n\n private static long scanHoldoutCorridorCells(Connection source, String dataset, HeapSampler heap)\n throws SQLException {\n long cellCount = 0L;\n\n String afterSegmentId = null;\n while (true) {\n List segmentPage = readHoldoutSegmentPage(source, dataset, afterSegmentId);\n if (segmentPage.isEmpty()) {\n break;\n }\n\n String firstSegmentId = segmentPage.getFirst();\n String lastSegmentId = segmentPage.getLast();\n int pageHardLimit = Math.addExact(\n Math.multiplyExact(segmentPage.size(), HistoricalCorridorModel.MAX_UNIQUE_CELLS),\n 1);\n\n String activeSegment = null;\n int expectedSequence = 0;\n java.util.HashSet uniqueCells =\n new java.util.HashSet<>(HistoricalCorridorModel.MAX_UNIQUE_CELLS);\n int pageCellCount = 0;\n\n try (PreparedStatement statement = source.prepareStatement(\"\"\"\n select segment_id,sequence_index,corridor_h3_r7,progress_permille,\n minimum_altitude_ft,maximum_altitude_ft,altitude_sum_ft,altitude_evidence_count\n from p7_local_holdout_corridor\n where dataset_id=? and segment_id>=? and segment_id<=?\n order by segment_id,sequence_index\n limit ?\n \"\"\")) {\n statement.setFetchSize(SOURCE_FETCH_SIZE);\n statement.setString(1, dataset);\n statement.setString(2, firstSegmentId);\n statement.setString(3, lastSegmentId);\n statement.setInt(4, pageHardLimit);\n\n try (ResultSet rs = statement.executeQuery()) {\n while (rs.next()) {\n pageCellCount++;\n if (pageCellCount >= pageHardLimit) {\n throw new IllegalStateException(\n \"bounded HOLDOUT corridor page exceeded model maximum between \"\n + firstSegmentId + \" and \" + lastSegmentId);\n }\n\n String segmentId = rs.getString(1);\n if (!Objects.equals(activeSegment, segmentId)) {\n validateCompletedCellSequence(activeSegment, expectedSequence, uniqueCells);\n activeSegment = segmentId;\n expectedSequence = 0;\n uniqueCells.clear();\n }\n\n int sequence = rs.getInt(2);\n String h3 = rs.getString(3);\n int progress = rs.getInt(4);\n long altitudeEvidence = rs.getLong(8);\n\n if (sequence != expectedSequence\n || sequence < 0\n || sequence >= HistoricalCorridorModel.MAX_UNIQUE_CELLS\n || h3 == null\n || progress < 0\n || progress > 1000\n || altitudeEvidence < 1L\n || !uniqueCells.add(h3)) {\n throw new IllegalStateException(\n \"invalid streamed local HOLDOUT corridor row at segment=\"\n + segmentId + \" sequence=\" + sequence);\n }\n\n expectedSequence++;\n cellCount++;\n heap.sample(cellCount);\n }\n }\n }\n\n validateCompletedCellSequence(activeSegment, expectedSequence, uniqueCells);\n afterSegmentId = lastSegmentId;\n }\n\n return cellCount;\n }\n\n private static List readHoldoutSegmentPage(\n Connection source,\n String dataset,\n String afterSegmentId\n ) throws SQLException {\n String sql = afterSegmentId == null\n ? \"\"\"\n select segment_id\n from p7_local_holdout\n where dataset_id=?\n order by segment_id\n limit ?\n \"\"\"\n : \"\"\"\n select segment_id\n from p7_local_holdout\n where dataset_id=? and segment_id>?\n order by segment_id\n limit ?\n \"\"\";\n\n java.util.ArrayList result = new java.util.ArrayList<>(HOLDOUT_SEGMENT_PAGE_SIZE);\n try (PreparedStatement statement = source.prepareStatement(sql)) {\n statement.setString(1, dataset);\n if (afterSegmentId == null) {\n statement.setInt(2, HOLDOUT_SEGMENT_PAGE_SIZE);\n } else {\n statement.setString(2, afterSegmentId);\n statement.setInt(3, HOLDOUT_SEGMENT_PAGE_SIZE);\n }\n\n try (ResultSet rs = statement.executeQuery()) {\n while (rs.next()) {\n result.add(rs.getString(1));\n }\n }\n }\n return List.copyOf(result);\n }\n\n private static void validateCompletedCellSequence(\n String segmentId,\n int cellCount,\n java.util.Set uniqueCells\n ) {\n if (segmentId == null) {\n return;\n }\n if (cellCount < 1\n || cellCount > HistoricalCorridorModel.MAX_UNIQUE_CELLS\n || uniqueCells.size() != cellCount) {\n throw new IllegalStateException(\n \"invalid complete HOLDOUT corridor at segment=\" + segmentId + \" cells=\" + cellCount);\n }\n }\n private static long streamPatterns(Connection source, Connection target, StreamingLoadRequest request)\n throws Exception {\n String sql = \"\"\"\n insert into historical_bootstrap_pattern_aggregates(\n pattern_id,dataset_id,derivation_version,pattern_kind,subject_key,start_h3_r5,end_h3_r5,\n evidence_count,qualified_evidence_count,first_seen_at,last_seen_at,minimum_altitude_ft,\n maximum_altitude_ft,mean_altitude_sum_ft,altitude_evidence_count,duration_seconds_sum,\n observation_count_sum,adsb_icao_count,adsr_icao_count,tisb_icao_count,mlat_count,created_at)\n values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)\n \"\"\";\n try (PreparedStatement read = source.prepareStatement(\"\"\"\n select pattern_kind,subject_key,start_h3_r5,end_h3_r5,evidence_count,clean_evidence_count,\n first_occurrence_ms,last_occurrence_ms,minimum_altitude_ft,maximum_altitude_ft,\n mean_altitude_sum_ft,altitude_evidence_count,duration_seconds_sum,observation_count_sum,source_counts\n from p7_local_patterns where dataset_id=?\n \"\"\");\n PreparedStatement write = target.prepareStatement(sql)) {\n read.setFetchSize(SOURCE_FETCH_SIZE);\n read.setString(1, request.datasetId().toString());\n int batch = 0;\n long written = 0L;\n try (ResultSet rs = read.executeQuery()) {\n while (rs.next()) {\n String kind = rs.getString(1);\n String subject = rs.getString(2);\n String start = rs.getString(3);\n String end = rs.getString(4);\n UUID patternId = patternUuid(request.datasetId(), request.derivationVersion(),\n kind, subject, start, end);\n long[] sources = decodeSourceCounts(rs.getString(15));\n\n int index = 1;\n write.setObject(index++, patternId);\n write.setObject(index++, request.datasetId());\n write.setString(index++, request.derivationVersion());\n write.setString(index++, kind);\n write.setString(index++, subject);\n write.setString(index++, start);\n write.setString(index++, end);\n write.setLong(index++, rs.getLong(5));\n write.setLong(index++, rs.getLong(6));\n write.setTimestamp(index++, Timestamp.from(Instant.ofEpochMilli(rs.getLong(7))));\n write.setTimestamp(index++, Timestamp.from(Instant.ofEpochMilli(rs.getLong(8))));\n write.setInt(index++, rs.getInt(9));\n write.setInt(index++, rs.getInt(10));\n write.setLong(index++, rs.getLong(11));\n write.setLong(index++, rs.getLong(12));\n write.setLong(index++, rs.getLong(13));\n write.setLong(index++, rs.getLong(14));\n write.setLong(index++, sources[0]);\n write.setLong(index++, sources[1]);\n write.setLong(index++, sources[2]);\n write.setLong(index++, sources[3]);\n write.setTimestamp(index, Timestamp.from(request.loadedAt()));\n write.addBatch();\n written++;\n if (++batch == BATCH_SIZE) {\n write.executeBatch();\n batch = 0;\n }\n }\n }\n if (batch != 0) write.executeBatch();\n return written;\n }\n }\n\n private static long streamHourBins(Connection source, Connection target, StreamingLoadRequest request)\n throws Exception {\n try (PreparedStatement read = source.prepareStatement(\"\"\"\n select pattern_kind,subject_key,start_h3_r5,end_h3_r5,hour_of_week,evidence_count\n from p7_local_hour_bins where dataset_id=?\n \"\"\");\n PreparedStatement write = target.prepareStatement(\n \"insert into historical_bootstrap_hour_bins(pattern_id,hour_of_week,evidence_count) values(?,?,?)\")) {\n read.setFetchSize(SOURCE_FETCH_SIZE);\n read.setString(1, request.datasetId().toString());\n int batch = 0;\n long written = 0L;\n try (ResultSet rs = read.executeQuery()) {\n while (rs.next()) {\n write.setObject(1, patternUuid(\n request.datasetId(), request.derivationVersion(),\n rs.getString(1), rs.getString(2), rs.getString(3), rs.getString(4)));\n write.setShort(2, rs.getShort(5));\n write.setLong(3, rs.getLong(6));\n write.addBatch();\n written++;\n if (++batch == BATCH_SIZE) {\n write.executeBatch();\n batch = 0;\n }\n }\n }\n if (batch != 0) write.executeBatch();\n return written;\n }\n }\n\n private static HistoricalCorridorPackWriter.WriteResult streamCorridors(\n Connection source, Connection target, StreamingLoadRequest request\n ) throws SQLException {\n HistoricalCorridorPackWriter.WriteResult result =\n HistoricalCorridorPackWriter.streamLocal(source, target, request.datasetId(), request.loadedAt());\n if (result.cellCount() < 0L || result.packCount() < 0L\n || (result.cellCount() > 0 && result.packCount() == 0)) {\n throw new IllegalStateException(\"invalid compact corridor persistence result\");\n }\n return result;\n }\n\n private static void streamHoldoutSegments(Connection source, Connection target, StreamingLoadRequest request)\n throws Exception {\n try (PreparedStatement read = source.prepareStatement(\"\"\"\n select source_date,segment_id,icao24,callsign,first_observed_ms,last_observed_ms,start_h3_r5,end_h3_r5,\n minimum_altitude_ft,maximum_altitude_ft,mean_altitude_ft,altitude_evidence_count,\n duration_seconds,observation_count,source_counts,corridor_overflow\n from p7_local_holdout where dataset_id=?\n \"\"\");\n PreparedStatement write = target.prepareStatement(\"\"\"\n insert into historical_bootstrap_holdout_segments(\n dataset_id,source_date,segment_id,icao24,callsign,first_observed_at,last_observed_at,\n start_h3_r5,end_h3_r5,minimum_altitude_ft,maximum_altitude_ft,mean_altitude_ft,\n altitude_evidence_count,duration_seconds,observation_count,adsb_icao_count,adsr_icao_count,\n tisb_icao_count,mlat_count,delete_after,corridor_overflow)\n values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)\n \"\"\")) {\n read.setFetchSize(SOURCE_FETCH_SIZE);\n read.setString(1, request.datasetId().toString());\n int batch = 0;\n try (ResultSet rs = read.executeQuery()) {\n while (rs.next()) {\n long[] sources = decodeSourceCounts(rs.getString(15));\n int index = 1;\n write.setObject(index++, request.datasetId());\n write.setDate(index++, rs.getDate(1));\n write.setString(index++, rs.getString(2));\n write.setString(index++, rs.getString(3));\n write.setString(index++, persistableCallsignOrNull(rs.getString(4)));\n write.setTimestamp(index++, Timestamp.from(Instant.ofEpochMilli(rs.getLong(5))));\n write.setTimestamp(index++, Timestamp.from(Instant.ofEpochMilli(rs.getLong(6))));\n write.setString(index++, rs.getString(7));\n write.setString(index++, rs.getString(8));\n write.setInt(index++, rs.getInt(9));\n write.setInt(index++, rs.getInt(10));\n write.setInt(index++, rs.getInt(11));\n write.setLong(index++, rs.getLong(12));\n write.setLong(index++, rs.getLong(13));\n write.setLong(index++, rs.getLong(14));\n write.setLong(index++, sources[0]);\n write.setLong(index++, sources[1]);\n write.setLong(index++, sources[2]);\n write.setLong(index++, sources[3]);\n write.setTimestamp(index++, Timestamp.from(request.holdoutDeleteAfter()));\n write.setBoolean(index, rs.getBoolean(16));\n write.addBatch();\n if (++batch == BATCH_SIZE) {\n write.executeBatch();\n batch = 0;\n }\n }\n }\n if (batch != 0) write.executeBatch();\n }\n }\n\n private static void streamHoldoutCorridorCells(\n Connection source,\n Connection target,\n StreamingLoadRequest request\n ) throws SQLException {\n try (PreparedStatement write = target.prepareStatement(\"\"\"\n insert into historical_bootstrap_holdout_corridor_cells(\n dataset_id,segment_id,sequence_index,corridor_h3_r7,progress_permille,\n minimum_altitude_ft,maximum_altitude_ft,altitude_sum_ft,altitude_evidence_count)\n values(?,?,?,?,?,?,?,?,?)\n \"\"\")) {\n String dataset = request.datasetId().toString();\n String afterSegmentId = null;\n int batch = 0;\n\n while (true) {\n List segmentPage = readHoldoutSegmentPage(source, dataset, afterSegmentId);\n if (segmentPage.isEmpty()) {\n break;\n }\n\n String firstSegmentId = segmentPage.getFirst();\n String lastSegmentId = segmentPage.getLast();\n int pageHardLimit = Math.addExact(\n Math.multiplyExact(segmentPage.size(), HistoricalCorridorModel.MAX_UNIQUE_CELLS),\n 1);\n int pageCellCount = 0;\n\n try (PreparedStatement read = source.prepareStatement(\"\"\"\n select segment_id,sequence_index,corridor_h3_r7,progress_permille,\n minimum_altitude_ft,maximum_altitude_ft,altitude_sum_ft,altitude_evidence_count\n from p7_local_holdout_corridor\n where dataset_id=? and segment_id>=? and segment_id<=?\n order by segment_id,sequence_index\n limit ?\n \"\"\")) {\n read.setFetchSize(SOURCE_FETCH_SIZE);\n read.setString(1, dataset);\n read.setString(2, firstSegmentId);\n read.setString(3, lastSegmentId);\n read.setInt(4, pageHardLimit);\n\n try (ResultSet rs = read.executeQuery()) {\n while (rs.next()) {\n pageCellCount++;\n if (pageCellCount >= pageHardLimit) {\n throw new IllegalStateException(\n \"bounded HOLDOUT corridor staging page exceeded model maximum between \"\n + firstSegmentId + \" and \" + lastSegmentId);\n }\n\n write.setObject(1, request.datasetId());\n write.setString(2, rs.getString(1));\n write.setShort(3, rs.getShort(2));\n write.setString(4, rs.getString(3));\n write.setShort(5, rs.getShort(4));\n write.setInt(6, rs.getInt(5));\n write.setInt(7, rs.getInt(6));\n write.setLong(8, rs.getLong(7));\n write.setLong(9, rs.getLong(8));\n write.addBatch();\n\n if (++batch == BATCH_SIZE) {\n write.executeBatch();\n batch = 0;\n }\n }\n }\n }\n\n afterSegmentId = lastSegmentId;\n }\n\n if (batch != 0) {\n write.executeBatch();\n }\n }\n }\n private static void insertDataset(Connection connection, StreamingLoadRequest request) throws SQLException {\n DatasetWindow window = request.manifest().window();\n try (PreparedStatement statement = connection.prepareStatement(\"\"\"\n insert into historical_bootstrap_datasets(\n dataset_id,dataset_purpose,provider,source_repository,source_year,license_identifier,\n from_date,to_date,as_of_date,lookback_days,learning_end,embargo_date,holdout_start,\n importer_version,parser_version,archive_safety_profile,eligibility_version,segmentation_version,\n derivation_version,semantic_fingerprint,coverage_version,coverage_hash,manifest_sha256,state,\n scoring_eligible_from,scoring_expires_at,created_at)\n values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)\n \"\"\")) {\n int index = 1;\n statement.setObject(index++, request.datasetId());\n statement.setString(index++, window.purpose().name());\n statement.setString(index++, request.manifest().provider());\n statement.setString(index++, sourceRepository(request.manifest()));\n statement.setShort(index++, (short) sourceYear(request.manifest()));\n statement.setString(index++, HistoricalBootstrapPostgresPersistence.LICENSE_IDENTIFIER);\n statement.setDate(index++, Date.valueOf(window.fromDate()));\n statement.setDate(index++, Date.valueOf(window.toDate()));\n setDate(statement, index++, window.asOfDate());\n if (window.lookbackDays() == null) {\n statement.setObject(index++, null);\n } else {\n statement.setInt(index++, window.lookbackDays());\n }\n setDate(statement, index++, window.learningEnd());\n setDate(statement, index++, window.embargoDate());\n setDate(statement, index++, window.holdoutStart());\n statement.setString(index++, IMPORTER_VERSION);\n statement.setString(index++, GateBSemanticFingerprint.PARSER_VERSION);\n statement.setString(index++, ArchiveSafetyLimits.defaults().semanticFingerprintComponent());\n statement.setString(index++, HistoricalEligibilityV1.VERSION);\n statement.setString(index++, HistoricalSegmenterV1.VERSION);\n statement.setString(index++, request.derivationVersion());\n statement.setString(index++, request.semanticFingerprint());\n statement.setString(index++, request.coverageVersion());\n statement.setString(index++, request.coverageHash());\n statement.setString(index++, request.manifest().manifestSha256());\n statement.setString(index++, \"PLANNED\");\n setTimestamp(statement, index++, request.scoringEligibleFrom());\n setTimestamp(statement, index++, request.scoringExpiresAt());\n statement.setTimestamp(index, Timestamp.from(request.loadedAt()));\n statement.executeUpdate();\n }\n }\n\n private static void insertDaysAndAssets(\n Connection connection,\n StreamingLoadRequest request,\n Map localDays\n ) throws SQLException {\n for (ResolvedSourceDay manifestDay : request.manifest().days()) {\n LocalDay local = localDays.get(manifestDay.sourceDate());\n HistoricalBootstrapPostgresPersistence.DayLoadMetrics metrics =\n request.dayMetrics().get(manifestDay.sourceDate());\n SourceRelease release = manifestDay.selectedRelease();\n ResolvedArchiveFamily family = manifestDay.selectedFamily();\n\n try (PreparedStatement statement = connection.prepareStatement(\"\"\"\n insert into historical_bootstrap_days(\n dataset_id,source_date,experiment_role,selected_release_id,selected_release_tag,\n selected_release_variant,terminal_quality,reason_code,selected_archive_family,selected_bytes,\n parsed_trace_count,trace_parse_failure_count,malformed_point_count,accepted_point_count,\n rejected_point_count,finalized_segment_count,day_manifest_sha256,day_delta_checksum,qualified_at)\n values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)\n \"\"\")) {\n statement.setObject(1, request.datasetId());\n statement.setDate(2, Date.valueOf(manifestDay.sourceDate()));\n statement.setString(3, manifestDay.role().name());\n if (release == null) {\n statement.setObject(4, null);\n statement.setString(5, null);\n statement.setString(6, null);\n } else {\n statement.setLong(4, release.releaseId());\n statement.setString(5, release.tag());\n statement.setString(6, release.variant().name());\n }\n statement.setString(7, local.quality().name());\n statement.setString(8, metrics.reasonCode());\n statement.setString(9, family == null ? null : family.kind().name());\n statement.setLong(10, family == null ? 0L : family.totalBytes());\n statement.setInt(11, metrics.parsedTraceCount());\n statement.setInt(12, metrics.traceParseFailureCount());\n statement.setLong(13, metrics.malformedPointCount());\n statement.setLong(14, metrics.acceptedPointCount());\n statement.setLong(15, metrics.rejectedPointCount());\n statement.setLong(16, metrics.finalizedSegmentCount());\n statement.setString(17, local.dayManifestSha256());\n statement.setString(18, local.dayDeltaChecksum());\n statement.setTimestamp(19, Timestamp.from(metrics.qualifiedAt()));\n statement.executeUpdate();\n }\n\n if (family != null) {\n short ordinal = 0;\n for (SourceAsset asset : family.assets()) {\n if (asset.sha256() == null) {\n throw new IllegalArgumentException(\"selected source asset is missing SHA-256\");\n }\n try (PreparedStatement statement = connection.prepareStatement(\"\"\"\n insert into historical_bootstrap_assets(\n dataset_id,source_date,asset_ordinal,release_asset_id,filename,byte_size,\n upstream_sha256,verified_sha256,source_repository,release_tag,archive_family)\n values(?,?,?,?,?,?,?,?,?,?,?)\n \"\"\")) {\n statement.setObject(1, request.datasetId());\n statement.setDate(2, Date.valueOf(manifestDay.sourceDate()));\n statement.setShort(3, ordinal++);\n statement.setLong(4, asset.assetId());\n statement.setString(5, asset.name());\n statement.setLong(6, asset.byteSize());\n statement.setString(7, asset.sha256());\n statement.setString(8,\n local.quality() == LocalDayCommitStore.EvidenceQualityStatus.QUALIFIED\n ? asset.sha256() : null);\n statement.setString(9, release.repository());\n statement.setString(10, release.tag());\n statement.setString(11, family.kind().name());\n statement.executeUpdate();\n }\n }\n }\n }\n }\n\n // The source is independently qualified before opening the PostgreSQL transaction.\n // Every writer uses plain INSERT (no ON CONFLICT / ignored rows), checked JDBC batches,\n // immutable PK/FK/check constraints and a single atomic commit. Comparing the counters\n // accumulated during the insert stream with the independently measured source counts\n // avoids two full scans of multi-GB target tables and expansion of R7 corridor arrays.\n static void validateStreamedCounts(LocalSummary summary, StreamedCounts written) {\n requireCount(\"staged bootstrap pattern\", summary.patternCount(), written.patternCount());\n requireCount(\"staged bootstrap hour-bin\", summary.hourBinCount(), written.hourBinCount());\n requireCount(\"staged bootstrap corridor cell\", summary.corridorCount(), written.corridorCellCount());\n if (written.corridorCellCount() > 0 && written.corridorPackCount() < 1) {\n throw new IllegalStateException(\"staged bootstrap corridor packs missing\");\n }\n }\n\n private static void validateStaged(Connection connection, UUID datasetId,\n LocalSummary summary, StreamedCounts written)\n throws SQLException {\n validateStreamedCounts(summary, written);\n assertCount(connection, \"historical_bootstrap_days\", datasetId, summary.dayCount());\n assertCount(connection, \"historical_bootstrap_holdout_segments\", datasetId, summary.holdoutCount());\n assertCount(connection, \"historical_bootstrap_holdout_corridor_cells\", datasetId, summary.holdoutCorridorCount());\n\n if (targetExists(connection,\n \"select 1 from historical_bootstrap_ready_pattern_aggregates where dataset_id=? limit 1\", datasetId)\n || targetExists(connection,\n \"select 1 from historical_bootstrap_ready_route_corridor_aggregates where dataset_id=? limit 1\",\n datasetId)) {\n throw new IllegalStateException(\"STAGED bootstrap evidence became visible through READY seam\");\n }\n }\n\n private static void validateReady(Connection connection, UUID datasetId, LocalSummary summary)\n throws SQLException {\n long readyDatasets = targetScalar(connection,\n \"select count(*) from historical_bootstrap_ready_datasets where dataset_id=?\", datasetId);\n // STAGED counts have already been checked against the independently qualified\n // local state. READY views only add the READY dataset predicate, so a bounded\n // existence probe proves visibility without re-counting every row / R7 child.\n boolean patternsVisible = targetExists(connection,\n \"select 1 from historical_bootstrap_ready_pattern_aggregates where dataset_id=? limit 1\",\n datasetId);\n boolean hourBinsVisible = targetExists(connection, \"\"\"\n select 1 from historical_bootstrap_ready_hour_bins h\n join historical_bootstrap_pattern_aggregates p on p.pattern_id=h.pattern_id\n where p.dataset_id=? limit 1\n \"\"\", datasetId);\n boolean corridorsVisible = targetExists(connection,\n \"select 1 from historical_bootstrap_ready_route_corridor_aggregates where dataset_id=? limit 1\",\n datasetId);\n if (readyDatasets != 1L\n || patternsVisible != (summary.patternCount() > 0)\n || hourBinsVisible != (summary.hourBinCount() > 0)\n || corridorsVisible != (summary.corridorCount() > 0)) {\n throw new IllegalStateException(\"READY bootstrap read seam does not match validated streamed snapshot\");\n }\n }\n\n private static boolean targetExists(Connection connection, String sql, UUID datasetId)\n throws SQLException {\n try (PreparedStatement statement = connection.prepareStatement(sql)) {\n statement.setObject(1, datasetId);\n try (ResultSet rs = statement.executeQuery()) {\n return rs.next();\n }\n }\n }\n\n record StreamedCounts(long patternCount, long hourBinCount, long corridorCellCount, long corridorPackCount) {\n }\n\n private static void rejectExistingDataset(Connection connection, UUID datasetId) throws SQLException {\n try (PreparedStatement statement = connection.prepareStatement(\n \"select state from historical_bootstrap_datasets where dataset_id=?\")) {\n statement.setObject(1, datasetId);\n try (ResultSet rs = statement.executeQuery()) {\n if (rs.next()) {\n throw new IllegalStateException(\"historical bootstrap dataset ID already exists and is immutable: \"\n + datasetId + \" state=\" + rs.getString(1));\n }\n }\n }\n }\n\n private static void transition(Connection connection, UUID datasetId, String state, Instant readyAt)\n throws SQLException {\n String sql = readyAt == null\n ? \"update historical_bootstrap_datasets set state=? where dataset_id=?\"\n : \"update historical_bootstrap_datasets set state=?,ready_at=? where dataset_id=?\";\n try (PreparedStatement statement = connection.prepareStatement(sql)) {\n statement.setString(1, state);\n if (readyAt == null) {\n statement.setObject(2, datasetId);\n } else {\n statement.setTimestamp(2, Timestamp.from(readyAt));\n statement.setObject(3, datasetId);\n }\n if (statement.executeUpdate() != 1) {\n throw new IllegalStateException(\"dataset lifecycle transition failed: \" + state);\n }\n }\n }\n\n private static void assertCount(Connection connection, String table, UUID datasetId, long expected)\n throws SQLException {\n long actual = targetScalar(connection, \"select count(*) from \" + table + \" where dataset_id=?\", datasetId);\n if (actual != expected) {\n throw new IllegalStateException(table + \" cardinality mismatch: expected=\" + expected + \" actual=\" + actual);\n }\n }\n\n private static long scalar(Connection connection, String sql, String dataset) throws SQLException {\n try (PreparedStatement statement = connection.prepareStatement(sql)) {\n statement.setString(1, dataset);\n try (ResultSet rs = statement.executeQuery()) {\n if (!rs.next()) throw new IllegalStateException(\"scalar query returned no row\");\n return rs.getLong(1);\n }\n }\n }\n\n private static long scalar(Connection connection, String sql, String first, String second) throws SQLException {\n try (PreparedStatement statement = connection.prepareStatement(sql)) {\n statement.setString(1, first);\n statement.setString(2, second);\n try (ResultSet rs = statement.executeQuery()) {\n if (!rs.next()) throw new IllegalStateException(\"scalar query returned no row\");\n return rs.getLong(1);\n }\n }\n }\n\n private static long targetScalar(Connection connection, String sql, UUID datasetId) throws SQLException {\n try (PreparedStatement statement = connection.prepareStatement(sql)) {\n statement.setObject(1, datasetId);\n try (ResultSet rs = statement.executeQuery()) {\n if (!rs.next()) throw new IllegalStateException(\"scalar query returned no row\");\n return rs.getLong(1);\n }\n }\n }\n\n private static long targetScalar(Connection connection, String sql, UUID first, UUID second)\n throws SQLException {\n try (PreparedStatement statement = connection.prepareStatement(sql)) {\n statement.setObject(1, first);\n statement.setObject(2, second);\n try (ResultSet rs = statement.executeQuery()) {\n if (!rs.next()) throw new IllegalStateException(\"scalar query returned no row\");\n return rs.getLong(1);\n }\n }\n }\n\n private static UUID patternUuid(\n UUID datasetId,\n String derivationVersion,\n String kind,\n String subject,\n String start,\n String end\n ) {\n return uuidV5(datasetId, derivationVersion + \"|\" + kind + \"|\" + subject + \"|\" + start + \"|\" + end);\n }\n\n private static UUID uuidV5(UUID namespace, String name) {\n try {\n MessageDigest sha1 = MessageDigest.getInstance(\"SHA-1\");\n sha1.update(ByteBuffer.allocate(16)\n .putLong(namespace.getMostSignificantBits())\n .putLong(namespace.getLeastSignificantBits())\n .array());\n byte[] bytes = sha1.digest(name.getBytes(StandardCharsets.UTF_8));\n bytes[6] = (byte) ((bytes[6] & 0x0f) | 0x50);\n bytes[8] = (byte) ((bytes[8] & 0x3f) | 0x80);\n ByteBuffer buffer = ByteBuffer.wrap(bytes);\n return new UUID(buffer.getLong(), buffer.getLong());\n } catch (NoSuchAlgorithmException exception) {\n throw new IllegalStateException(\"SHA-1 unavailable\", exception);\n }\n }\n\n private static long[] decodeSourceCounts(String value) {\n if (value == null) throw new IllegalStateException(\"missing compact source counts\");\n String[] values = value.split(\",\", -1);\n if (values.length != 4) throw new IllegalStateException(\"invalid compact source-count encoding\");\n try {\n return new long[] {\n Long.parseLong(values[0]),\n Long.parseLong(values[1]),\n Long.parseLong(values[2]),\n Long.parseLong(values[3])\n };\n } catch (NumberFormatException exception) {\n throw new IllegalStateException(\"invalid compact source-count encoding\", exception);\n }\n }\n\n private static String sourceRepository(BootstrapSourceManifest manifest) {\n List repositories = manifest.days().stream()\n .flatMap(day -> day.candidateReleases().stream())\n .map(SourceRelease::repository)\n .distinct()\n .sorted()\n .toList();\n if (repositories.size() != 1) {\n throw new IllegalArgumentException(\"initial P7 persistence requires exactly one frozen source repository\");\n }\n return repositories.getFirst();\n }\n\n private static int sourceYear(BootstrapSourceManifest manifest) {\n List years = manifest.days().stream()\n .map(day -> day.sourceDate().getYear())\n .distinct()\n .sorted()\n .toList();\n if (years.size() != 1) {\n throw new IllegalArgumentException(\"initial P7 persistence requires one source year per dataset\");\n }\n return years.getFirst();\n }\n\n private static void requireCount(String name, long expected, long actual) {\n if (expected != actual) {\n throw new IllegalStateException(name + \" count mismatch: expected=\" + expected + \" actual=\" + actual);\n }\n }\n\n private static void requireHash(String value, String name) {\n if (value == null || !value.matches(\"[0-9a-f]{64}\")) {\n throw new IllegalArgumentException(name + \" must be lowercase SHA-256\");\n }\n }\n\n private static void requireText(String value, int max, String name) {\n if (value == null || value.isBlank() || value.length() > max) {\n throw new IllegalArgumentException(name + \" is missing or unbounded\");\n }\n }\n\n private static void setDate(PreparedStatement statement, int index, LocalDate value) throws SQLException {\n if (value == null) statement.setDate(index, null);\n else statement.setDate(index, Date.valueOf(value));\n }\n\n private static void setTimestamp(PreparedStatement statement, int index, Instant value) throws SQLException {\n if (value == null) statement.setTimestamp(index, null);\n else statement.setTimestamp(index, Timestamp.from(value));\n }\n\n private record LocalDay(\n LocalDate sourceDate,\n String dayManifestSha256,\n String semanticFingerprint,\n String dayDeltaChecksum,\n LocalDayCommitStore.EvidenceQualityStatus quality\n ) {\n }\n\n private static final class HeapSampler {\n private long peakBytes = usedHeap();\n\n void sample(long rowCount) {\n if ((rowCount & 0x3fffL) == 0L) {\n peakBytes = Math.max(peakBytes, usedHeap());\n }\n }\n\n long peakBytes() {\n return Math.max(peakBytes, usedHeap());\n }\n\n private static long usedHeap() {\n Runtime runtime = Runtime.getRuntime();\n return runtime.totalMemory() - runtime.freeMemory();\n }\n }\n\n record LocalSummary(\n int dayCount,\n long patternCount,\n long hourBinCount,\n long corridorCount,\n long holdoutCount,\n long holdoutCorridorCount,\n long normalizedHoldoutCallsignCount,\n long finalCarryCount,\n long maxObservedHeapBytes\n ) {\n }\n\n record StreamingLoadRequest(\n Path stateDatabase,\n UUID datasetId,\n BootstrapSourceManifest manifest,\n String semanticFingerprint,\n String coverageVersion,\n String coverageHash,\n String derivationVersion,\n Map dayMetrics,\n Instant loadedAt,\n Instant holdoutDeleteAfter,\n Instant scoringEligibleFrom,\n Instant scoringExpiresAt\n ) {\n StreamingLoadRequest {\n dayMetrics = dayMetrics == null ? Map.of() : Map.copyOf(new TreeMap<>(dayMetrics));\n }\n }\n\n record StreamingLoadResult(\n UUID datasetId,\n int dayCount,\n long patternCount,\n long hourBinCount,\n long corridorCount,\n long holdoutCount,\n long holdoutCorridorCount,\n Instant readyAt\n ) {\n }\n}\n" }, { "path": "backend/historical-bootstrap-tool/src/main/java/hu/gombos/flightspotter/historicalbootstrap/HistoricalBootstrapRetentionMaintenance.java", "encoding": "UTF-8", "sha256": "aaf380859b9af98f0ae755eb6219bf65d1d2b786a09717ab3f1999cd2a0f1ca9", "content": "package hu.gombos.flightspotter.historicalbootstrap;\n\nimport java.sql.Connection;\nimport java.sql.PreparedStatement;\nimport java.sql.ResultSet;\nimport java.sql.Timestamp;\nimport java.time.Instant;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.Objects;\nimport java.util.UUID;\n\n/**\n * Bounded lifecycle maintenance for heavy historical scoring evidence.\n *\n *

The dataset metadata row is intentionally retained because immutable regional truth seals\n * reference it for audit/provenance. Once scoring retention expires, the dataset transitions\n * READY -> RETIRED -> EXPIRED and only its heavy child evidence is physically deleted.

\n */\nfinal class HistoricalBootstrapRetentionMaintenance {\n private HistoricalBootstrapRetentionMaintenance() {\n }\n\n static RetentionResult purgeExpired(\n Connection connection,\n Instant now,\n int maximumDatasets\n ) throws Exception {\n Objects.requireNonNull(connection, \"connection\");\n Objects.requireNonNull(now, \"now\");\n if (maximumDatasets < 1 || maximumDatasets > 8) {\n throw new IllegalArgumentException(\"maximumDatasets must be between 1 and 8\");\n }\n\n List candidates = candidates(connection, now, maximumDatasets);\n int purged = 0;\n long deletedRows = 0L;\n for (UUID datasetId : candidates) {\n PurgeResult result = purgeOne(connection, datasetId, now);\n if (result.purged()) {\n purged++;\n deletedRows += result.deletedRows();\n }\n }\n return new RetentionResult(candidates.size(), purged, deletedRows);\n }\n\n private static List candidates(\n Connection connection,\n Instant now,\n int maximumDatasets\n ) throws Exception {\n try (PreparedStatement statement = connection.prepareStatement(\"\"\"\n select dataset_id\n from historical_bootstrap_datasets\n where dataset_purpose='BOOTSTRAP_SCORING'\n and state='READY'\n and scoring_expires_at is not null\n and scoring_expires_at <= ?\n order by scoring_expires_at, dataset_id\n limit ?\n \"\"\")) {\n statement.setTimestamp(1, Timestamp.from(now));\n statement.setInt(2, maximumDatasets);\n try (ResultSet result = statement.executeQuery()) {\n List ids = new ArrayList<>();\n while (result.next()) {\n ids.add(result.getObject(1, UUID.class));\n }\n return List.copyOf(ids);\n }\n }\n }\n\n private static PurgeResult purgeOne(\n Connection connection,\n UUID datasetId,\n Instant now\n ) throws Exception {\n boolean previousAutoCommit = connection.getAutoCommit();\n connection.setAutoCommit(false);\n try {\n if (!lockEligible(connection, datasetId, now)) {\n connection.rollback();\n return new PurgeResult(false, 0L);\n }\n\n try (PreparedStatement retire = connection.prepareStatement(\"\"\"\n update historical_bootstrap_datasets\n set state='RETIRED', retired_at=?\n where dataset_id=? and state='READY'\n \"\"\")) {\n retire.setTimestamp(1, Timestamp.from(now));\n retire.setObject(2, datasetId);\n if (retire.executeUpdate() != 1) {\n throw new IllegalStateException(\n \"retention READY -> RETIRED transition lost race\");\n }\n }\n\n try (PreparedStatement expire = connection.prepareStatement(\"\"\"\n update historical_bootstrap_datasets\n set state='EXPIRED'\n where dataset_id=? and state='RETIRED'\n \"\"\")) {\n expire.setObject(1, datasetId);\n if (expire.executeUpdate() != 1) {\n throw new IllegalStateException(\n \"retention RETIRED -> EXPIRED transition failed\");\n }\n }\n\n long deleted = 0L;\n deleted += delete(connection, \"historical_bootstrap_route_corridor_packs\", datasetId);\n deleted += delete(connection, \"historical_bootstrap_route_corridor_aggregates\", datasetId);\n deleted += delete(connection, \"historical_bootstrap_holdout_segments\", datasetId);\n deleted += delete(connection, \"historical_bootstrap_pattern_aggregates\", datasetId);\n deleted += delete(connection, \"historical_bootstrap_assets\", datasetId);\n deleted += delete(connection, \"historical_bootstrap_days\", datasetId);\n\n connection.commit();\n return new PurgeResult(true, deleted);\n } catch (Exception failure) {\n connection.rollback();\n throw failure;\n } finally {\n connection.setAutoCommit(previousAutoCommit);\n }\n }\n\n private static boolean lockEligible(\n Connection connection,\n UUID datasetId,\n Instant now\n ) throws Exception {\n try (PreparedStatement statement = connection.prepareStatement(\"\"\"\n select 1\n from historical_bootstrap_datasets\n where dataset_id=?\n and dataset_purpose='BOOTSTRAP_SCORING'\n and state='READY'\n and scoring_expires_at is not null\n and scoring_expires_at <= ?\n for update\n \"\"\")) {\n statement.setObject(1, datasetId);\n statement.setTimestamp(2, Timestamp.from(now));\n try (ResultSet result = statement.executeQuery()) {\n return result.next();\n }\n }\n }\n\n private static long delete(\n Connection connection,\n String table,\n UUID datasetId\n ) throws Exception {\n try (PreparedStatement statement =\n connection.prepareStatement(\"delete from \" + table + \" where dataset_id=?\")) {\n statement.setObject(1, datasetId);\n return statement.executeUpdate();\n }\n }\n\n record RetentionResult(\n int selectedDatasets,\n int purgedDatasets,\n long deletedRows\n ) {\n }\n\n private record PurgeResult(boolean purged, long deletedRows) {\n }\n}\n" }, { "path": "backend/historical-bootstrap-tool/src/main/java/hu/gombos/flightspotter/historicalbootstrap/HistoricalCorridorEvidence.java", "encoding": "UTF-8", "sha256": "01ac004daaf9d9b3b31f13a57dfe2d0ab6d95cc5233403f3f8bba714a809afb0", "content": "package hu.gombos.flightspotter.historicalbootstrap;\n\nimport java.time.Duration;\nimport java.time.Instant;\nimport java.util.List;\n\n/** Bounded H3 R7 route-shape evidence retained by the P7 Hungary/v3 semantic generation. */\nfinal class HistoricalCorridorModel {\n static final String VERSION = \"p7-corridor-v1\";\n static final int MAX_UNIQUE_CELLS = 512;\n\n private HistoricalCorridorModel() {\n }\n\n static List finalizeCells(\n List cells,\n boolean overflow,\n Instant segmentStart,\n Instant segmentEnd\n ) {\n if (overflow || cells == null || cells.isEmpty()) return List.of();\n if (segmentStart == null || segmentEnd == null || segmentEnd.isBefore(segmentStart)) {\n throw new IllegalArgumentException(\"valid segment bounds are required for corridor finalization\");\n }\n long totalMillis = Duration.between(segmentStart, segmentEnd).toMillis();\n java.util.ArrayList result = new java.util.ArrayList<>(cells.size());\n for (int index = 0; index < cells.size(); index++) {\n HistoricalCorridorCellCarry cell = cells.get(index);\n long elapsedMillis = Duration.between(segmentStart, cell.firstEnteredAt()).toMillis();\n int progress = totalMillis <= 0L ? 0\n : Math.toIntExact(Math.max(0L, Math.min(1000L, Math.round(elapsedMillis * 1000d / totalMillis))));\n result.add(new HistoricalCorridorCellEvidence(\n index,\n cell.h3R7(),\n progress,\n cell.minimumAltitudeFt(),\n cell.maximumAltitudeFt(),\n cell.altitudeSumFt(),\n cell.altitudeEvidenceCount()\n ));\n }\n return List.copyOf(result);\n }\n\n static boolean validR7(String value) {\n return value != null && value.matches(\"87[0-9a-f]{13}\");\n }\n}\n\nrecord HistoricalCorridorCellCarry(\n String h3R7,\n Instant firstEnteredAt,\n int minimumAltitudeFt,\n int maximumAltitudeFt,\n long altitudeSumFt,\n long altitudeEvidenceCount\n) {\n HistoricalCorridorCellCarry {\n if (!HistoricalCorridorModel.validR7(h3R7)\n || firstEnteredAt == null\n || minimumAltitudeFt > maximumAltitudeFt\n || altitudeEvidenceCount < 1L) {\n throw new IllegalArgumentException(\"invalid corridor carry cell\");\n }\n }\n\n String canonical() {\n return h3R7 + \"|\" + firstEnteredAt.toEpochMilli() + \"|\" + minimumAltitudeFt + \"|\"\n + maximumAltitudeFt + \"|\" + altitudeSumFt + \"|\" + altitudeEvidenceCount;\n }\n}\n\nrecord HistoricalCorridorCellEvidence(\n int sequenceIndex,\n String h3R7,\n int firstEntryProgressPermille,\n int minimumAltitudeFt,\n int maximumAltitudeFt,\n long altitudeSumFt,\n long altitudeEvidenceCount\n) {\n HistoricalCorridorCellEvidence {\n if (sequenceIndex < 0 || sequenceIndex >= HistoricalCorridorModel.MAX_UNIQUE_CELLS\n || !HistoricalCorridorModel.validR7(h3R7)\n || firstEntryProgressPermille < 0 || firstEntryProgressPermille > 1000\n || minimumAltitudeFt > maximumAltitudeFt\n || altitudeEvidenceCount < 1L) {\n throw new IllegalArgumentException(\"invalid finalized corridor cell\");\n }\n }\n\n String canonical() {\n return sequenceIndex + \"|\" + h3R7 + \"|\" + firstEntryProgressPermille + \"|\"\n + minimumAltitudeFt + \"|\" + maximumAltitudeFt + \"|\" + altitudeSumFt + \"|\"\n + altitudeEvidenceCount;\n }\n}\n\nrecord HistoricalRouteCorridorKey(\n String startH3R5,\n String endH3R5,\n String corridorH3R7\n) implements Comparable {\n HistoricalRouteCorridorKey {\n if (startH3R5 == null || !startH3R5.matches(\"85[0-9a-f]{13}\")\n || endH3R5 == null || !endH3R5.matches(\"85[0-9a-f]{13}\")\n || !HistoricalCorridorModel.validR7(corridorH3R7)) {\n throw new IllegalArgumentException(\"invalid route corridor key\");\n }\n }\n\n String canonical() {\n return startH3R5 + \"|\" + endH3R5 + \"|\" + corridorH3R7;\n }\n\n @Override\n public int compareTo(HistoricalRouteCorridorKey other) {\n int compared = startH3R5.compareTo(other.startH3R5);\n if (compared != 0) return compared;\n compared = endH3R5.compareTo(other.endH3R5);\n if (compared != 0) return compared;\n return corridorH3R7.compareTo(other.corridorH3R7);\n }\n}\n\nrecord HistoricalRouteCorridorStats(\n long traversalCount,\n long progressPermilleSum,\n int minimumAltitudeFt,\n int maximumAltitudeFt,\n long altitudeSumFt,\n long altitudeEvidenceCount\n) {\n HistoricalRouteCorridorStats {\n if (traversalCount < 1L\n || progressPermilleSum < 0L || progressPermilleSum > Math.multiplyExact(traversalCount, 1000L)\n || minimumAltitudeFt > maximumAltitudeFt\n || altitudeEvidenceCount < traversalCount) {\n throw new IllegalArgumentException(\"invalid route corridor aggregate\");\n }\n }\n\n static HistoricalRouteCorridorStats from(HistoricalCorridorCellEvidence cell) {\n return new HistoricalRouteCorridorStats(\n 1L,\n cell.firstEntryProgressPermille(),\n cell.minimumAltitudeFt(),\n cell.maximumAltitudeFt(),\n cell.altitudeSumFt(),\n cell.altitudeEvidenceCount()\n );\n }\n\n static HistoricalRouteCorridorStats merge(\n HistoricalRouteCorridorStats left,\n HistoricalRouteCorridorStats right\n ) {\n return new HistoricalRouteCorridorStats(\n Math.addExact(left.traversalCount(), right.traversalCount()),\n Math.addExact(left.progressPermilleSum(), right.progressPermilleSum()),\n Math.min(left.minimumAltitudeFt(), right.minimumAltitudeFt()),\n Math.max(left.maximumAltitudeFt(), right.maximumAltitudeFt()),\n Math.addExact(left.altitudeSumFt(), right.altitudeSumFt()),\n Math.addExact(left.altitudeEvidenceCount(), right.altitudeEvidenceCount())\n );\n }\n}\n" }, { "path": "backend/historical-bootstrap-tool/src/main/java/hu/gombos/flightspotter/historicalbootstrap/HistoricalCorridorPackWriter.java", "encoding": "UTF-8", "sha256": "856151e48d34c5cad4b1f10e56b92c6d45d6e7a52b7ddb2da56a5353f63fdf0c", "content": "package hu.gombos.flightspotter.historicalbootstrap;\n\nimport java.sql.Array;\nimport java.sql.Connection;\nimport java.sql.PreparedStatement;\nimport java.sql.ResultSet;\nimport java.sql.SQLException;\nimport java.sql.Timestamp;\nimport java.time.Instant;\nimport java.util.ArrayList;\nimport java.util.Comparator;\nimport java.util.LinkedHashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.UUID;\n\n/**\n * Writes exact route-corridor evidence as bounded route x R5-parent packs.\n *\n *

The source remains one exact row per R7 cell. Packing changes only PostgreSQL storage\n * representation: the READY compatibility view expands every packed cell back to the legacy row\n * contract. A pack is bounded by the H3 R5 -> R7 hierarchy (at most 49 children), and streaming\n * state is bounded to one coarse route at a time.

\n */\nfinal class HistoricalCorridorPackWriter {\n static final int MAX_R7_CELLS_PER_PARENT = 49;\n private static final int PACK_BATCH_SIZE = 256;\n\n private HistoricalCorridorPackWriter() {\n }\n\n static WriteResult streamLocal(\n Connection source,\n Connection target,\n UUID datasetId,\n Instant loadedAt\n ) throws SQLException {\n try (PreparedStatement read = source.prepareStatement(\"\"\"\n select start_h3_r5,end_h3_r5,corridor_h3_r7,traversal_count,progress_permille_sum,\n minimum_altitude_ft,maximum_altitude_ft,altitude_sum_ft,altitude_evidence_count\n from p7_local_corridor_aggregates\n where dataset_id=?\n order by start_h3_r5,end_h3_r5,corridor_h3_r7\n \"\"\");\n PackBatchWriter writer = new PackBatchWriter(target, datasetId, loadedAt)) {\n read.setFetchSize(HistoricalBootstrapPostgresStreamingPersistence.SOURCE_FETCH_SIZE);\n read.setString(1, datasetId.toString());\n\n String activeStart = null;\n String activeEnd = null;\n Map routePacks = new LinkedHashMap<>();\n long cells = 0L;\n\n try (ResultSet rs = read.executeQuery()) {\n while (rs.next()) {\n String start = rs.getString(1);\n String end = rs.getString(2);\n String r7 = rs.getString(3);\n if (activeStart != null && (!activeStart.equals(start) || !activeEnd.equals(end))) {\n writer.addAll(routePacks.values());\n routePacks.clear();\n }\n activeStart = start;\n activeEnd = end;\n\n HistoricalRouteCorridorStats stats = new HistoricalRouteCorridorStats(\n rs.getLong(4),\n rs.getLong(5),\n rs.getInt(6),\n rs.getInt(7),\n rs.getLong(8),\n rs.getLong(9));\n append(routePacks, start, end, r7, stats);\n cells++;\n }\n }\n writer.addAll(routePacks.values());\n return new WriteResult(cells, writer.packCount());\n }\n }\n\n static WriteResult insertSnapshot(\n Connection target,\n UUID datasetId,\n Instant loadedAt,\n Map corridors\n ) throws SQLException {\n List> entries =\n new ArrayList<>(corridors.entrySet());\n entries.sort(Map.Entry.comparingByKey(Comparator.naturalOrder()));\n\n try (PackBatchWriter writer = new PackBatchWriter(target, datasetId, loadedAt)) {\n String activeStart = null;\n String activeEnd = null;\n Map routePacks = new LinkedHashMap<>();\n long cells = 0L;\n\n for (Map.Entry entry : entries) {\n HistoricalRouteCorridorKey key = entry.getKey();\n if (activeStart != null\n && (!activeStart.equals(key.startH3R5()) || !activeEnd.equals(key.endH3R5()))) {\n writer.addAll(routePacks.values());\n routePacks.clear();\n }\n activeStart = key.startH3R5();\n activeEnd = key.endH3R5();\n append(routePacks, key.startH3R5(), key.endH3R5(), key.corridorH3R7(), entry.getValue());\n cells++;\n }\n writer.addAll(routePacks.values());\n return new WriteResult(cells, writer.packCount());\n }\n }\n\n private static void append(\n Map packs,\n String start,\n String end,\n String corridorH3R7,\n HistoricalRouteCorridorStats stats\n ) {\n String parent = HistoricalH3Projection.corridorParentR5(corridorH3R7);\n packs.computeIfAbsent(parent, ignored -> new PackBuilder(start, end, parent))\n .add(corridorH3R7, stats);\n }\n\n record WriteResult(long cellCount, long packCount) {\n }\n\n private static final class PackBuilder {\n private final String start;\n private final String end;\n private final String parent;\n private final List r7 = new ArrayList<>(MAX_R7_CELLS_PER_PARENT);\n private final List traversal = new ArrayList<>(MAX_R7_CELLS_PER_PARENT);\n private final List progress = new ArrayList<>(MAX_R7_CELLS_PER_PARENT);\n private final List minimumAltitude = new ArrayList<>(MAX_R7_CELLS_PER_PARENT);\n private final List maximumAltitude = new ArrayList<>(MAX_R7_CELLS_PER_PARENT);\n private final List altitudeSum = new ArrayList<>(MAX_R7_CELLS_PER_PARENT);\n private final List altitudeEvidence = new ArrayList<>(MAX_R7_CELLS_PER_PARENT);\n\n private PackBuilder(String start, String end, String parent) {\n this.start = start;\n this.end = end;\n this.parent = parent;\n }\n\n private void add(String corridorH3R7, HistoricalRouteCorridorStats stats) {\n if (r7.size() >= MAX_R7_CELLS_PER_PARENT) {\n throw new IllegalStateException(\"R5 corridor pack exceeds 49 R7 children: \" + parent);\n }\n if (r7.contains(corridorH3R7)) {\n throw new IllegalStateException(\"duplicate R7 corridor cell in pack: \" + corridorH3R7);\n }\n r7.add(corridorH3R7);\n traversal.add(stats.traversalCount());\n progress.add(stats.progressPermilleSum());\n minimumAltitude.add(stats.minimumAltitudeFt());\n maximumAltitude.add(stats.maximumAltitudeFt());\n altitudeSum.add(stats.altitudeSumFt());\n altitudeEvidence.add(stats.altitudeEvidenceCount());\n }\n }\n\n private static final class PackBatchWriter implements AutoCloseable {\n private final Connection connection;\n private final UUID datasetId;\n private final Instant loadedAt;\n private final PreparedStatement statement;\n private final List arrays = new ArrayList<>(PACK_BATCH_SIZE * 7);\n private int pending;\n private long packCount;\n private boolean closed;\n\n private PackBatchWriter(Connection connection, UUID datasetId, Instant loadedAt) throws SQLException {\n this.connection = connection;\n this.datasetId = datasetId;\n this.loadedAt = loadedAt;\n this.statement = connection.prepareStatement(\"\"\"\n insert into historical_bootstrap_route_corridor_packs(\n dataset_id,start_h3_r5,end_h3_r5,corridor_h3_r5,corridor_h3_r7,\n traversal_count,progress_permille_sum,minimum_altitude_ft,maximum_altitude_ft,\n altitude_sum_ft,altitude_evidence_count,created_at)\n values(?,?,?,?,?,?,?,?,?,?,?,?)\n \"\"\");\n }\n\n private void addAll(Iterable packs) throws SQLException {\n for (PackBuilder pack : packs) {\n add(pack);\n }\n }\n\n private void add(PackBuilder pack) throws SQLException {\n statement.setObject(1, datasetId);\n statement.setString(2, pack.start);\n statement.setString(3, pack.end);\n statement.setString(4, pack.parent);\n statement.setArray(5, array(\"varchar\", pack.r7.toArray(String[]::new)));\n statement.setArray(6, array(\"bigint\", pack.traversal.toArray(Long[]::new)));\n statement.setArray(7, array(\"bigint\", pack.progress.toArray(Long[]::new)));\n statement.setArray(8, array(\"integer\", pack.minimumAltitude.toArray(Integer[]::new)));\n statement.setArray(9, array(\"integer\", pack.maximumAltitude.toArray(Integer[]::new)));\n statement.setArray(10, array(\"bigint\", pack.altitudeSum.toArray(Long[]::new)));\n statement.setArray(11, array(\"bigint\", pack.altitudeEvidence.toArray(Long[]::new)));\n statement.setTimestamp(12, Timestamp.from(loadedAt));\n statement.addBatch();\n pending++;\n packCount++;\n if (pending >= PACK_BATCH_SIZE) {\n flush();\n }\n }\n\n private Array array(String type, Object[] values) throws SQLException {\n Array value = connection.createArrayOf(type, values);\n arrays.add(value);\n return value;\n }\n\n private long packCount() {\n return packCount;\n }\n\n private void flush() throws SQLException {\n if (pending == 0) {\n return;\n }\n SQLException failure = null;\n try {\n statement.executeBatch();\n } catch (SQLException exception) {\n failure = exception;\n throw exception;\n } finally {\n SQLException freeFailure = null;\n for (Array array : arrays) {\n try {\n array.free();\n } catch (SQLException exception) {\n if (freeFailure == null) {\n freeFailure = exception;\n } else {\n freeFailure.addSuppressed(exception);\n }\n }\n }\n arrays.clear();\n pending = 0;\n if (failure == null && freeFailure != null) {\n throw freeFailure;\n }\n }\n }\n\n @Override\n public void close() throws SQLException {\n if (closed) {\n return;\n }\n closed = true;\n SQLException failure = null;\n try {\n flush();\n } catch (SQLException exception) {\n failure = exception;\n }\n try {\n statement.close();\n } catch (SQLException exception) {\n if (failure == null) {\n failure = exception;\n } else {\n failure.addSuppressed(exception);\n }\n }\n if (failure != null) {\n throw failure;\n }\n }\n }\n}\n" }, { "path": "backend/historical-bootstrap-tool/src/main/java/hu/gombos/flightspotter/historicalbootstrap/HistoricalDayDelta.java", "encoding": "UTF-8", "sha256": "91d6f79aba656e6a54bbc0fa86a21c66cbe2c591f9c4e662e4aeff339aa6c558", "content": "package hu.gombos.flightspotter.historicalbootstrap;\n\nimport java.nio.charset.StandardCharsets;\nimport java.security.MessageDigest;\nimport java.security.NoSuchAlgorithmException;\nimport java.time.Instant;\nimport java.time.ZoneOffset;\nimport java.util.ArrayList;\nimport java.util.Comparator;\nimport java.util.HexFormat;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.NavigableMap;\nimport java.util.TreeMap;\n\n/** Sealed compact contribution set for exactly one source-day commit. */\nfinal class HistoricalDayDelta {\n static final String VERSION = \"p7-day-delta-v2-corridor\";\n\n private final DatasetWindow.DateRole role;\n private final NavigableMap patterns;\n private final NavigableMap hourBins;\n private final NavigableMap corridorAggregates;\n private final List holdoutOutcomes;\n private final int excludedSegmentCount;\n private final String checksum;\n\n private HistoricalDayDelta(\n DatasetWindow.DateRole role,\n NavigableMap patterns,\n NavigableMap hourBins,\n NavigableMap corridorAggregates,\n List holdoutOutcomes,\n int excludedSegmentCount\n ) {\n this.role = role;\n // Builder owns these collections exclusively and is sealed before this constructor is called.\n // Wrapping them in place avoids a full duplicate of the source-day aggregate state exactly at\n // the peak-memory point of a real EUROPE_V1 day.\n this.patterns = java.util.Collections.unmodifiableNavigableMap(patterns);\n this.hourBins = java.util.Collections.unmodifiableNavigableMap(hourBins);\n this.corridorAggregates = java.util.Collections.unmodifiableNavigableMap(corridorAggregates);\n this.holdoutOutcomes = java.util.Collections.unmodifiableList(holdoutOutcomes);\n this.excludedSegmentCount = excludedSegmentCount;\n this.checksum = computeChecksum();\n }\n\n static Builder builder(DatasetWindow.DateRole role) {\n return new Builder(role);\n }\n\n static HistoricalDayDelta fromSegments(DatasetWindow.DateRole role, List segments) {\n if (segments == null) throw new IllegalArgumentException(\"segments are required\");\n Builder builder = builder(role);\n segments.forEach(builder::accept);\n return builder.seal();\n }\n\n DatasetWindow.DateRole role() { return role; }\n NavigableMap patterns() { return patterns; }\n NavigableMap hourBins() { return hourBins; }\n NavigableMap corridorAggregates() { return corridorAggregates; }\n List holdoutOutcomes() { return holdoutOutcomes; }\n int excludedSegmentCount() { return excludedSegmentCount; }\n String checksum() { return checksum; }\n\n boolean hasEvidence() {\n return !patterns.isEmpty() || !hourBins.isEmpty() || !corridorAggregates.isEmpty() || !holdoutOutcomes.isEmpty();\n }\n\n private String computeChecksum() {\n try {\n MessageDigest digest = MessageDigest.getInstance(\"SHA-256\");\n ChecksumWriter writer = new ChecksumWriter(digest);\n writer.append(VERSION).append('\\n')\n .append(\"corridorModel=\").append(HistoricalCorridorModel.VERSION).append('\\n')\n .append(\"role=\").append(role.name()).append('\\n')\n .append(\"excluded=\").append(excludedSegmentCount).append('\\n');\n for (Map.Entry entry : patterns.entrySet()) {\n HistoricalPatternStats stats = entry.getValue();\n writer.append(\"pattern|\").append(entry.getKey().canonical()).append('|')\n .append(stats.evidenceCount()).append('|')\n .append(stats.cleanEvidenceCount()).append('|')\n .append(stats.firstOccurrenceAt().toEpochMilli()).append('|')\n .append(stats.lastOccurrenceAt().toEpochMilli()).append('|')\n .append(stats.minimumAltitudeFt()).append('|')\n .append(stats.maximumAltitudeFt()).append('|')\n .append(stats.meanAltitudeSumFt()).append('|')\n .append(stats.altitudeEvidenceCount()).append('|')\n .append(stats.durationSecondsSum()).append('|')\n .append(stats.observationCountSum()).append('|')\n .append(stats.sourceCounts().canonical()).append('\\n');\n }\n for (Map.Entry entry : hourBins.entrySet()) {\n writer.append(\"hour|\").append(entry.getKey().patternKey().canonical()).append('|')\n .append(entry.getKey().hourOfWeek()).append('|').append(entry.getValue()).append('\\n');\n }\n for (Map.Entry entry : corridorAggregates.entrySet()) {\n HistoricalRouteCorridorStats stats = entry.getValue();\n writer.append(\"corridor|\").append(entry.getKey().canonical()).append('|')\n .append(stats.traversalCount()).append('|')\n .append(stats.progressPermilleSum()).append('|')\n .append(stats.minimumAltitudeFt()).append('|')\n .append(stats.maximumAltitudeFt()).append('|')\n .append(stats.altitudeSumFt()).append('|')\n .append(stats.altitudeEvidenceCount()).append('\\n');\n }\n for (HistoricalHoldoutOutcome outcome : holdoutOutcomes) {\n writer.append(\"holdout|\").append(outcome.canonical()).append('\\n');\n }\n return HexFormat.of().formatHex(digest.digest());\n } catch (NoSuchAlgorithmException exception) {\n throw new IllegalStateException(\"SHA-256 unavailable\", exception);\n }\n }\n\n private static final class ChecksumWriter {\n private final MessageDigest digest;\n\n private ChecksumWriter(MessageDigest digest) {\n this.digest = digest;\n }\n\n private ChecksumWriter append(Object value) {\n digest.update(String.valueOf(value).getBytes(StandardCharsets.UTF_8));\n return this;\n }\n\n private ChecksumWriter append(char value) {\n digest.update((byte) value);\n return this;\n }\n }\n\n private static List keys(HistoricalSegment segment) {\n List result = new ArrayList<>(3);\n result.add(new HistoricalPatternKey(HistoricalPatternKind.ROUTE, \"-\", segment.startH3R5(), segment.endH3R5()));\n result.add(new HistoricalPatternKey(HistoricalPatternKind.AIRCRAFT_ROUTE, segment.icao24(),\n segment.startH3R5(), segment.endH3R5()));\n if (segment.callsignEvidence().size() == 1) {\n String callsign = segment.callsignEvidence().getFirst();\n if (callsign != null && callsign.matches(\"[A-Z0-9][A-Z0-9-]{0,31}\")) {\n result.add(new HistoricalPatternKey(HistoricalPatternKind.CALLSIGN_ROUTE, callsign,\n segment.startH3R5(), segment.endH3R5()));\n }\n }\n return List.copyOf(result);\n }\n\n private static short hourOfWeek(Instant instant) {\n var utc = instant.atZone(ZoneOffset.UTC);\n return (short) ((utc.getDayOfWeek().getValue() - 1) * 24 + utc.getHour());\n }\n\n /** Incremental compact source-day accumulator; finalized flight segments are never retained as a learning ledger. */\n static final class Builder {\n private final DatasetWindow.DateRole role;\n private final TreeMap patterns = new TreeMap<>();\n private final TreeMap hourBins = new TreeMap<>();\n private final TreeMap corridors = new TreeMap<>();\n private final List holdout = new ArrayList<>();\n private int excluded;\n private boolean sealed;\n\n private Builder(DatasetWindow.DateRole role) {\n if (role == null) throw new IllegalArgumentException(\"role is required\");\n this.role = role;\n }\n\n void accept(HistoricalSegment segment) {\n if (sealed) throw new IllegalStateException(\"day-delta builder is already sealed\");\n if (segment == null) return;\n if (role == DatasetWindow.DateRole.EMBARGO || segment.boundaryTruncated()) {\n excluded = Math.addExact(excluded, 1);\n return;\n }\n if (role == DatasetWindow.DateRole.HOLDOUT) {\n holdout.add(HistoricalHoldoutOutcome.from(segment));\n return;\n }\n short hour = hourOfWeek(segment.firstObservedAt());\n for (HistoricalPatternKey key : keys(segment)) {\n patterns.merge(key, HistoricalPatternStats.from(segment), HistoricalPatternStats::merge);\n hourBins.merge(new HistoricalPatternHourKey(key, hour), 1L, Math::addExact);\n }\n if (!segment.corridorOverflow()) {\n for (HistoricalCorridorCellEvidence cell : segment.corridorCells()) {\n HistoricalRouteCorridorKey key = new HistoricalRouteCorridorKey(\n segment.startH3R5(), segment.endH3R5(), cell.h3R7());\n corridors.merge(key, HistoricalRouteCorridorStats.from(cell), HistoricalRouteCorridorStats::merge);\n }\n }\n }\n\n HistoricalDayDelta seal() {\n if (sealed) throw new IllegalStateException(\"day-delta builder is already sealed\");\n sealed = true;\n holdout.sort(Comparator.comparing(HistoricalHoldoutOutcome::segmentId));\n return new HistoricalDayDelta(role, patterns, hourBins, corridors, holdout, excluded);\n }\n }\n}\n\nenum HistoricalPatternKind {\n ROUTE,\n AIRCRAFT_ROUTE,\n CALLSIGN_ROUTE\n}\n\nrecord HistoricalPatternKey(\n HistoricalPatternKind kind,\n String subjectKey,\n String startH3R5,\n String endH3R5\n) implements Comparable {\n HistoricalPatternKey {\n if (kind == null || subjectKey == null || subjectKey.isBlank() || subjectKey.length() > 64\n || startH3R5 == null || !startH3R5.matches(\"85[0-9a-f]{13}\")\n || endH3R5 == null || !endH3R5.matches(\"85[0-9a-f]{13}\")) {\n throw new IllegalArgumentException(\"invalid historical pattern key\");\n }\n }\n\n String canonical() { return kind.name() + \"|\" + subjectKey + \"|\" + startH3R5 + \"|\" + endH3R5; }\n\n @Override\n public int compareTo(HistoricalPatternKey other) {\n int compared = kind.compareTo(other.kind);\n if (compared != 0) return compared;\n compared = subjectKey.compareTo(other.subjectKey);\n if (compared != 0) return compared;\n compared = startH3R5.compareTo(other.startH3R5);\n if (compared != 0) return compared;\n return endH3R5.compareTo(other.endH3R5);\n }\n}\n\nrecord HistoricalPatternHourKey(HistoricalPatternKey patternKey, short hourOfWeek)\n implements Comparable {\n HistoricalPatternHourKey {\n if (patternKey == null || hourOfWeek < 0 || hourOfWeek > 167) {\n throw new IllegalArgumentException(\"invalid historical hour-bin key\");\n }\n }\n\n @Override\n public int compareTo(HistoricalPatternHourKey other) {\n int compared = patternKey.compareTo(other.patternKey);\n return compared != 0 ? compared : Short.compare(hourOfWeek, other.hourOfWeek);\n }\n}\n\nrecord HistoricalPatternStats(\n long evidenceCount,\n long cleanEvidenceCount,\n Instant firstOccurrenceAt,\n Instant lastOccurrenceAt,\n int minimumAltitudeFt,\n int maximumAltitudeFt,\n long meanAltitudeSumFt,\n long altitudeEvidenceCount,\n long durationSecondsSum,\n long observationCountSum,\n HistoricalSourceCounts sourceCounts\n) {\n HistoricalPatternStats {\n if (evidenceCount < 1L || cleanEvidenceCount < 0L || cleanEvidenceCount > evidenceCount\n || firstOccurrenceAt == null || lastOccurrenceAt == null || lastOccurrenceAt.isBefore(firstOccurrenceAt)\n || minimumAltitudeFt > maximumAltitudeFt\n || altitudeEvidenceCount < 0L || altitudeEvidenceCount > evidenceCount\n || durationSecondsSum < 0L || observationCountSum < 1L\n || sourceCounts == null || sourceCounts.total() != observationCountSum) {\n throw new IllegalArgumentException(\"invalid historical pattern stats\");\n }\n }\n\n static HistoricalPatternStats from(HistoricalSegment segment) {\n return new HistoricalPatternStats(\n 1L, 1L, segment.firstObservedAt(), segment.firstObservedAt(),\n segment.minimumAltitudeFt(), segment.maximumAltitudeFt(), segment.meanAltitudeFt(), 1L,\n segment.durationSeconds(), segment.observationCount(), segment.sourceCounts());\n }\n\n static HistoricalPatternStats merge(HistoricalPatternStats left, HistoricalPatternStats right) {\n return new HistoricalPatternStats(\n Math.addExact(left.evidenceCount, right.evidenceCount),\n Math.addExact(left.cleanEvidenceCount, right.cleanEvidenceCount),\n left.firstOccurrenceAt.isBefore(right.firstOccurrenceAt) ? left.firstOccurrenceAt : right.firstOccurrenceAt,\n left.lastOccurrenceAt.isAfter(right.lastOccurrenceAt) ? left.lastOccurrenceAt : right.lastOccurrenceAt,\n Math.min(left.minimumAltitudeFt, right.minimumAltitudeFt),\n Math.max(left.maximumAltitudeFt, right.maximumAltitudeFt),\n Math.addExact(left.meanAltitudeSumFt, right.meanAltitudeSumFt),\n Math.addExact(left.altitudeEvidenceCount, right.altitudeEvidenceCount),\n Math.addExact(left.durationSecondsSum, right.durationSecondsSum),\n Math.addExact(left.observationCountSum, right.observationCountSum),\n left.sourceCounts.plus(right.sourceCounts));\n }\n}\n\nrecord HistoricalHoldoutOutcome(\n String segmentId,\n String icao24,\n String callsign,\n Instant firstObservedAt,\n Instant lastObservedAt,\n String startH3R5,\n String endH3R5,\n int minimumAltitudeFt,\n int maximumAltitudeFt,\n int meanAltitudeFt,\n long altitudeEvidenceCount,\n long durationSeconds,\n long observationCount,\n HistoricalSourceCounts sourceCounts,\n List corridorCells,\n boolean corridorOverflow\n) {\n HistoricalHoldoutOutcome {\n corridorCells = List.copyOf(corridorCells == null ? List.of() : corridorCells);\n if (segmentId == null || !segmentId.matches(\"[0-9a-f]{64}\")\n || icao24 == null || !icao24.matches(\"[0-9a-f]{6}\")\n || firstObservedAt == null || lastObservedAt == null || lastObservedAt.isBefore(firstObservedAt)\n || startH3R5 == null || !startH3R5.matches(\"85[0-9a-f]{13}\")\n || endH3R5 == null || !endH3R5.matches(\"85[0-9a-f]{13}\")\n || minimumAltitudeFt > maximumAltitudeFt\n || altitudeEvidenceCount < 1L || observationCount < 1L || altitudeEvidenceCount > observationCount\n || durationSeconds < 0L || sourceCounts == null || sourceCounts.total() != observationCount) {\n throw new IllegalArgumentException(\"invalid historical holdout outcome\");\n }\n if (corridorOverflow && !corridorCells.isEmpty()) {\n throw new IllegalArgumentException(\"overflow holdout corridor must discard its prefix\");\n }\n if (!corridorOverflow && corridorCells.isEmpty()) {\n throw new IllegalArgumentException(\"complete holdout corridor cannot be empty\");\n }\n if (corridorCells.size() > HistoricalCorridorModel.MAX_UNIQUE_CELLS) {\n throw new IllegalArgumentException(\"holdout corridor exceeds bounded cell cap\");\n }\n java.util.HashSet unique = new java.util.HashSet<>();\n for (int index = 0; index < corridorCells.size(); index++) {\n HistoricalCorridorCellEvidence cell = corridorCells.get(index);\n if (cell.sequenceIndex() != index || !unique.add(cell.h3R7())) {\n throw new IllegalArgumentException(\"holdout corridor must be ordered and unique\");\n }\n }\n }\n\n HistoricalHoldoutOutcome(\n String segmentId, String icao24, String callsign, Instant firstObservedAt, Instant lastObservedAt,\n String startH3R5, String endH3R5, int minimumAltitudeFt, int maximumAltitudeFt, int meanAltitudeFt,\n long altitudeEvidenceCount, long durationSeconds, long observationCount, HistoricalSourceCounts sourceCounts\n ) {\n this(segmentId, icao24, callsign, firstObservedAt, lastObservedAt, startH3R5, endH3R5,\n minimumAltitudeFt, maximumAltitudeFt, meanAltitudeFt, altitudeEvidenceCount, durationSeconds,\n observationCount, sourceCounts, List.of(), true);\n }\n\n static HistoricalHoldoutOutcome from(HistoricalSegment segment) {\n String callsign = segment.callsignEvidence().size() == 1 ? segment.callsignEvidence().getFirst() : null;\n return new HistoricalHoldoutOutcome(\n segment.segmentId(), segment.icao24(), callsign, segment.firstObservedAt(), segment.lastObservedAt(),\n segment.startH3R5(), segment.endH3R5(), segment.minimumAltitudeFt(), segment.maximumAltitudeFt(),\n segment.meanAltitudeFt(), 1L, segment.durationSeconds(), segment.observationCount(), segment.sourceCounts(),\n segment.corridorCells(), segment.corridorOverflow());\n }\n\n String canonical() {\n StringBuilder value = new StringBuilder(segmentId).append('|').append(icao24).append('|')\n .append(callsign == null ? \"-\" : callsign).append('|')\n .append(firstObservedAt.toEpochMilli()).append('|').append(lastObservedAt.toEpochMilli()).append('|')\n .append(startH3R5).append('|').append(endH3R5).append('|')\n .append(minimumAltitudeFt).append('|').append(maximumAltitudeFt).append('|').append(meanAltitudeFt).append('|')\n .append(altitudeEvidenceCount).append('|').append(durationSeconds).append('|').append(observationCount)\n .append('|').append(sourceCounts.canonical()).append('|').append(corridorOverflow);\n for (HistoricalCorridorCellEvidence cell : corridorCells) value.append('|').append(cell.canonical());\n return value.toString();\n }\n}\n" }, { "path": "backend/historical-bootstrap-tool/src/main/java/hu/gombos/flightspotter/historicalbootstrap/HistoricalEligibilityV1.java", "encoding": "UTF-8", "sha256": "b61729d2a9d24f2a3d42c67fbf80fe45053e29462ed5723784df2e11b5bb8868", "content": "package hu.gombos.flightspotter.historicalbootstrap;\n\nimport java.time.LocalDate;\nimport java.time.ZoneOffset;\nimport java.util.Locale;\nimport java.util.Set;\n\n/** Exact clean-evidence gate frozen as p7-source-eligibility-v1. */\nfinal class HistoricalEligibilityV1 {\n static final String VERSION = \"p7-source-eligibility-v1\";\n private static final Set USABLE_POSITION_SOURCES = Set.of(\"adsb_icao\", \"adsr_icao\", \"tisb_icao\", \"mlat\");\n\n Decision evaluate(ReadsbTrace trace, ReadsbTracePoint point, LocalDate sourceDate, BootstrapCoverageSpec coverage) {\n if (trace == null || point == null || sourceDate == null || coverage == null) return Decision.rejected(RejectionReason.MISSING_REQUIRED_FIELD);\n String icao = trace.icao24();\n if (icao == null || !icao.matches(\"[0-9a-f]{6}\")) return Decision.rejected(icao != null && icao.startsWith(\"~\") ? RejectionReason.NON_ICAO_IDENTITY : RejectionReason.INVALID_ICAO_IDENTITY);\n if (point.observedAt() == null || !point.observedAt().atZone(ZoneOffset.UTC).toLocalDate().equals(sourceDate)) return Decision.rejected(RejectionReason.OUTSIDE_SOURCE_DAY);\n if (!validCoordinate(point.latitude(), point.longitude())) return Decision.rejected(RejectionReason.INVALID_COORDINATES);\n if (point.stalePosition()) return Decision.rejected(RejectionReason.STALE_POSITION);\n String positionSource = normalize(point.positionSource());\n if (positionSource == null || !USABLE_POSITION_SOURCES.contains(positionSource)) return Decision.rejected(RejectionReason.UNSUPPORTED_POSITION_SOURCE);\n if (point.groundSpeedKt() != null && (!Double.isFinite(point.groundSpeedKt()) || point.groundSpeedKt() < 0d || point.groundSpeedKt() > 2_000d)) return Decision.rejected(RejectionReason.IMPOSSIBLE_GROUND_SPEED);\n Double altitude = point.altitudeFt();\n if (point.onGround() || altitude == null || !Double.isFinite(altitude)) return Decision.rejected(RejectionReason.MISSING_USABLE_ALTITUDE);\n if (altitude < -2_000d || altitude > 100_000d) return Decision.rejected(RejectionReason.ALTITUDE_OUT_OF_DOMAIN);\n if (altitude <= 0d) return Decision.rejected(RejectionReason.NOT_AIRBORNE);\n int altitudeFt = Math.toIntExact(Math.round(altitude));\n String h3Cell;\n String corridorH3Cell;\n try {\n h3Cell = coverage.cellFor(point.latitude(), point.longitude());\n corridorH3Cell = coverage.corridorCellFor(point.latitude(), point.longitude());\n } catch (RuntimeException exception) {\n return Decision.rejected(RejectionReason.INVALID_COORDINATES);\n }\n if (!coverage.containsCell(h3Cell)) return Decision.rejected(RejectionReason.OUTSIDE_COVERAGE);\n Integer groundSpeedKt = point.groundSpeedKt() == null ? null : Math.toIntExact(Math.round(point.groundSpeedKt()));\n return Decision.accepted(new EligiblePoint(icao, trace.registration(), trace.typeCode(), point.callsign(), positionSource,\n point.observedAt(), point.latitude(), point.longitude(), altitudeFt, point.altitudeSource(), groundSpeedKt,\n h3Cell, corridorH3Cell, point.newLeg()));\n }\n\n private static boolean validCoordinate(Double latitude, Double longitude) {\n return latitude != null && longitude != null && Double.isFinite(latitude) && Double.isFinite(longitude)\n && latitude >= -90d && latitude <= 90d && longitude >= -180d && longitude <= 180d;\n }\n\n private static String normalize(String value) {\n return value == null || value.isBlank() ? null : value.trim().toLowerCase(Locale.ROOT);\n }\n\n record Decision(EligiblePoint point, RejectionReason rejectionReason) {\n static Decision accepted(EligiblePoint point) { return new Decision(point, null); }\n static Decision rejected(RejectionReason reason) { return new Decision(null, reason); }\n boolean accepted() { return point != null; }\n }\n\n record EligiblePoint(String icao24, String registration, String typeCode, String callsign, String positionSource,\n java.time.Instant observedAt, double latitude, double longitude, int altitudeFt,\n HistoricalAltitudeSource altitudeSource, Integer groundSpeedKt, String h3R5, String h3R7,\n boolean newLeg) {\n EligiblePoint {\n if (!HistoricalCorridorModel.validR7(h3R7)) throw new IllegalArgumentException(\"eligible point requires H3 R7 corridor identity\");\n }\n }\n\n enum RejectionReason {\n MISSING_REQUIRED_FIELD, NON_ICAO_IDENTITY, INVALID_ICAO_IDENTITY, OUTSIDE_SOURCE_DAY, INVALID_COORDINATES,\n STALE_POSITION, UNSUPPORTED_POSITION_SOURCE, IMPOSSIBLE_GROUND_SPEED, MISSING_USABLE_ALTITUDE,\n ALTITUDE_OUT_OF_DOMAIN, NOT_AIRBORNE, OUTSIDE_COVERAGE\n }\n}\n" }, { "path": "backend/historical-bootstrap-tool/src/main/java/hu/gombos/flightspotter/historicalbootstrap/HistoricalH3Projection.java", "encoding": "UTF-8", "sha256": "80f265a549a0ec732bd6581c0b2b234618e59991d837ce94357cd71d2020a064", "content": "package hu.gombos.flightspotter.historicalbootstrap;\n\nimport hu.gombos.flightspotter.common.geo.H3Service;\nimport hu.gombos.flightspotter.common.geo.UberH3Service;\n\n/** Deterministic coarse-cell projection shared by historical persistence seams. */\nfinal class HistoricalH3Projection {\n private static final H3Service H3 = new UberH3Service();\n\n private HistoricalH3Projection() {\n }\n\n static String corridorParentR5(String corridorH3R7) {\n if (corridorH3R7 == null || !corridorH3R7.matches(\"^87[0-9a-f]{13}$\")) {\n throw new IllegalArgumentException(\"corridor H3 R7 identity is invalid\");\n }\n String parent = H3.cellToParent(corridorH3R7, 5);\n if (!parent.matches(\"^85[0-9a-f]{13}$\")) {\n throw new IllegalStateException(\"corridor H3 R5 parent projection is invalid\");\n }\n return parent;\n }\n}\n" }, { "path": "backend/historical-bootstrap-tool/src/main/java/hu/gombos/flightspotter/historicalbootstrap/HistoricalSegment.java", "encoding": "UTF-8", "sha256": "46f4273839a044160a64280b680cdc48398cb6d2011e6172a4fdd8d88140dc15", "content": "package hu.gombos.flightspotter.historicalbootstrap;\n\nimport java.nio.charset.StandardCharsets;\nimport java.security.MessageDigest;\nimport java.security.NoSuchAlgorithmException;\nimport java.time.Duration;\nimport java.time.Instant;\nimport java.util.HashSet;\nimport java.util.HexFormat;\nimport java.util.List;\n\nrecord HistoricalSegment(\n String segmentId,\n String icao24,\n Instant firstObservedAt,\n Instant lastObservedAt,\n long observationCount,\n String startH3R5,\n String endH3R5,\n int minimumAltitudeFt,\n int maximumAltitudeFt,\n long altitudeSumFt,\n long altitudeEvidenceCount,\n HistoricalSourceCounts sourceCounts,\n List callsignEvidence,\n HistoricalSegmentFinalizationReason finalizationReason,\n boolean boundaryTruncated,\n List corridorCells,\n boolean corridorOverflow\n) {\n HistoricalSegment {\n callsignEvidence = List.copyOf(callsignEvidence);\n corridorCells = List.copyOf(corridorCells == null ? List.of() : corridorCells);\n if (segmentId == null || !segmentId.matches(\"[0-9a-f]{64}\")\n || icao24 == null || !icao24.matches(\"[0-9a-f]{6}\")\n || firstObservedAt == null || lastObservedAt == null || lastObservedAt.isBefore(firstObservedAt)\n || observationCount < 1L || altitudeEvidenceCount < 1L || altitudeEvidenceCount > observationCount\n || sourceCounts == null || sourceCounts.total() != observationCount\n || startH3R5 == null || !startH3R5.matches(\"85[0-9a-f]{13}\")\n || endH3R5 == null || !endH3R5.matches(\"85[0-9a-f]{13}\")\n || minimumAltitudeFt > maximumAltitudeFt\n || callsignEvidence.size() > 3 || finalizationReason == null) {\n throw new IllegalArgumentException(\"invalid compact historical segment\");\n }\n validateCorridor(corridorCells, corridorOverflow);\n }\n\n HistoricalSegment(\n String segmentId,\n String icao24,\n Instant firstObservedAt,\n Instant lastObservedAt,\n long observationCount,\n String startH3R5,\n String endH3R5,\n int minimumAltitudeFt,\n int maximumAltitudeFt,\n long altitudeSumFt,\n long altitudeEvidenceCount,\n HistoricalSourceCounts sourceCounts,\n List callsignEvidence,\n HistoricalSegmentFinalizationReason finalizationReason,\n boolean boundaryTruncated\n ) {\n this(segmentId, icao24, firstObservedAt, lastObservedAt, observationCount, startH3R5, endH3R5,\n minimumAltitudeFt, maximumAltitudeFt, altitudeSumFt, altitudeEvidenceCount, sourceCounts,\n callsignEvidence, finalizationReason, boundaryTruncated, List.of(), true);\n }\n\n int meanAltitudeFt() {\n return Math.toIntExact(Math.round((double) altitudeSumFt / altitudeEvidenceCount));\n }\n\n long durationSeconds() {\n return Duration.between(firstObservedAt, lastObservedAt).toSeconds();\n }\n\n static HistoricalSegment fromCarry(HistoricalSegmentCarry carry, HistoricalSegmentFinalizationReason reason) {\n String identity = \"p7-seg-v1|\" + carry.icao24() + \"|\" + carry.firstObservedAt().toEpochMilli()\n + \"|\" + carry.lastObservedAt().toEpochMilli() + \"|\" + carry.observationCount()\n + \"|\" + carry.startH3R5() + \"|\" + carry.endH3R5() + \"|\" + carry.altitudeSumFt()\n + \"|\" + carry.sourceCounts().canonical();\n return new HistoricalSegment(\n sha256(identity),\n carry.icao24(),\n carry.firstObservedAt(),\n carry.lastObservedAt(),\n carry.observationCount(),\n carry.startH3R5(),\n carry.endH3R5(),\n carry.minimumAltitudeFt(),\n carry.maximumAltitudeFt(),\n carry.altitudeSumFt(),\n carry.altitudeEvidenceCount(),\n carry.sourceCounts(),\n carry.callsignEvidence(),\n reason,\n carry.boundaryTruncated(),\n HistoricalCorridorModel.finalizeCells(\n carry.corridorCells(), carry.corridorOverflow(), carry.firstObservedAt(), carry.lastObservedAt()),\n carry.corridorOverflow()\n );\n }\n\n private static void validateCorridor(List cells, boolean overflow) {\n if (overflow && !cells.isEmpty()) throw new IllegalArgumentException(\"overflow corridor must discard its prefix\");\n if (!overflow && cells.isEmpty()) throw new IllegalArgumentException(\"complete segment corridor cannot be empty\");\n if (cells.size() > HistoricalCorridorModel.MAX_UNIQUE_CELLS) {\n throw new IllegalArgumentException(\"segment corridor exceeds bounded cell cap\");\n }\n HashSet unique = new HashSet<>();\n for (int index = 0; index < cells.size(); index++) {\n HistoricalCorridorCellEvidence cell = cells.get(index);\n if (cell.sequenceIndex() != index || !unique.add(cell.h3R7())) {\n throw new IllegalArgumentException(\"segment corridor must be ordered and unique\");\n }\n }\n }\n\n private static String sha256(String value) {\n try {\n return HexFormat.of().formatHex(MessageDigest.getInstance(\"SHA-256\")\n .digest(value.getBytes(StandardCharsets.UTF_8)));\n } catch (NoSuchAlgorithmException exception) {\n throw new IllegalStateException(\"SHA-256 unavailable\", exception);\n }\n }\n}\n\nrecord HistoricalSegmentCarry(\n String icao24,\n Instant firstObservedAt,\n Instant lastObservedAt,\n long observationCount,\n double lastLatitude,\n double lastLongitude,\n String startH3R5,\n String endH3R5,\n int minimumAltitudeFt,\n int maximumAltitudeFt,\n long altitudeSumFt,\n long altitudeEvidenceCount,\n HistoricalSourceCounts sourceCounts,\n List callsignEvidence,\n String primaryCallsign,\n boolean boundaryTruncated,\n List corridorCells,\n boolean corridorOverflow\n) {\n HistoricalSegmentCarry {\n callsignEvidence = List.copyOf(callsignEvidence);\n corridorCells = List.copyOf(corridorCells == null ? List.of() : corridorCells);\n if (icao24 == null || !icao24.matches(\"[0-9a-f]{6}\")\n || firstObservedAt == null || lastObservedAt == null || lastObservedAt.isBefore(firstObservedAt)\n || observationCount < 1L || !Double.isFinite(lastLatitude) || !Double.isFinite(lastLongitude)\n || startH3R5 == null || !startH3R5.matches(\"85[0-9a-f]{13}\")\n || endH3R5 == null || !endH3R5.matches(\"85[0-9a-f]{13}\")\n || minimumAltitudeFt > maximumAltitudeFt\n || altitudeEvidenceCount < 1L || altitudeEvidenceCount > observationCount\n || sourceCounts == null || sourceCounts.total() != observationCount\n || callsignEvidence.size() > 3) {\n throw new IllegalArgumentException(\"invalid compact historical carry\");\n }\n if (corridorOverflow && !corridorCells.isEmpty()) {\n throw new IllegalArgumentException(\"overflow carry corridor must discard its prefix\");\n }\n if (!corridorOverflow && corridorCells.isEmpty()) {\n throw new IllegalArgumentException(\"complete carry corridor cannot be empty\");\n }\n if (corridorCells.size() > HistoricalCorridorModel.MAX_UNIQUE_CELLS) {\n throw new IllegalArgumentException(\"carry corridor exceeds bounded cell cap\");\n }\n HashSet unique = new HashSet<>();\n for (HistoricalCorridorCellCarry cell : corridorCells) {\n if (!unique.add(cell.h3R7())) throw new IllegalArgumentException(\"carry corridor cells must be unique\");\n if (cell.firstEnteredAt().isBefore(firstObservedAt) || cell.firstEnteredAt().isAfter(lastObservedAt)) {\n throw new IllegalArgumentException(\"carry corridor entry lies outside segment time bounds\");\n }\n }\n }\n\n HistoricalSegmentCarry(\n String icao24,\n Instant firstObservedAt,\n Instant lastObservedAt,\n long observationCount,\n double lastLatitude,\n double lastLongitude,\n String startH3R5,\n String endH3R5,\n int minimumAltitudeFt,\n int maximumAltitudeFt,\n long altitudeSumFt,\n long altitudeEvidenceCount,\n HistoricalSourceCounts sourceCounts,\n List callsignEvidence,\n String primaryCallsign,\n boolean boundaryTruncated\n ) {\n this(icao24, firstObservedAt, lastObservedAt, observationCount, lastLatitude, lastLongitude,\n startH3R5, endH3R5, minimumAltitudeFt, maximumAltitudeFt, altitudeSumFt, altitudeEvidenceCount,\n sourceCounts, callsignEvidence, primaryCallsign, boundaryTruncated, List.of(), true);\n }\n}\n\nenum HistoricalSegmentFinalizationReason {\n NEW_LEG,\n MAX_DURATION,\n GAP,\n CALLSIGN_CHANGE,\n SPATIAL_JUMP,\n QUIET,\n WINDOW_END\n}\n" }, { "path": "backend/historical-bootstrap-tool/src/main/java/hu/gombos/flightspotter/historicalbootstrap/HistoricalSegmenterV1.java", "encoding": "UTF-8", "sha256": "8eb2503d26243e89d09e322cf9eb2a80abf067d728147baa11127dd795fd77c9", "content": "package hu.gombos.flightspotter.historicalbootstrap;\n\nimport java.time.Duration;\nimport java.time.Instant;\nimport java.util.ArrayList;\nimport java.util.Comparator;\nimport java.util.LinkedHashMap;\nimport java.util.List;\n\n/** Deterministic p7-seg-v1 segmentation with bounded p7-corridor-v1 route-shape carry. */\nfinal class HistoricalSegmenterV1 {\n static final String VERSION = \"p7-seg-v1\";\n private static final Duration MAX_SEGMENT_DURATION = Duration.ofHours(18);\n private static final Duration GAP = Duration.ofMinutes(20);\n\n static Duration carryHorizon() {\n return GAP;\n }\n\n Result process(\n HistoricalSegmentCarry incomingCarry,\n List points,\n boolean boundaryTruncatedOnFirstSegment\n ) {\n if (points == null) throw new IllegalArgumentException(\"points are required\");\n HistoricalSegmentCarry current = incomingCarry;\n CorridorAccumulator corridor = current == null ? null : CorridorAccumulator.from(current);\n List finalized = new ArrayList<>();\n int outOfOrderRejected = 0;\n boolean firstStarted = incomingCarry != null;\n\n for (HistoricalEligibilityV1.EligiblePoint point : points) {\n if (point == null) continue;\n if (current == null) {\n current = initial(point, !firstStarted && boundaryTruncatedOnFirstSegment);\n corridor = CorridorAccumulator.from(current);\n firstStarted = true;\n continue;\n }\n if (!current.icao24().equals(point.icao24())) {\n throw new IllegalArgumentException(\"one segmenter call may contain only one ICAO24\");\n }\n if (point.observedAt().isBefore(current.lastObservedAt())) {\n outOfOrderRejected = Math.addExact(outOfOrderRejected, 1);\n continue;\n }\n if (point.observedAt().equals(current.lastObservedAt())) {\n current = mergeEqualTimestamp(current, point);\n continue;\n }\n HistoricalSegmentFinalizationReason reason = splitReason(current, point);\n if (reason != null) {\n current = withCorridor(current, corridor);\n finalized.add(HistoricalSegment.fromCarry(current, reason));\n current = initial(point, false);\n corridor = CorridorAccumulator.from(current);\n } else {\n current = update(current, point);\n corridor.accept(point);\n }\n }\n if (current != null) current = withCorridor(current, corridor);\n return new Result(List.copyOf(finalized), current, outOfOrderRejected);\n }\n\n HistoricalSegment finalizeQuiet(HistoricalSegmentCarry carry, Instant now) {\n if (carry == null || now == null || carry.lastObservedAt().plus(GAP).isAfter(now)) return null;\n return HistoricalSegment.fromCarry(carry, HistoricalSegmentFinalizationReason.QUIET);\n }\n\n HistoricalSegment finalizeWindowEnd(HistoricalSegmentCarry carry) {\n return carry == null ? null : HistoricalSegment.fromCarry(carry, HistoricalSegmentFinalizationReason.WINDOW_END);\n }\n\n private static HistoricalSegmentFinalizationReason splitReason(\n HistoricalSegmentCarry state,\n HistoricalEligibilityV1.EligiblePoint point\n ) {\n if (point.newLeg()) return HistoricalSegmentFinalizationReason.NEW_LEG;\n if (!point.observedAt().isBefore(state.firstObservedAt().plus(MAX_SEGMENT_DURATION))) {\n return HistoricalSegmentFinalizationReason.MAX_DURATION;\n }\n if (point.observedAt().isAfter(state.lastObservedAt().plus(GAP))) {\n return HistoricalSegmentFinalizationReason.GAP;\n }\n if (state.primaryCallsign() != null && point.callsign() != null\n && !state.primaryCallsign().equals(point.callsign())) {\n return HistoricalSegmentFinalizationReason.CALLSIGN_CHANGE;\n }\n double elapsedHours = Duration.between(state.lastObservedAt(), point.observedAt()).toMillis() / 3_600_000d;\n double distanceNm = haversineKm(\n state.lastLatitude(), state.lastLongitude(), point.latitude(), point.longitude()) / 1.852d;\n if (distanceNm > 20d && elapsedHours > 0d && distanceNm / elapsedHours > 1_200d) {\n return HistoricalSegmentFinalizationReason.SPATIAL_JUMP;\n }\n return null;\n }\n\n private static HistoricalSegmentCarry initial(\n HistoricalEligibilityV1.EligiblePoint point,\n boolean boundaryTruncated\n ) {\n List callsigns = addCallsign(List.of(), point.callsign());\n HistoricalCorridorCellCarry corridor = new HistoricalCorridorCellCarry(\n point.h3R7(), point.observedAt(), point.altitudeFt(), point.altitudeFt(), point.altitudeFt(), 1L);\n return new HistoricalSegmentCarry(\n point.icao24(), point.observedAt(), point.observedAt(), 1L,\n point.latitude(), point.longitude(), point.h3R5(), point.h3R5(),\n point.altitudeFt(), point.altitudeFt(), point.altitudeFt(), 1L,\n HistoricalSourceCounts.one(point.positionSource()), callsigns, point.callsign(), boundaryTruncated,\n List.of(corridor), false\n );\n }\n\n private static HistoricalSegmentCarry mergeEqualTimestamp(\n HistoricalSegmentCarry state,\n HistoricalEligibilityV1.EligiblePoint point\n ) {\n return new HistoricalSegmentCarry(\n state.icao24(), state.firstObservedAt(), state.lastObservedAt(), state.observationCount(),\n state.lastLatitude(), state.lastLongitude(), state.startH3R5(), state.endH3R5(),\n state.minimumAltitudeFt(), state.maximumAltitudeFt(), state.altitudeSumFt(),\n state.altitudeEvidenceCount(), state.sourceCounts(),\n addCallsign(state.callsignEvidence(), point.callsign()),\n state.primaryCallsign(), state.boundaryTruncated(), state.corridorCells(), state.corridorOverflow()\n );\n }\n\n private static HistoricalSegmentCarry update(\n HistoricalSegmentCarry state,\n HistoricalEligibilityV1.EligiblePoint point\n ) {\n return new HistoricalSegmentCarry(\n state.icao24(), state.firstObservedAt(), point.observedAt(), Math.addExact(state.observationCount(), 1L),\n point.latitude(), point.longitude(), state.startH3R5(), point.h3R5(),\n Math.min(state.minimumAltitudeFt(), point.altitudeFt()),\n Math.max(state.maximumAltitudeFt(), point.altitudeFt()),\n Math.addExact(state.altitudeSumFt(), point.altitudeFt()),\n Math.addExact(state.altitudeEvidenceCount(), 1L),\n state.sourceCounts().plus(HistoricalSourceCounts.one(point.positionSource())),\n addCallsign(state.callsignEvidence(), point.callsign()),\n point.callsign() == null ? state.primaryCallsign() : point.callsign(),\n state.boundaryTruncated(), state.corridorCells(), state.corridorOverflow()\n );\n }\n\n private static HistoricalSegmentCarry withCorridor(HistoricalSegmentCarry state, CorridorAccumulator corridor) {\n CorridorSnapshot snapshot = corridor == null\n ? new CorridorSnapshot(List.of(), true)\n : corridor.snapshot();\n return new HistoricalSegmentCarry(\n state.icao24(), state.firstObservedAt(), state.lastObservedAt(), state.observationCount(),\n state.lastLatitude(), state.lastLongitude(), state.startH3R5(), state.endH3R5(),\n state.minimumAltitudeFt(), state.maximumAltitudeFt(), state.altitudeSumFt(),\n state.altitudeEvidenceCount(), state.sourceCounts(), state.callsignEvidence(), state.primaryCallsign(),\n state.boundaryTruncated(), snapshot.cells(), snapshot.overflow()\n );\n }\n\n private static List addCallsign(List existing, String callsign) {\n if (callsign == null || callsign.isBlank() || existing.contains(callsign)) return existing;\n List result = new ArrayList<>(existing);\n result.add(callsign);\n result.sort(Comparator.naturalOrder());\n return result.size() > 3 ? List.copyOf(result.subList(0, 3)) : List.copyOf(result);\n }\n\n private static double haversineKm(double aLat, double aLon, double bLat, double bLon) {\n double dLat = Math.toRadians(bLat - aLat);\n double dLon = Math.toRadians(bLon - aLon);\n double a = Math.pow(Math.sin(dLat / 2d), 2d)\n + Math.cos(Math.toRadians(aLat)) * Math.cos(Math.toRadians(bLat))\n * Math.pow(Math.sin(dLon / 2d), 2d);\n return 2d * 6_371.0088d * Math.atan2(Math.sqrt(a), Math.sqrt(1d - a));\n }\n\n record Result(List finalizedSegments, HistoricalSegmentCarry outgoingCarry, int outOfOrderRejected) {\n }\n\n private record CorridorSnapshot(List cells, boolean overflow) {\n CorridorSnapshot {\n cells = List.copyOf(cells);\n }\n }\n\n private static final class CorridorAccumulator {\n private final LinkedHashMap cells = new LinkedHashMap<>();\n private boolean overflow;\n\n static CorridorAccumulator from(HistoricalSegmentCarry carry) {\n CorridorAccumulator accumulator = new CorridorAccumulator();\n accumulator.overflow = carry.corridorOverflow();\n if (!accumulator.overflow) {\n for (HistoricalCorridorCellCarry cell : carry.corridorCells()) {\n accumulator.cells.put(cell.h3R7(), new MutableCorridorCell(cell));\n }\n }\n return accumulator;\n }\n\n void accept(HistoricalEligibilityV1.EligiblePoint point) {\n if (overflow) return;\n MutableCorridorCell existing = cells.get(point.h3R7());\n if (existing != null) {\n existing.accept(point.altitudeFt());\n return;\n }\n if (cells.size() >= HistoricalCorridorModel.MAX_UNIQUE_CELLS) {\n cells.clear();\n overflow = true;\n return;\n }\n cells.put(point.h3R7(), new MutableCorridorCell(point));\n }\n\n CorridorSnapshot snapshot() {\n if (overflow) return new CorridorSnapshot(List.of(), true);\n return new CorridorSnapshot(cells.values().stream().map(MutableCorridorCell::freeze).toList(), false);\n }\n }\n\n private static final class MutableCorridorCell {\n private final String h3R7;\n private final Instant firstEnteredAt;\n private int minimumAltitudeFt;\n private int maximumAltitudeFt;\n private long altitudeSumFt;\n private long altitudeEvidenceCount;\n\n MutableCorridorCell(HistoricalEligibilityV1.EligiblePoint point) {\n this.h3R7 = point.h3R7();\n this.firstEnteredAt = point.observedAt();\n this.minimumAltitudeFt = point.altitudeFt();\n this.maximumAltitudeFt = point.altitudeFt();\n this.altitudeSumFt = point.altitudeFt();\n this.altitudeEvidenceCount = 1L;\n }\n\n MutableCorridorCell(HistoricalCorridorCellCarry cell) {\n this.h3R7 = cell.h3R7();\n this.firstEnteredAt = cell.firstEnteredAt();\n this.minimumAltitudeFt = cell.minimumAltitudeFt();\n this.maximumAltitudeFt = cell.maximumAltitudeFt();\n this.altitudeSumFt = cell.altitudeSumFt();\n this.altitudeEvidenceCount = cell.altitudeEvidenceCount();\n }\n\n void accept(int altitudeFt) {\n minimumAltitudeFt = Math.min(minimumAltitudeFt, altitudeFt);\n maximumAltitudeFt = Math.max(maximumAltitudeFt, altitudeFt);\n altitudeSumFt = Math.addExact(altitudeSumFt, altitudeFt);\n altitudeEvidenceCount = Math.addExact(altitudeEvidenceCount, 1L);\n }\n\n HistoricalCorridorCellCarry freeze() {\n return new HistoricalCorridorCellCarry(\n h3R7, firstEnteredAt, minimumAltitudeFt, maximumAltitudeFt, altitudeSumFt, altitudeEvidenceCount);\n }\n }\n}\n" }, { "path": "backend/historical-bootstrap-tool/src/main/java/hu/gombos/flightspotter/historicalbootstrap/HistoricalSourceCounts.java", "encoding": "UTF-8", "sha256": "de10d4db1027cabcdd622b51e1c0fd4b4ba2d9540ae7ce036d073ad3884077e5", "content": "package hu.gombos.flightspotter.historicalbootstrap;\n\n/** Fixed-cardinality accepted-position provenance for p7-source-eligibility-v1. */\nrecord HistoricalSourceCounts(\n long adsbIcao,\n long adsrIcao,\n long tisbIcao,\n long mlat\n) {\n HistoricalSourceCounts {\n if (adsbIcao < 0L || adsrIcao < 0L || tisbIcao < 0L || mlat < 0L) {\n throw new IllegalArgumentException(\"historical source counts cannot be negative\");\n }\n Math.addExact(Math.addExact(adsbIcao, adsrIcao), Math.addExact(tisbIcao, mlat));\n }\n\n static HistoricalSourceCounts zero() {\n return new HistoricalSourceCounts(0L, 0L, 0L, 0L);\n }\n\n static HistoricalSourceCounts one(String positionSource) {\n if (positionSource == null) {\n throw new IllegalArgumentException(\"accepted historical point requires position source\");\n }\n return switch (positionSource) {\n case \"adsb_icao\" -> new HistoricalSourceCounts(1L, 0L, 0L, 0L);\n case \"adsr_icao\" -> new HistoricalSourceCounts(0L, 1L, 0L, 0L);\n case \"tisb_icao\" -> new HistoricalSourceCounts(0L, 0L, 1L, 0L);\n case \"mlat\" -> new HistoricalSourceCounts(0L, 0L, 0L, 1L);\n default -> throw new IllegalArgumentException(\"unsupported accepted historical position source: \" + positionSource);\n };\n }\n\n HistoricalSourceCounts plus(HistoricalSourceCounts other) {\n if (other == null) {\n throw new IllegalArgumentException(\"source counts to merge are required\");\n }\n return new HistoricalSourceCounts(\n Math.addExact(adsbIcao, other.adsbIcao),\n Math.addExact(adsrIcao, other.adsrIcao),\n Math.addExact(tisbIcao, other.tisbIcao),\n Math.addExact(mlat, other.mlat)\n );\n }\n\n long total() {\n return Math.addExact(Math.addExact(adsbIcao, adsrIcao), Math.addExact(tisbIcao, mlat));\n }\n\n String canonical() {\n return adsbIcao + \",\" + adsrIcao + \",\" + tisbIcao + \",\" + mlat;\n }\n}\n" }, { "path": "backend/historical-bootstrap-tool/src/main/java/hu/gombos/flightspotter/historicalbootstrap/HistoricalSourceTransportException.java", "encoding": "UTF-8", "sha256": "598a328fd98d94d362352b0fdae901c3627f944600c6c3598ac554eea4bdbd1e", "content": "package hu.gombos.flightspotter.historicalbootstrap;\n\nimport java.io.IOException;\n\n/**\n * Source-transport failure that may be retried by the bounded production historical feed.\n *\n *

This is intentionally distinct from immutable content-integrity failures and from local\n * filesystem/database failures. The feed may skip this source day for the current cycle and\n * continue with later days without pretending the failed day is qualified or permanently missing.

\n */\nfinal class HistoricalSourceTransportException extends IOException {\n HistoricalSourceTransportException(String message) {\n super(message);\n }\n\n HistoricalSourceTransportException(String message, Throwable cause) {\n super(message, cause);\n }\n}\n" }, { "path": "backend/historical-bootstrap-tool/src/main/java/hu/gombos/flightspotter/historicalbootstrap/ImmutableHistoricalSourceException.java", "encoding": "UTF-8", "sha256": "873de57b39a942ff1721b3dc456dd54afe73d5b60a3b85b22c9345bd6c362831", "content": "package hu.gombos.flightspotter.historicalbootstrap;\n\nimport java.io.IOException;\n\n/**\n * Terminal mismatch between immutable source metadata and the bytes actually served.\n *\n *

This is intentionally distinct from transient transport I/O. Once the bounded\n * verification path proves an immutable size/digest contradiction, retrying the same\n * source day cannot improve authority and must be sealed DEGRADED so later days can advance.

\n */\nfinal class ImmutableHistoricalSourceException extends IOException {\n private final String reasonCode;\n\n ImmutableHistoricalSourceException(String reasonCode, String message) {\n super(message);\n if (reasonCode == null || !reasonCode.matches(\"[A-Z0-9_]{1,64}\")) {\n throw new IllegalArgumentException(\"immutable source reason must be a bounded identifier\");\n }\n this.reasonCode = reasonCode;\n }\n\n String reasonCode() {\n return reasonCode;\n }\n}\n" }, { "path": "backend/historical-bootstrap-tool/src/main/java/hu/gombos/flightspotter/historicalbootstrap/LocalDayCommitStore.java", "encoding": "UTF-8", "sha256": "630cb7aaa7533f221492c760ff290e689cb441154050023607a85d0813d79113", "content": "package hu.gombos.flightspotter.historicalbootstrap;\n\nimport java.nio.file.Files;\nimport java.nio.file.Path;\nimport java.sql.Connection;\nimport java.sql.Date;\nimport java.sql.DriverManager;\nimport java.sql.PreparedStatement;\nimport java.sql.ResultSet;\nimport java.sql.SQLException;\nimport java.sql.Statement;\nimport java.time.Instant;\nimport java.time.LocalDate;\nimport java.util.Base64;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.TreeMap;\nimport java.util.UUID;\n\n/**\n * Local transactional P7 state. This is not V69 and is never a production service database.\n * A source day contributes only inside one H2 transaction that also writes its commit marker,\n * bounded corridor evidence and outgoing compact carry.\n */\nfinal class LocalDayCommitStore implements AutoCloseable {\n private final Connection connection;\n private final ArchiveSafetyLimits limits;\n\n private LocalDayCommitStore(Connection connection, ArchiveSafetyLimits limits) throws SQLException {\n this.connection = connection;\n this.limits = limits;\n initializeSchema();\n }\n\n static LocalDayCommitStore open(Path databaseFile, ArchiveSafetyLimits limits) throws Exception {\n if (databaseFile == null) throw new IllegalArgumentException(\"databaseFile is required\");\n Path absolute = databaseFile.toAbsolutePath().normalize();\n if (absolute.getParent() != null) Files.createDirectories(absolute.getParent());\n Connection connection = DriverManager.getConnection(\n \"jdbc:h2:file:\" + absolute + \";AUTO_SERVER=FALSE;DB_CLOSE_ON_EXIT=FALSE\");\n return new LocalDayCommitStore(connection, limits);\n }\n\n static LocalDayCommitStore openInMemory(String name, ArchiveSafetyLimits limits) throws SQLException {\n String safe = name == null ? UUID.randomUUID().toString() : name.replaceAll(\"[^A-Za-z0-9_-]\", \"_\");\n return new LocalDayCommitStore(\n DriverManager.getConnection(\"jdbc:h2:mem:\" + safe + \";DB_CLOSE_DELAY=-1\"), limits);\n }\n\n static void compactFully(Path databaseFile) throws Exception {\n if (databaseFile == null) throw new IllegalArgumentException(\"databaseFile is required\");\n Path absolute = databaseFile.toAbsolutePath().normalize();\n Path mvStore = absolute.resolveSibling(absolute.getFileName() + \".mv.db\");\n if (!Files.exists(mvStore)) return;\n try (Connection connection = DriverManager.getConnection(\n \"jdbc:h2:file:\" + absolute + \";IFEXISTS=TRUE;DB_CLOSE_ON_EXIT=FALSE\");\n Statement statement = connection.createStatement()) {\n statement.execute(\"SHUTDOWN COMPACT\");\n }\n }\n\n synchronized void apply(\n ReplayMode mode,\n DayIdentity identity,\n HistoricalDayDelta delta,\n Map outgoingCarry,\n EvidenceQualityStatus quality,\n FailureInjector injector\n ) throws SQLException {\n if (mode == null || identity == null) throw new IllegalArgumentException(\"mode and identity are required\");\n FailureInjector effective = injector == null ? FailureInjector.none() : injector;\n if (mode == ReplayMode.FULL_DAY_REBUILD) {\n if (delta == null || quality == null) {\n throw new IllegalArgumentException(\"FULL_DAY_REBUILD requires delta and quality\");\n }\n commitFullDay(identity, delta, safeCarry(outgoingCarry), quality, effective);\n } else {\n if (delta != null || quality != null) {\n throw new IllegalArgumentException(\"CARRY_ONLY_RECONSTRUCTION cannot receive contribution payload\");\n }\n reconstructCarry(identity, safeCarry(outgoingCarry), effective);\n }\n }\n\n synchronized boolean isCommitted(DayIdentity identity) throws SQLException {\n return committedDay(identity) != null;\n }\n\n synchronized CommittedDay committedDay(DayIdentity identity) throws SQLException {\n if (identity == null) throw new IllegalArgumentException(\"identity is required\");\n Marker marker = marker(identity.datasetId(), identity.sourceDate());\n if (marker == null) return null;\n verifyIdentity(marker, identity);\n return new CommittedDay(marker.deltaChecksum(), marker.quality());\n }\n\n synchronized Snapshot snapshot(UUID datasetId) throws SQLException {\n if (datasetId == null) throw new IllegalArgumentException(\"datasetId is required\");\n String dataset = datasetId.toString();\n TreeMap patterns = new TreeMap<>();\n try (PreparedStatement statement = connection.prepareStatement(\"\"\"\n select pattern_kind,subject_key,start_h3_r5,end_h3_r5,evidence_count,clean_evidence_count,\n first_occurrence_ms,last_occurrence_ms,minimum_altitude_ft,maximum_altitude_ft,\n mean_altitude_sum_ft,altitude_evidence_count,duration_seconds_sum,observation_count_sum,source_counts\n from p7_local_patterns where dataset_id=?\n order by pattern_kind,subject_key,start_h3_r5,end_h3_r5\n \"\"\")) {\n statement.setString(1, dataset);\n try (ResultSet rs = statement.executeQuery()) {\n while (rs.next()) {\n HistoricalPatternKey key = new HistoricalPatternKey(\n HistoricalPatternKind.valueOf(rs.getString(1)), rs.getString(2), rs.getString(3), rs.getString(4));\n patterns.put(key, new HistoricalPatternStats(\n rs.getLong(5), rs.getLong(6), Instant.ofEpochMilli(rs.getLong(7)), Instant.ofEpochMilli(rs.getLong(8)),\n rs.getInt(9), rs.getInt(10), rs.getLong(11), rs.getLong(12), rs.getLong(13), rs.getLong(14),\n decodeSourceCounts(rs.getString(15))));\n }\n }\n }\n\n TreeMap hourBins = new TreeMap<>();\n try (PreparedStatement statement = connection.prepareStatement(\"\"\"\n select pattern_kind,subject_key,start_h3_r5,end_h3_r5,hour_of_week,evidence_count\n from p7_local_hour_bins where dataset_id=?\n order by pattern_kind,subject_key,start_h3_r5,end_h3_r5,hour_of_week\n \"\"\")) {\n statement.setString(1, dataset);\n try (ResultSet rs = statement.executeQuery()) {\n while (rs.next()) {\n HistoricalPatternKey key = new HistoricalPatternKey(\n HistoricalPatternKind.valueOf(rs.getString(1)), rs.getString(2), rs.getString(3), rs.getString(4));\n hourBins.put(new HistoricalPatternHourKey(key, rs.getShort(5)), rs.getLong(6));\n }\n }\n }\n\n TreeMap corridors = new TreeMap<>();\n try (PreparedStatement statement = connection.prepareStatement(\"\"\"\n select start_h3_r5,end_h3_r5,corridor_h3_r7,traversal_count,progress_permille_sum,\n minimum_altitude_ft,maximum_altitude_ft,altitude_sum_ft,altitude_evidence_count\n from p7_local_corridor_aggregates where dataset_id=?\n order by start_h3_r5,end_h3_r5,corridor_h3_r7\n \"\"\")) {\n statement.setString(1, dataset);\n try (ResultSet rs = statement.executeQuery()) {\n while (rs.next()) {\n HistoricalRouteCorridorKey key = new HistoricalRouteCorridorKey(\n rs.getString(1), rs.getString(2), rs.getString(3));\n corridors.put(key, new HistoricalRouteCorridorStats(\n rs.getLong(4), rs.getLong(5), rs.getInt(6), rs.getInt(7), rs.getLong(8), rs.getLong(9)));\n }\n }\n }\n\n TreeMap holdout = new TreeMap<>();\n try (PreparedStatement statement = connection.prepareStatement(\"\"\"\n select segment_id,icao24,callsign,first_observed_ms,last_observed_ms,start_h3_r5,end_h3_r5,\n minimum_altitude_ft,maximum_altitude_ft,mean_altitude_ft,altitude_evidence_count,\n duration_seconds,observation_count,source_counts,corridor_overflow\n from p7_local_holdout where dataset_id=? order by segment_id\n \"\"\")) {\n statement.setString(1, dataset);\n try (ResultSet rs = statement.executeQuery()) {\n while (rs.next()) {\n String segmentId = rs.getString(1);\n boolean overflow = rs.getBoolean(15);\n HistoricalHoldoutOutcome outcome = new HistoricalHoldoutOutcome(\n segmentId, rs.getString(2), rs.getString(3), Instant.ofEpochMilli(rs.getLong(4)),\n Instant.ofEpochMilli(rs.getLong(5)), rs.getString(6), rs.getString(7), rs.getInt(8),\n rs.getInt(9), rs.getInt(10), rs.getLong(11), rs.getLong(12), rs.getLong(13),\n decodeSourceCounts(rs.getString(14)), readHoldoutCorridor(dataset, segmentId), overflow);\n holdout.put(outcome.segmentId(), outcome);\n }\n }\n }\n\n TreeMap carry = readCarry(dataset);\n int committedDays;\n try (PreparedStatement statement = connection.prepareStatement(\n \"select count(*) from p7_local_committed_days where dataset_id=?\")) {\n statement.setString(1, dataset);\n try (ResultSet rs = statement.executeQuery()) {\n rs.next();\n committedDays = rs.getInt(1);\n }\n }\n return new Snapshot(committedDays, Map.copyOf(patterns), Map.copyOf(hourBins), Map.copyOf(corridors),\n Map.copyOf(holdout), Map.copyOf(carry));\n }\n\n /**\n * Lightweight operational view for long Gate D execution.\n *\n * Unlike snapshot(), this never materializes historical aggregate rows,\n * HOLDOUT outcomes, or HOLDOUT corridor cells as Java objects.\n */\n synchronized ExecutionState executionState(UUID datasetId) throws SQLException {\n if (datasetId == null) throw new IllegalArgumentException(\"datasetId is required\");\n String dataset = datasetId.toString();\n return new ExecutionState(\n countRows(\"select count(*) from p7_local_committed_days where dataset_id=?\", dataset),\n countRows(\"select count(*) from p7_local_patterns where dataset_id=?\", dataset),\n countRows(\"select count(*) from p7_local_hour_bins where dataset_id=?\", dataset),\n countRows(\"select count(*) from p7_local_corridor_aggregates where dataset_id=?\", dataset),\n countRows(\"select count(*) from p7_local_holdout where dataset_id=?\", dataset),\n readCarry(dataset)\n );\n }\n\n private int countRows(String sql, String dataset) throws SQLException {\n try (PreparedStatement statement = connection.prepareStatement(sql)) {\n statement.setString(1, dataset);\n try (ResultSet rs = statement.executeQuery()) {\n rs.next();\n return Math.toIntExact(rs.getLong(1));\n }\n }\n }\n\n private void commitFullDay(\n DayIdentity identity,\n HistoricalDayDelta delta,\n Map outgoingCarry,\n EvidenceQualityStatus quality,\n FailureInjector injector\n ) throws SQLException {\n validateIdentity(identity);\n validateFullDay(delta, outgoingCarry, quality);\n Marker existing = marker(identity.datasetId(), identity.sourceDate());\n if (existing != null) {\n verifyIdentity(existing, identity);\n throw new IllegalStateException(\"committed source day cannot run FULL_DAY_REBUILD again\");\n }\n\n boolean committed = false;\n boolean restoreAutoCommit = true;\n connection.setAutoCommit(false);\n try {\n injector.hit(FailurePoint.BEFORE_DELTA_MERGE);\n for (Map.Entry entry : delta.patterns().entrySet()) {\n mergePattern(identity.datasetId(), entry.getKey(), entry.getValue());\n }\n for (Map.Entry entry : delta.hourBins().entrySet()) {\n mergeHourBin(identity.datasetId(), entry.getKey(), entry.getValue());\n }\n for (Map.Entry entry\n : delta.corridorAggregates().entrySet()) {\n mergeCorridor(identity.datasetId(), entry.getKey(), entry.getValue());\n }\n for (HistoricalHoldoutOutcome outcome : delta.holdoutOutcomes()) {\n insertHoldout(identity.datasetId(), identity.sourceDate(), outcome);\n }\n injector.hit(FailurePoint.AFTER_DELTA_MERGE);\n replaceCarries(identity.datasetId(), outgoingCarry);\n injector.hit(FailurePoint.AFTER_CARRY_REPLACE);\n insertMarker(identity, delta.checksum(), quality);\n injector.hit(FailurePoint.BEFORE_COMMIT);\n connection.commit();\n committed = true;\n injector.hit(FailurePoint.AFTER_COMMIT);\n } catch (SQLException | RuntimeException | Error failure) {\n if (!committed) {\n try {\n connection.rollback();\n } catch (SQLException rollbackFailure) {\n restoreAutoCommit = false;\n failure.addSuppressed(rollbackFailure);\n try {\n connection.close();\n } catch (SQLException closeFailure) {\n failure.addSuppressed(closeFailure);\n }\n }\n }\n throw failure;\n } finally {\n if (restoreAutoCommit) connection.setAutoCommit(true);\n }\n }\n\n private void reconstructCarry(\n DayIdentity identity,\n Map reconstructed,\n FailureInjector injector\n ) throws SQLException {\n validateIdentity(identity);\n Marker existing = marker(identity.datasetId(), identity.sourceDate());\n if (existing == null) {\n throw new IllegalStateException(\"carry-only reconstruction requires an already committed source day\");\n }\n verifyIdentity(existing, identity);\n LocalDate latest = latestCommittedDate(identity.datasetId());\n if (!identity.sourceDate().equals(latest)) {\n throw new IllegalStateException(\"carry-only reconstruction is allowed only for the latest committed source day\");\n }\n\n boolean committed = false;\n boolean restoreAutoCommit = true;\n connection.setAutoCommit(false);\n try {\n replaceCarries(identity.datasetId(), reconstructed);\n injector.hit(FailurePoint.AFTER_CARRY_REPLACE);\n injector.hit(FailurePoint.BEFORE_COMMIT);\n connection.commit();\n committed = true;\n injector.hit(FailurePoint.AFTER_COMMIT);\n } catch (SQLException | RuntimeException | Error failure) {\n if (!committed) {\n try {\n connection.rollback();\n } catch (SQLException rollbackFailure) {\n restoreAutoCommit = false;\n failure.addSuppressed(rollbackFailure);\n try {\n connection.close();\n } catch (SQLException closeFailure) {\n failure.addSuppressed(closeFailure);\n }\n }\n }\n throw failure;\n } finally {\n if (restoreAutoCommit) connection.setAutoCommit(true);\n }\n }\n\n private void mergePattern(UUID datasetId, HistoricalPatternKey key, HistoricalPatternStats incoming) throws SQLException {\n HistoricalPatternStats existing = null;\n try (PreparedStatement statement = connection.prepareStatement(\"\"\"\n select evidence_count,clean_evidence_count,first_occurrence_ms,last_occurrence_ms,\n minimum_altitude_ft,maximum_altitude_ft,mean_altitude_sum_ft,altitude_evidence_count,\n duration_seconds_sum,observation_count_sum,source_counts\n from p7_local_patterns\n where dataset_id=? and pattern_kind=? and subject_key=? and start_h3_r5=? and end_h3_r5=?\n \"\"\")) {\n bindPatternKey(statement, datasetId, key);\n try (ResultSet rs = statement.executeQuery()) {\n if (rs.next()) {\n existing = new HistoricalPatternStats(\n rs.getLong(1), rs.getLong(2), Instant.ofEpochMilli(rs.getLong(3)), Instant.ofEpochMilli(rs.getLong(4)),\n rs.getInt(5), rs.getInt(6), rs.getLong(7), rs.getLong(8), rs.getLong(9), rs.getLong(10),\n decodeSourceCounts(rs.getString(11)));\n }\n }\n }\n HistoricalPatternStats merged = existing == null ? incoming : HistoricalPatternStats.merge(existing, incoming);\n if (existing == null) {\n try (PreparedStatement statement = connection.prepareStatement(\"\"\"\n insert into p7_local_patterns(dataset_id,pattern_kind,subject_key,start_h3_r5,end_h3_r5,\n evidence_count,clean_evidence_count,first_occurrence_ms,last_occurrence_ms,minimum_altitude_ft,\n maximum_altitude_ft,mean_altitude_sum_ft,altitude_evidence_count,duration_seconds_sum,\n observation_count_sum,source_counts)\n values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)\n \"\"\")) {\n bindPatternKey(statement, datasetId, key);\n bindPatternStats(statement, 6, merged);\n statement.executeUpdate();\n }\n } else {\n try (PreparedStatement statement = connection.prepareStatement(\"\"\"\n update p7_local_patterns set evidence_count=?,clean_evidence_count=?,first_occurrence_ms=?,last_occurrence_ms=?,\n minimum_altitude_ft=?,maximum_altitude_ft=?,mean_altitude_sum_ft=?,altitude_evidence_count=?,\n duration_seconds_sum=?,observation_count_sum=?,source_counts=?\n where dataset_id=? and pattern_kind=? and subject_key=? and start_h3_r5=? and end_h3_r5=?\n \"\"\")) {\n bindPatternStats(statement, 1, merged);\n statement.setString(12, datasetId.toString());\n statement.setString(13, key.kind().name());\n statement.setString(14, key.subjectKey());\n statement.setString(15, key.startH3R5());\n statement.setString(16, key.endH3R5());\n statement.executeUpdate();\n }\n }\n }\n\n private void mergeHourBin(UUID datasetId, HistoricalPatternHourKey key, long incoming) throws SQLException {\n Long existing = null;\n try (PreparedStatement statement = connection.prepareStatement(\"\"\"\n select evidence_count from p7_local_hour_bins\n where dataset_id=? and pattern_kind=? and subject_key=? and start_h3_r5=? and end_h3_r5=? and hour_of_week=?\n \"\"\")) {\n bindPatternKey(statement, datasetId, key.patternKey());\n statement.setShort(6, key.hourOfWeek());\n try (ResultSet rs = statement.executeQuery()) {\n if (rs.next()) existing = rs.getLong(1);\n }\n }\n long merged = existing == null ? incoming : Math.addExact(existing, incoming);\n if (existing == null) {\n try (PreparedStatement statement = connection.prepareStatement(\"\"\"\n insert into p7_local_hour_bins(dataset_id,pattern_kind,subject_key,start_h3_r5,end_h3_r5,hour_of_week,evidence_count)\n values(?,?,?,?,?,?,?)\n \"\"\")) {\n bindPatternKey(statement, datasetId, key.patternKey());\n statement.setShort(6, key.hourOfWeek());\n statement.setLong(7, merged);\n statement.executeUpdate();\n }\n } else {\n try (PreparedStatement statement = connection.prepareStatement(\"\"\"\n update p7_local_hour_bins set evidence_count=?\n where dataset_id=? and pattern_kind=? and subject_key=? and start_h3_r5=? and end_h3_r5=? and hour_of_week=?\n \"\"\")) {\n statement.setLong(1, merged);\n statement.setString(2, datasetId.toString());\n statement.setString(3, key.patternKey().kind().name());\n statement.setString(4, key.patternKey().subjectKey());\n statement.setString(5, key.patternKey().startH3R5());\n statement.setString(6, key.patternKey().endH3R5());\n statement.setShort(7, key.hourOfWeek());\n statement.executeUpdate();\n }\n }\n }\n\n private void mergeCorridor(\n UUID datasetId,\n HistoricalRouteCorridorKey key,\n HistoricalRouteCorridorStats incoming\n ) throws SQLException {\n HistoricalRouteCorridorStats existing = null;\n try (PreparedStatement statement = connection.prepareStatement(\"\"\"\n select traversal_count,progress_permille_sum,minimum_altitude_ft,maximum_altitude_ft,\n altitude_sum_ft,altitude_evidence_count\n from p7_local_corridor_aggregates\n where dataset_id=? and start_h3_r5=? and end_h3_r5=? and corridor_h3_r7=?\n \"\"\")) {\n statement.setString(1, datasetId.toString());\n statement.setString(2, key.startH3R5());\n statement.setString(3, key.endH3R5());\n statement.setString(4, key.corridorH3R7());\n try (ResultSet rs = statement.executeQuery()) {\n if (rs.next()) {\n existing = new HistoricalRouteCorridorStats(\n rs.getLong(1), rs.getLong(2), rs.getInt(3), rs.getInt(4), rs.getLong(5), rs.getLong(6));\n }\n }\n }\n HistoricalRouteCorridorStats merged = existing == null\n ? incoming : HistoricalRouteCorridorStats.merge(existing, incoming);\n if (existing == null) {\n try (PreparedStatement statement = connection.prepareStatement(\"\"\"\n insert into p7_local_corridor_aggregates(\n dataset_id,start_h3_r5,end_h3_r5,corridor_h3_r7,traversal_count,progress_permille_sum,\n minimum_altitude_ft,maximum_altitude_ft,altitude_sum_ft,altitude_evidence_count)\n values(?,?,?,?,?,?,?,?,?,?)\n \"\"\")) {\n statement.setString(1, datasetId.toString());\n statement.setString(2, key.startH3R5());\n statement.setString(3, key.endH3R5());\n statement.setString(4, key.corridorH3R7());\n bindCorridorStats(statement, 5, merged);\n statement.executeUpdate();\n }\n } else {\n try (PreparedStatement statement = connection.prepareStatement(\"\"\"\n update p7_local_corridor_aggregates set traversal_count=?,progress_permille_sum=?,\n minimum_altitude_ft=?,maximum_altitude_ft=?,altitude_sum_ft=?,altitude_evidence_count=?\n where dataset_id=? and start_h3_r5=? and end_h3_r5=? and corridor_h3_r7=?\n \"\"\")) {\n bindCorridorStats(statement, 1, merged);\n statement.setString(7, datasetId.toString());\n statement.setString(8, key.startH3R5());\n statement.setString(9, key.endH3R5());\n statement.setString(10, key.corridorH3R7());\n statement.executeUpdate();\n }\n }\n }\n\n private void insertHoldout(UUID datasetId, LocalDate sourceDate, HistoricalHoldoutOutcome outcome) throws SQLException {\n try (PreparedStatement statement = connection.prepareStatement(\"\"\"\n insert into p7_local_holdout(dataset_id,source_date,segment_id,icao24,callsign,first_observed_ms,last_observed_ms,\n start_h3_r5,end_h3_r5,minimum_altitude_ft,maximum_altitude_ft,mean_altitude_ft,altitude_evidence_count,\n duration_seconds,observation_count,source_counts,corridor_overflow)\n values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)\n \"\"\")) {\n statement.setString(1, datasetId.toString());\n statement.setDate(2, Date.valueOf(sourceDate));\n statement.setString(3, outcome.segmentId());\n statement.setString(4, outcome.icao24());\n statement.setString(5, outcome.callsign());\n statement.setLong(6, outcome.firstObservedAt().toEpochMilli());\n statement.setLong(7, outcome.lastObservedAt().toEpochMilli());\n statement.setString(8, outcome.startH3R5());\n statement.setString(9, outcome.endH3R5());\n statement.setInt(10, outcome.minimumAltitudeFt());\n statement.setInt(11, outcome.maximumAltitudeFt());\n statement.setInt(12, outcome.meanAltitudeFt());\n statement.setLong(13, outcome.altitudeEvidenceCount());\n statement.setLong(14, outcome.durationSeconds());\n statement.setLong(15, outcome.observationCount());\n statement.setString(16, encodeSourceCounts(outcome.sourceCounts()));\n statement.setBoolean(17, outcome.corridorOverflow());\n statement.executeUpdate();\n }\n if (!outcome.corridorOverflow()) {\n try (PreparedStatement statement = connection.prepareStatement(\"\"\"\n insert into p7_local_holdout_corridor(\n dataset_id,segment_id,sequence_index,corridor_h3_r7,progress_permille,\n minimum_altitude_ft,maximum_altitude_ft,altitude_sum_ft,altitude_evidence_count)\n values(?,?,?,?,?,?,?,?,?)\n \"\"\")) {\n for (HistoricalCorridorCellEvidence cell : outcome.corridorCells()) {\n statement.setString(1, datasetId.toString());\n statement.setString(2, outcome.segmentId());\n statement.setInt(3, cell.sequenceIndex());\n statement.setString(4, cell.h3R7());\n statement.setInt(5, cell.firstEntryProgressPermille());\n statement.setInt(6, cell.minimumAltitudeFt());\n statement.setInt(7, cell.maximumAltitudeFt());\n statement.setLong(8, cell.altitudeSumFt());\n statement.setLong(9, cell.altitudeEvidenceCount());\n statement.addBatch();\n }\n statement.executeBatch();\n }\n }\n }\n\n private List readHoldoutCorridor(String datasetId, String segmentId) throws SQLException {\n java.util.ArrayList result = new java.util.ArrayList<>();\n try (PreparedStatement statement = connection.prepareStatement(\"\"\"\n select sequence_index,corridor_h3_r7,progress_permille,minimum_altitude_ft,maximum_altitude_ft,\n altitude_sum_ft,altitude_evidence_count\n from p7_local_holdout_corridor where dataset_id=? and segment_id=? order by sequence_index\n \"\"\")) {\n statement.setString(1, datasetId);\n statement.setString(2, segmentId);\n try (ResultSet rs = statement.executeQuery()) {\n while (rs.next()) {\n result.add(new HistoricalCorridorCellEvidence(\n rs.getInt(1), rs.getString(2), rs.getInt(3), rs.getInt(4), rs.getInt(5),\n rs.getLong(6), rs.getLong(7)));\n }\n }\n }\n return List.copyOf(result);\n }\n\n private void replaceCarries(UUID datasetId, Map carries) throws SQLException {\n if (carries.size() > limits.maxCarryAircraft()) {\n throw new IllegalArgumentException(\"outgoing carry exceeds configured aircraft bound\");\n }\n try (PreparedStatement delete = connection.prepareStatement(\"delete from p7_local_carry where dataset_id=?\")) {\n delete.setString(1, datasetId.toString());\n delete.executeUpdate();\n }\n try (PreparedStatement insert = connection.prepareStatement(\"\"\"\n insert into p7_local_carry(dataset_id,icao24,first_observed_ms,last_observed_ms,observation_count,\n last_latitude,last_longitude,start_h3_r5,end_h3_r5,minimum_altitude_ft,maximum_altitude_ft,\n altitude_sum_ft,altitude_evidence_count,source_counts,callsign_evidence,primary_callsign,boundary_truncated,\n corridor_state,corridor_overflow)\n values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)\n \"\"\")) {\n for (Map.Entry entry : new TreeMap<>(carries).entrySet()) {\n HistoricalSegmentCarry carry = entry.getValue();\n if (carry == null || !entry.getKey().equals(carry.icao24())) {\n throw new IllegalArgumentException(\"carry map key must equal carry ICAO24\");\n }\n insert.setString(1, datasetId.toString());\n insert.setString(2, carry.icao24());\n insert.setLong(3, carry.firstObservedAt().toEpochMilli());\n insert.setLong(4, carry.lastObservedAt().toEpochMilli());\n insert.setLong(5, carry.observationCount());\n insert.setDouble(6, carry.lastLatitude());\n insert.setDouble(7, carry.lastLongitude());\n insert.setString(8, carry.startH3R5());\n insert.setString(9, carry.endH3R5());\n insert.setInt(10, carry.minimumAltitudeFt());\n insert.setInt(11, carry.maximumAltitudeFt());\n insert.setLong(12, carry.altitudeSumFt());\n insert.setLong(13, carry.altitudeEvidenceCount());\n insert.setString(14, encodeSourceCounts(carry.sourceCounts()));\n insert.setString(15, encodeList(carry.callsignEvidence()));\n insert.setString(16, carry.primaryCallsign());\n insert.setBoolean(17, carry.boundaryTruncated());\n insert.setString(18, encodeCorridorCarry(carry.corridorCells()));\n insert.setBoolean(19, carry.corridorOverflow());\n insert.addBatch();\n }\n insert.executeBatch();\n }\n }\n\n private TreeMap readCarry(String datasetId) throws SQLException {\n TreeMap result = new TreeMap<>();\n try (PreparedStatement statement = connection.prepareStatement(\"\"\"\n select icao24,first_observed_ms,last_observed_ms,observation_count,last_latitude,last_longitude,\n start_h3_r5,end_h3_r5,minimum_altitude_ft,maximum_altitude_ft,altitude_sum_ft,\n altitude_evidence_count,source_counts,callsign_evidence,primary_callsign,boundary_truncated,\n corridor_state,corridor_overflow\n from p7_local_carry where dataset_id=? order by icao24\n \"\"\")) {\n statement.setString(1, datasetId);\n try (ResultSet rs = statement.executeQuery()) {\n while (rs.next()) {\n HistoricalSegmentCarry carry = new HistoricalSegmentCarry(\n rs.getString(1), Instant.ofEpochMilli(rs.getLong(2)), Instant.ofEpochMilli(rs.getLong(3)),\n rs.getLong(4), rs.getDouble(5), rs.getDouble(6), rs.getString(7), rs.getString(8),\n rs.getInt(9), rs.getInt(10), rs.getLong(11), rs.getLong(12), decodeSourceCounts(rs.getString(13)),\n decodeList(rs.getString(14)), rs.getString(15), rs.getBoolean(16),\n decodeCorridorCarry(rs.getString(17)), rs.getBoolean(18));\n result.put(carry.icao24(), carry);\n }\n }\n }\n return result;\n }\n\n private void insertMarker(DayIdentity identity, String deltaChecksum, EvidenceQualityStatus quality) throws SQLException {\n try (PreparedStatement statement = connection.prepareStatement(\"\"\"\n insert into p7_local_committed_days(dataset_id,source_date,day_manifest_hash,semantic_fingerprint,\n delta_checksum,evidence_quality) values(?,?,?,?,?,?)\n \"\"\")) {\n statement.setString(1, identity.datasetId().toString());\n statement.setDate(2, Date.valueOf(identity.sourceDate()));\n statement.setString(3, identity.dayManifestHash());\n statement.setString(4, identity.semanticFingerprint());\n statement.setString(5, deltaChecksum);\n statement.setString(6, quality.name());\n statement.executeUpdate();\n }\n }\n\n private Marker marker(UUID datasetId, LocalDate sourceDate) throws SQLException {\n try (PreparedStatement statement = connection.prepareStatement(\"\"\"\n select day_manifest_hash,semantic_fingerprint,delta_checksum,evidence_quality\n from p7_local_committed_days where dataset_id=? and source_date=?\n \"\"\")) {\n statement.setString(1, datasetId.toString());\n statement.setDate(2, Date.valueOf(sourceDate));\n try (ResultSet rs = statement.executeQuery()) {\n if (!rs.next()) return null;\n return new Marker(rs.getString(1), rs.getString(2), rs.getString(3),\n EvidenceQualityStatus.valueOf(rs.getString(4)));\n }\n }\n }\n\n private LocalDate latestCommittedDate(UUID datasetId) throws SQLException {\n try (PreparedStatement statement = connection.prepareStatement(\n \"select max(source_date) from p7_local_committed_days where dataset_id=?\")) {\n statement.setString(1, datasetId.toString());\n try (ResultSet rs = statement.executeQuery()) {\n if (!rs.next() || rs.getDate(1) == null) return null;\n return rs.getDate(1).toLocalDate();\n }\n }\n }\n\n private static void verifyIdentity(Marker marker, DayIdentity identity) {\n if (!marker.dayManifestHash().equals(identity.dayManifestHash())\n || !marker.semanticFingerprint().equals(identity.semanticFingerprint())) {\n throw new IllegalStateException(\"committed day identity does not match requested replay identity\");\n }\n }\n\n private static void validateIdentity(DayIdentity identity) {\n if (identity.datasetId() == null || identity.sourceDate() == null\n || identity.dayManifestHash() == null || !identity.dayManifestHash().matches(\"[0-9a-f]{64}\")\n || identity.semanticFingerprint() == null || !identity.semanticFingerprint().matches(\"[0-9a-f]{64}\")) {\n throw new IllegalArgumentException(\"invalid immutable source-day identity\");\n }\n }\n\n private static void validateFullDay(\n HistoricalDayDelta delta,\n Map carry,\n EvidenceQualityStatus quality\n ) {\n if (quality != EvidenceQualityStatus.QUALIFIED && (delta.hasEvidence() || !carry.isEmpty())) {\n throw new IllegalArgumentException(\"DEGRADED/GAP day cannot emit clean evidence or carry in v1\");\n }\n if (delta.role() == DatasetWindow.DateRole.EMBARGO && (delta.hasEvidence() || !carry.isEmpty())) {\n throw new IllegalArgumentException(\"EMBARGO day must emit neither evidence nor carry\");\n }\n }\n\n private static Map safeCarry(Map carry) {\n return carry == null ? Map.of() : Map.copyOf(carry);\n }\n\n private static void bindPatternKey(PreparedStatement statement, UUID datasetId, HistoricalPatternKey key)\n throws SQLException {\n statement.setString(1, datasetId.toString());\n statement.setString(2, key.kind().name());\n statement.setString(3, key.subjectKey());\n statement.setString(4, key.startH3R5());\n statement.setString(5, key.endH3R5());\n }\n\n private static void bindPatternStats(PreparedStatement statement, int offset, HistoricalPatternStats stats)\n throws SQLException {\n statement.setLong(offset, stats.evidenceCount());\n statement.setLong(offset + 1, stats.cleanEvidenceCount());\n statement.setLong(offset + 2, stats.firstOccurrenceAt().toEpochMilli());\n statement.setLong(offset + 3, stats.lastOccurrenceAt().toEpochMilli());\n statement.setInt(offset + 4, stats.minimumAltitudeFt());\n statement.setInt(offset + 5, stats.maximumAltitudeFt());\n statement.setLong(offset + 6, stats.meanAltitudeSumFt());\n statement.setLong(offset + 7, stats.altitudeEvidenceCount());\n statement.setLong(offset + 8, stats.durationSecondsSum());\n statement.setLong(offset + 9, stats.observationCountSum());\n statement.setString(offset + 10, encodeSourceCounts(stats.sourceCounts()));\n }\n\n private static void bindCorridorStats(PreparedStatement statement, int offset, HistoricalRouteCorridorStats stats)\n throws SQLException {\n statement.setLong(offset, stats.traversalCount());\n statement.setLong(offset + 1, stats.progressPermilleSum());\n statement.setInt(offset + 2, stats.minimumAltitudeFt());\n statement.setInt(offset + 3, stats.maximumAltitudeFt());\n statement.setLong(offset + 4, stats.altitudeSumFt());\n statement.setLong(offset + 5, stats.altitudeEvidenceCount());\n }\n\n private static String encodeSourceCounts(HistoricalSourceCounts counts) {\n if (counts == null) throw new IllegalArgumentException(\"source counts are required\");\n return counts.canonical();\n }\n\n private static HistoricalSourceCounts decodeSourceCounts(String encoded) {\n if (encoded == null) throw new IllegalStateException(\"source counts are missing from local state\");\n String[] values = encoded.split(\",\", -1);\n if (values.length != 4) throw new IllegalStateException(\"invalid local source-count encoding\");\n try {\n return new HistoricalSourceCounts(\n Long.parseLong(values[0]), Long.parseLong(values[1]),\n Long.parseLong(values[2]), Long.parseLong(values[3]));\n } catch (NumberFormatException exception) {\n throw new IllegalStateException(\"invalid local source-count encoding\", exception);\n }\n }\n\n private static String encodeList(List values) {\n Base64.Encoder encoder = Base64.getUrlEncoder().withoutPadding();\n return values.stream().map(value -> encoder.encodeToString(value.getBytes(java.nio.charset.StandardCharsets.UTF_8)))\n .reduce((left, right) -> left + \".\" + right).orElse(\"\");\n }\n\n private static List decodeList(String encoded) {\n if (encoded == null || encoded.isBlank()) return List.of();\n Base64.Decoder decoder = Base64.getUrlDecoder();\n return java.util.Arrays.stream(encoded.split(\"\\\\.\"))\n .map(value -> new String(decoder.decode(value), java.nio.charset.StandardCharsets.UTF_8))\n .toList();\n }\n\n private static String encodeCorridorCarry(List cells) {\n if (cells == null || cells.isEmpty()) return \"\";\n String encoded = cells.stream().map(HistoricalCorridorCellCarry::canonical)\n .reduce((left, right) -> left + \";\" + right).orElse(\"\");\n if (encoded.length() > 65_535) throw new IllegalArgumentException(\"encoded corridor carry exceeds hard bound\");\n return encoded;\n }\n\n private static List decodeCorridorCarry(String encoded) {\n if (encoded == null || encoded.isBlank()) return List.of();\n java.util.ArrayList result = new java.util.ArrayList<>();\n for (String item : encoded.split(\";\", -1)) {\n String[] values = item.split(\"\\\\|\", -1);\n if (values.length != 6) throw new IllegalStateException(\"invalid local corridor carry encoding\");\n try {\n result.add(new HistoricalCorridorCellCarry(\n values[0], Instant.ofEpochMilli(Long.parseLong(values[1])),\n Integer.parseInt(values[2]), Integer.parseInt(values[3]),\n Long.parseLong(values[4]), Long.parseLong(values[5])));\n } catch (RuntimeException exception) {\n throw new IllegalStateException(\"invalid local corridor carry encoding\", exception);\n }\n }\n if (result.size() > HistoricalCorridorModel.MAX_UNIQUE_CELLS) {\n throw new IllegalStateException(\"local corridor carry exceeds bounded cell cap\");\n }\n return List.copyOf(result);\n }\n\n private void initializeSchema() throws SQLException {\n try (Statement statement = connection.createStatement()) {\n statement.execute(\"\"\"\n create table if not exists p7_local_committed_days(\n dataset_id varchar(36) not null,\n source_date date not null,\n day_manifest_hash varchar(64) not null,\n semantic_fingerprint varchar(64) not null,\n delta_checksum varchar(64) not null,\n evidence_quality varchar(16) not null,\n primary key(dataset_id,source_date))\n \"\"\");\n statement.execute(\"\"\"\n create table if not exists p7_local_patterns(\n dataset_id varchar(36) not null, pattern_kind varchar(24) not null, subject_key varchar(64) not null,\n start_h3_r5 varchar(15) not null, end_h3_r5 varchar(15) not null,\n evidence_count bigint not null, clean_evidence_count bigint not null,\n first_occurrence_ms bigint not null, last_occurrence_ms bigint not null,\n minimum_altitude_ft int not null, maximum_altitude_ft int not null,\n mean_altitude_sum_ft bigint not null, altitude_evidence_count bigint not null,\n duration_seconds_sum bigint not null, observation_count_sum bigint not null,\n source_counts varchar(128) not null,\n primary key(dataset_id,pattern_kind,subject_key,start_h3_r5,end_h3_r5))\n \"\"\");\n statement.execute(\"\"\"\n create table if not exists p7_local_hour_bins(\n dataset_id varchar(36) not null, pattern_kind varchar(24) not null, subject_key varchar(64) not null,\n start_h3_r5 varchar(15) not null, end_h3_r5 varchar(15) not null,\n hour_of_week smallint not null, evidence_count bigint not null,\n primary key(dataset_id,pattern_kind,subject_key,start_h3_r5,end_h3_r5,hour_of_week))\n \"\"\");\n statement.execute(\"\"\"\n create table if not exists p7_local_corridor_aggregates(\n dataset_id varchar(36) not null, start_h3_r5 varchar(15) not null, end_h3_r5 varchar(15) not null,\n corridor_h3_r7 varchar(15) not null, traversal_count bigint not null, progress_permille_sum bigint not null,\n minimum_altitude_ft int not null, maximum_altitude_ft int not null, altitude_sum_ft bigint not null,\n altitude_evidence_count bigint not null,\n primary key(dataset_id,start_h3_r5,end_h3_r5,corridor_h3_r7))\n \"\"\");\n statement.execute(\"\"\"\n create table if not exists p7_local_holdout(\n dataset_id varchar(36) not null, source_date date not null, segment_id varchar(64) not null,\n icao24 varchar(6) not null, callsign varchar(64), first_observed_ms bigint not null,\n last_observed_ms bigint not null, start_h3_r5 varchar(15) not null, end_h3_r5 varchar(15) not null,\n minimum_altitude_ft int not null, maximum_altitude_ft int not null, mean_altitude_ft int not null,\n altitude_evidence_count bigint not null, duration_seconds bigint not null, observation_count bigint not null,\n source_counts varchar(128) not null, corridor_overflow boolean not null default true,\n primary key(dataset_id,segment_id))\n \"\"\");\n statement.execute(\"alter table p7_local_holdout add column if not exists corridor_overflow boolean not null default true\");\n statement.execute(\"\"\"\n create table if not exists p7_local_holdout_corridor(\n dataset_id varchar(36) not null, segment_id varchar(64) not null, sequence_index int not null,\n corridor_h3_r7 varchar(15) not null, progress_permille int not null,\n minimum_altitude_ft int not null, maximum_altitude_ft int not null, altitude_sum_ft bigint not null,\n altitude_evidence_count bigint not null,\n primary key(dataset_id,segment_id,sequence_index),\n unique(dataset_id,segment_id,corridor_h3_r7))\n \"\"\");\n statement.execute(\"\"\"\n create table if not exists p7_local_carry(\n dataset_id varchar(36) not null, icao24 varchar(6) not null,\n first_observed_ms bigint not null, last_observed_ms bigint not null, observation_count bigint not null,\n last_latitude double precision not null, last_longitude double precision not null,\n start_h3_r5 varchar(15) not null, end_h3_r5 varchar(15) not null,\n minimum_altitude_ft int not null, maximum_altitude_ft int not null,\n altitude_sum_ft bigint not null, altitude_evidence_count bigint not null,\n source_counts varchar(128) not null,\n callsign_evidence varchar(2048) not null, primary_callsign varchar(128), boundary_truncated boolean not null,\n corridor_state varchar(65535) not null default '', corridor_overflow boolean not null default true,\n primary key(dataset_id,icao24))\n \"\"\");\n statement.execute(\"alter table p7_local_carry add column if not exists corridor_state varchar(65535) not null default ''\");\n statement.execute(\"alter table p7_local_carry add column if not exists corridor_overflow boolean not null default true\");\n }\n }\n\n @Override\n public synchronized void close() throws SQLException {\n connection.close();\n }\n\n enum ReplayMode {\n FULL_DAY_REBUILD,\n CARRY_ONLY_RECONSTRUCTION\n }\n\n enum EvidenceQualityStatus {\n QUALIFIED,\n DEGRADED,\n GAP\n }\n\n enum FailurePoint {\n BEFORE_DELTA_MERGE,\n AFTER_DELTA_MERGE,\n AFTER_CARRY_REPLACE,\n BEFORE_COMMIT,\n AFTER_COMMIT\n }\n\n @FunctionalInterface\n interface FailureInjector {\n void hit(FailurePoint point);\n\n static FailureInjector none() {\n return ignored -> { };\n }\n }\n\n record DayIdentity(UUID datasetId, LocalDate sourceDate, String dayManifestHash, String semanticFingerprint) {\n }\n\n record ExecutionState(\n int committedDayCount,\n int patternCount,\n int hourBinCount,\n int corridorAggregateCount,\n int holdoutOutcomeCount,\n Map carry\n ) {\n ExecutionState {\n if (committedDayCount < 0 || patternCount < 0 || hourBinCount < 0\n || corridorAggregateCount < 0 || holdoutOutcomeCount < 0 || carry == null) {\n throw new IllegalArgumentException(\"invalid lightweight execution state\");\n }\n carry = Map.copyOf(carry);\n }\n\n boolean hasAnyEvidence() {\n return patternCount > 0 || hourBinCount > 0 || corridorAggregateCount > 0\n || holdoutOutcomeCount > 0 || !carry.isEmpty();\n }\n }\n\n record Snapshot(\n int committedDayCount,\n Map patterns,\n Map hourBins,\n Map corridorAggregates,\n Map holdoutOutcomes,\n Map carry\n ) {\n }\n\n record CommittedDay(String deltaChecksum, EvidenceQualityStatus quality) {\n CommittedDay {\n if (deltaChecksum == null || !deltaChecksum.matches(\"[0-9a-f]{64}\") || quality == null) {\n throw new IllegalArgumentException(\"invalid committed day evidence\");\n }\n }\n }\n\n private record Marker(\n String dayManifestHash,\n String semanticFingerprint,\n String deltaChecksum,\n EvidenceQualityStatus quality\n ) {\n }\n}\n" }, { "path": "backend/historical-bootstrap-tool/src/main/java/hu/gombos/flightspotter/historicalbootstrap/ReadsbTrace.java", "encoding": "UTF-8", "sha256": "cd81b1c6d0e651fcd5bd84a09f28f067c501bf1c2560760f15c770093e0aaf9b", "content": "package hu.gombos.flightspotter.historicalbootstrap;\n\nimport java.time.Instant;\nimport java.util.List;\n\nrecord ReadsbTrace(\n String icao24,\n String registration,\n String typeCode,\n Instant baseTimestamp,\n List points,\n int malformedPointCount\n) {\n ReadsbTrace {\n points = List.copyOf(points);\n if (malformedPointCount < 0) {\n throw new IllegalArgumentException(\"malformedPointCount cannot be negative\");\n }\n }\n}\n\nrecord ReadsbTracePoint(\n Instant observedAt,\n Double latitude,\n Double longitude,\n Double altitudeFt,\n HistoricalAltitudeSource altitudeSource,\n Double groundSpeedKt,\n String callsign,\n String positionSource,\n boolean stalePosition,\n boolean newLeg,\n boolean onGround\n) {\n}\n\nenum HistoricalAltitudeSource {\n GEOMETRIC_TUPLE,\n GEOMETRIC_FLAGGED_BASE,\n GEOMETRIC_DETAILS,\n BAROMETRIC_FALLBACK,\n MISSING_OR_GROUND\n}\n" }, { "path": "backend/historical-bootstrap-tool/src/main/java/hu/gombos/flightspotter/historicalbootstrap/ReadsbTraceParser.java", "encoding": "UTF-8", "sha256": "b4a4ac03b306074e5c7c9968cce7292b3714ee5bf1eee1ff3f688ecdba4fbe85", "content": "package hu.gombos.flightspotter.historicalbootstrap;\n\nimport tools.jackson.core.JacksonException;\nimport tools.jackson.databind.JsonNode;\nimport tools.jackson.databind.ObjectMapper;\nimport tools.jackson.databind.json.JsonMapper;\n\nimport java.io.ByteArrayInputStream;\nimport java.io.FilterInputStream;\nimport java.io.IOException;\nimport java.io.InputStream;\nimport java.time.DateTimeException;\nimport java.time.Instant;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.Locale;\nimport java.util.zip.GZIPInputStream;\n\n/** Parser contract for readsb trace JSON used by p7-source-eligibility-v1. */\nfinal class ReadsbTraceParser {\n private static final int FLAG_STALE_POSITION = 1;\n private static final int FLAG_NEW_LEG = 2;\n private static final int FLAG_ALTITUDE_GEOMETRIC = 8;\n\n private final ArchiveSafetyLimits limits;\n private final ObjectMapper mapper;\n\n ReadsbTraceParser(ArchiveSafetyLimits limits) {\n this(limits, JsonMapper.builder().build());\n }\n\n ReadsbTraceParser(ArchiveSafetyLimits limits, ObjectMapper mapper) {\n this.limits = limits;\n this.mapper = mapper;\n }\n\n ReadsbTrace parseGzip(byte[] compressed) throws IOException {\n if (compressed == null || compressed.length == 0\n || compressed.length > limits.maxCompressedTraceBytes()) {\n throw new IOException(\"compressed trace outside configured hard bound\");\n }\n // Do not materialize a second maxInflatedTraceBytes-sized byte[] before Jackson parsing.\n // The bounded stream keeps inflation fail-closed while avoiding a large duplicate heap copy.\n try (GZIPInputStream gzip = new GZIPInputStream(new ByteArrayInputStream(compressed));\n InputStream bounded = new InflatedLimitInputStream(gzip, limits.maxInflatedTraceBytes())) {\n try {\n return parseRoot(mapper.readTree(bounded));\n } catch (JacksonException jacksonFailure) {\n // Jackson 3 makes syntax/databind exceptions unchecked. Normalize every Jackson-produced\n // read failure back to the checked parser boundary so source-day parse accounting remains\n // deterministic and malformed JSON cannot bypass the configured content policy.\n throw new IOException(\"trace JSON read failed: \" + jacksonFailure.getMessage(), jacksonFailure);\n }\n }\n }\n\n ReadsbTrace parseJson(byte[] json) throws IOException {\n if (json == null || json.length == 0 || json.length > limits.maxInflatedTraceBytes()) {\n throw new IOException(\"trace JSON outside configured hard bound\");\n }\n try {\n return parseRoot(mapper.readTree(json));\n } catch (JacksonException jacksonFailure) {\n throw new IOException(\"trace JSON read failed: \" + jacksonFailure.getMessage(), jacksonFailure);\n }\n }\n\n private ReadsbTrace parseRoot(JsonNode root) throws IOException {\n if (root == null || !root.isObject()) {\n throw new IOException(\"trace JSON root must be an object\");\n }\n String icao = normalizeIcao(text(root.get(\"icao\")));\n if (icao == null) {\n throw new IOException(\"trace JSON is missing aircraft identity\");\n }\n Double baseEpoch = finiteNumber(root.get(\"timestamp\"));\n if (baseEpoch == null) {\n throw new IOException(\"trace JSON has invalid base timestamp\");\n }\n Instant baseTimestamp = instant(baseEpoch);\n JsonNode traceNode = root.get(\"trace\");\n if (traceNode == null || !traceNode.isArray()) {\n throw new IOException(\"trace JSON is missing trace array\");\n }\n int tracePointCount = traceNode.size();\n if (tracePointCount > limits.maxTracePoints()) {\n throw new IOException(\"trace point count exceeds configured hard bound\"\n + \": icao=\" + icao\n + \" actualTracePoints=\" + tracePointCount\n + \" configuredMaxTracePoints=\" + limits.maxTracePoints());\n }\n\n String registration = normalizeUpper(text(root.get(\"r\")));\n String typeCode = normalizeUpper(text(root.get(\"t\")));\n String currentCallsign = null;\n int malformed = 0;\n List points = new ArrayList<>(tracePointCount);\n\n for (JsonNode tuple : traceNode) {\n try {\n ParsedPoint parsed = parsePoint(tuple, baseEpoch, currentCallsign);\n currentCallsign = parsed.callsignState();\n points.add(parsed.point());\n } catch (IllegalArgumentException | DateTimeException exception) {\n malformed = Math.addExact(malformed, 1);\n }\n }\n return new ReadsbTrace(icao, registration, typeCode, baseTimestamp, points, malformed);\n }\n\n private ParsedPoint parsePoint(JsonNode tuple, double baseEpoch, String priorCallsign) {\n if (tuple == null || !tuple.isArray() || tuple.size() < 7) {\n throw new IllegalArgumentException(\"trace point tuple is structurally incomplete\");\n }\n Double offset = finiteNumber(tuple.get(0));\n if (offset == null) {\n throw new IllegalArgumentException(\"trace point offset is invalid\");\n }\n Instant observedAt = instant(baseEpoch + offset);\n Double latitude = finiteNumber(tuple.get(1));\n Double longitude = finiteNumber(tuple.get(2));\n JsonNode baseAltitudeNode = tuple.get(3);\n Double baseAltitude = finiteNumber(baseAltitudeNode);\n boolean onGround = baseAltitudeNode != null && baseAltitudeNode.isString()\n && \"ground\".equalsIgnoreCase(baseAltitudeNode.stringValue());\n Double groundSpeed = finiteNumber(tuple.get(4));\n Integer flags = integer(tuple.get(6));\n if (flags == null || flags < 0) {\n throw new IllegalArgumentException(\"trace point flags are invalid\");\n }\n\n JsonNode details = tuple.size() > 8 && tuple.get(8) != null && tuple.get(8).isObject()\n ? tuple.get(8) : null;\n String callsign = priorCallsign;\n if (details != null && details.has(\"flight\")) {\n callsign = normalizeCallsign(text(details.get(\"flight\")));\n }\n\n // For 2022+ readsb traces, tuple[9] is serialized from this trace state's own addrtype.\n // details.type belongs to the aircraft detail object and is not a substitute for the point source.\n // Missing/null tuple source therefore stays unknown and clean v1 eligibility rejects it fail-closed.\n String positionSource = tuple.size() > 9 ? normalizeLower(text(tuple.get(9))) : null;\n\n Double tupleAltGeom = tuple.size() > 10 ? finiteNumber(tuple.get(10)) : null;\n Double detailsAltGeom = details == null ? null : finiteNumber(details.get(\"alt_geom\"));\n Altitude altitude = altitude(baseAltitude, tupleAltGeom, detailsAltGeom, flags, onGround);\n\n ReadsbTracePoint point = new ReadsbTracePoint(\n observedAt,\n latitude,\n longitude,\n altitude.value(),\n altitude.source(),\n groundSpeed,\n callsign,\n positionSource,\n (flags & FLAG_STALE_POSITION) != 0,\n (flags & FLAG_NEW_LEG) != 0,\n onGround\n );\n return new ParsedPoint(point, callsign);\n }\n\n /**\n * Frozen p7-source-eligibility-v1 altitude precedence:\n * 1) 2022+ tuple geometric altitude, 2) base altitude when readsb flag says it is geometric,\n * 3) details.alt_geom, 4) numeric base/barometric altitude. Ground/null remains unusable.\n */\n private static Altitude altitude(\n Double baseAltitude,\n Double tupleAltGeom,\n Double detailsAltGeom,\n int flags,\n boolean onGround\n ) {\n if (onGround) {\n return new Altitude(null, HistoricalAltitudeSource.MISSING_OR_GROUND);\n }\n if (tupleAltGeom != null) {\n return new Altitude(tupleAltGeom, HistoricalAltitudeSource.GEOMETRIC_TUPLE);\n }\n if (baseAltitude != null && (flags & FLAG_ALTITUDE_GEOMETRIC) != 0) {\n return new Altitude(baseAltitude, HistoricalAltitudeSource.GEOMETRIC_FLAGGED_BASE);\n }\n if (detailsAltGeom != null) {\n return new Altitude(detailsAltGeom, HistoricalAltitudeSource.GEOMETRIC_DETAILS);\n }\n if (baseAltitude != null) {\n return new Altitude(baseAltitude, HistoricalAltitudeSource.BAROMETRIC_FALLBACK);\n }\n return new Altitude(null, HistoricalAltitudeSource.MISSING_OR_GROUND);\n }\n\n private static Instant instant(double epochSeconds) {\n if (!Double.isFinite(epochSeconds)) {\n throw new IllegalArgumentException(\"timestamp is not finite\");\n }\n long seconds = (long) Math.floor(epochSeconds);\n long nanos = Math.round((epochSeconds - seconds) * 1_000_000_000d);\n if (nanos == 1_000_000_000L) {\n seconds = Math.addExact(seconds, 1L);\n nanos = 0L;\n }\n return Instant.ofEpochSecond(seconds, nanos);\n }\n\n private static String normalizeIcao(String value) {\n String normalized = normalizeLower(value);\n return normalized == null ? null : normalized;\n }\n\n private static String normalizeCallsign(String value) {\n String normalized = normalizeUpper(value);\n return normalized != null && normalized.length() <= 32 ? normalized : null;\n }\n\n private static String normalizeUpper(String value) {\n if (value == null || value.isBlank()) {\n return null;\n }\n return value.trim().toUpperCase(Locale.ROOT);\n }\n\n private static String normalizeLower(String value) {\n if (value == null || value.isBlank()) {\n return null;\n }\n return value.trim().toLowerCase(Locale.ROOT);\n }\n\n private static String text(JsonNode node) {\n return node != null && node.isString() ? node.stringValue() : null;\n }\n\n private static Double finiteNumber(JsonNode node) {\n if (node == null || !node.isNumber()) {\n return null;\n }\n double value = node.doubleValue();\n return Double.isFinite(value) ? value : null;\n }\n\n private static Integer integer(JsonNode node) {\n if (node == null || !node.isIntegralNumber() || !node.canConvertToInt()) {\n return null;\n }\n return node.intValue();\n }\n\n private record Altitude(Double value, HistoricalAltitudeSource source) {\n }\n\n private record ParsedPoint(ReadsbTracePoint point, String callsignState) {\n }\n\n private static final class InflatedLimitInputStream extends FilterInputStream {\n private final long maximumBytes;\n private long consumed;\n\n private InflatedLimitInputStream(InputStream input, long maximumBytes) {\n super(input);\n if (maximumBytes < 1L) throw new IllegalArgumentException(\"maximumBytes must be positive\");\n this.maximumBytes = maximumBytes;\n }\n\n @Override\n public int read() throws IOException {\n int value = super.read();\n if (value >= 0) account(1L);\n return value;\n }\n\n @Override\n public int read(byte[] buffer, int offset, int length) throws IOException {\n int read = super.read(buffer, offset, length);\n if (read > 0) account(read);\n return read;\n }\n\n private void account(long bytes) throws IOException {\n consumed = Math.addExact(consumed, bytes);\n if (consumed > maximumBytes) {\n throw new IOException(\"inflated trace exceeds configured hard bound\");\n }\n }\n }\n}\n" }, { "path": "backend/historical-bootstrap-tool/src/main/java/hu/gombos/flightspotter/historicalbootstrap/RegionalHistoricalDailyMain.java", "encoding": "UTF-8", "sha256": "ecc944229c81b696f6e798e90a44932c13f856d5e557be9ccd28bd9885425564", "content": "package hu.gombos.flightspotter.historicalbootstrap;\n\nimport hu.gombos.flightspotter.common.geo.UberH3Service;\nimport hu.gombos.flightspotter.forecastmodel.CoverageDomain;\nimport hu.gombos.flightspotter.forecastmodel.CoverageDomainRegistry;\nimport hu.gombos.flightspotter.forecastmodel.CoverageRegionId;\nimport hu.gombos.flightspotter.forecastmodel.RegionalHistoricalTruthAuthority;\n\nimport javax.sql.DataSource;\nimport java.io.PrintWriter;\nimport java.nio.charset.StandardCharsets;\nimport java.nio.file.Files;\nimport java.nio.file.Path;\nimport java.sql.Connection;\nimport java.sql.DriverManager;\nimport java.sql.SQLException;\nimport java.sql.SQLFeatureNotSupportedException;\nimport java.time.Duration;\nimport java.time.Instant;\nimport java.time.LocalDate;\nimport java.util.Map;\nimport java.util.Objects;\nimport java.util.Properties;\nimport java.util.Set;\nimport java.util.UUID;\nimport java.util.logging.Logger;\n\n/**\n * Production-operable one-day regional historical evidence job.\n *\n *

Execution is intentionally outside the hot Spring serving processes. Immutable upstream\n * assets are digest-cached, raw archive scratch is deleted by Gate B after every attempt, the\n * compact H2 staging DB is restartable and can be deleted after PostgreSQL READY + immutable\n * regional truth seal.

\n */\npublic final class RegionalHistoricalDailyMain {\n private RegionalHistoricalDailyMain() { }\n\n public static void main(String[] rawArgs) throws Exception {\n Args args = Args.parse(rawArgs);\n BootstrapSourceManifest manifest = BootstrapSourceManifestIO.read(args.manifest());\n LocalDate day = args.sourceDate();\n requireOneDayScoringManifest(manifest, day);\n\n UberH3Service h3 = new UberH3Service();\n CoverageDomain domain = CoverageDomainRegistry.sourceControlled(h3)\n .require(args.regionId());\n BootstrapCoverageSpec coverage = BootstrapCoverageSpec.fromDomain(domain, h3);\n DataSource dataSource = new DriverManagerDataSource(args.jdbcUrl(), args.jdbcUser(), args.jdbcPassword());\n RegionalHistoricalTruthAuthority authority =\n RegionalHistoricalTruthAuthorityStore.authority(domain, args.maturityAfterDayEnd());\n RegionalHistoricalTruthAuthorityStore authorityStore =\n new RegionalHistoricalTruthAuthorityStore(dataSource);\n\n ResolvedSourceDay sourceDay = manifest.days().getFirst();\n validateSourceContract(sourceDay);\n\n System.out.println(\"regionId=\" + domain.regionId());\n System.out.println(\"coverageVersion=\" + domain.coverageVersion());\n System.out.println(\"coverageHash=\" + domain.evidenceCoverageHash());\n System.out.println(\"regionCanonicalSha256=\" + domain.canonicalSha256());\n System.out.println(\"truthAuthorityGeneration=\" + RegionalHistoricalTruthAuthority.GENERATION);\n System.out.println(\"truthAuthorityFingerprint=\" + authority.fingerprint());\n System.out.println(\"sourceDate=\" + day);\n\n if (sourceDay.preflightStatus() != SourcePreflightStatus.PREFLIGHT_OK) {\n String quality = gapStatus(sourceDay.preflightStatus()) ? \"GAP\" : \"DEGRADED\";\n String reason = sourceDay.preflightStatus().name();\n var seal = authorityStore.sealIncomplete(authority, manifest, day, quality, reason);\n System.out.println(\"terminalQuality=\" + quality);\n System.out.println(\"truthStatus=\" + seal.status());\n System.out.println(\"sealSha256=\" + seal.sealSha256());\n System.out.println(\"NO_PROVIDER_PAYLOAD_OPEN=true\");\n return;\n }\n\n GateBContentPolicy contentPolicy =\n new GateBContentPolicy(args.maximumTraceParseFailures(), args.minimumAcceptedPoints());\n GateBExecutionBudget budget = new GateBExecutionBudget(\n args.maximumTransferBytes(),\n args.scratchReserveBytes(),\n args.maximumAssetAttempts(),\n contentPolicy);\n UUID datasetId = deterministicRegionalDatasetId(manifest, day, coverage, contentPolicy);\n GateDPreparedDayMetricsStore preparedMetrics =\n new GateDPreparedDayMetricsStore(preparedMetricsDirectory(args.stateDatabase()));\n\n String existing = datasetState(dataSource, datasetId);\n if (\"READY\".equals(existing)) {\n var seal = authorityStore.sealQualified(authority, manifest, datasetId, day);\n System.out.println(\"datasetId=\" + datasetId);\n System.out.println(\"datasetState=READY\");\n System.out.println(\"idempotentReplay=true\");\n System.out.println(\"sealSha256=\" + seal.sealSha256());\n cleanupCompactState(args.stateDatabase());\n preparedMetrics.delete(day);\n return;\n }\n if (existing != null) {\n throw new IllegalStateException(\"deterministic regional dataset already exists in non-READY state: \"\n + datasetId + \" state=\" + existing);\n }\n\n GateBExecutionPlan plan = GateBExecutionPlan.create(\n manifest, day, datasetId, coverage, args.scratchRoot(), args.stateDatabase(), budget);\n HistoricalBootstrapGateBMain.requireRepresentativeRole(plan.role());\n\n GateBSourceDayRunner.PreparedDayMetrics prepared =\n recoverCommittedPreparedMetrics(plan, preparedMetrics);\n boolean resumedCommittedLocalDay = prepared != null;\n if (prepared == null) {\n try {\n GateBAssetSource upstream = new GitHubReleaseAssetSource(args.requestTimeout());\n GateBAssetSource cached = new DigestCachingGateBAssetSource(\n upstream, args.assetCache(), args.assetCacheMaximumBytes());\n new GateBSourceDayRunner(ArchiveSafetyLimits.defaults(), cached)\n .run(plan, coverage, preparedMetrics::write);\n prepared = recoverCommittedPreparedMetrics(plan, preparedMetrics);\n if (prepared == null) {\n throw new IllegalStateException(\n \"Gate B returned without a recoverable committed regional source day\");\n }\n } catch (GateBSourceDayRunner.ContentQualityFailure qualityFailure) {\n var seal = authorityStore.sealIncomplete(\n authority, manifest, day, \"DEGRADED\", qualityFailure.reasonCode());\n System.out.println(\"terminalQuality=DEGRADED\");\n System.out.println(\"reasonCode=\" + qualityFailure.reasonCode());\n System.out.println(\"truthStatus=\" + seal.status());\n System.out.println(\"sealSha256=\" + seal.sealSha256());\n cleanupCompactState(args.stateDatabase());\n preparedMetrics.delete(day);\n return;\n } catch (ImmutableHistoricalSourceException sourceFailure) {\n var seal = authorityStore.sealIncomplete(\n authority, manifest, day, \"DEGRADED\", sourceFailure.reasonCode());\n System.out.println(\"terminalQuality=DEGRADED\");\n System.out.println(\"reasonCode=\" + sourceFailure.reasonCode());\n System.out.println(\"truthStatus=\" + seal.status());\n System.out.println(\"sealSha256=\" + seal.sealSha256());\n cleanupCompactState(args.stateDatabase());\n preparedMetrics.delete(day);\n return;\n }\n }\n System.out.println(\"localCommittedDayResume=\" + resumedCommittedLocalDay);\n\n // One-day scoring datasets deliberately cut cross-midnight carry. The authority policy\n // makes UTC boundary hours ineligible, so this cannot create a false negative.\n try (LocalDayCommitStore state = LocalDayCommitStore.open(\n args.stateDatabase(), ArchiveSafetyLimits.defaults())) {\n LocalDayCommitStore.ExecutionState before = state.executionState(datasetId);\n if (!before.carry().isEmpty()) {\n state.apply(\n LocalDayCommitStore.ReplayMode.CARRY_ONLY_RECONSTRUCTION,\n plan.dayIdentity(),\n null,\n Map.of(),\n null,\n LocalDayCommitStore.FailureInjector.none());\n }\n if (!state.executionState(datasetId).carry().isEmpty()) {\n throw new IllegalStateException(\"one-day regional historical state still contains final carry\");\n }\n }\n\n Instant loadedAt = Instant.now();\n var metrics = new HistoricalBootstrapPostgresPersistence.DayLoadMetrics(\n day,\n prepared.parsedTraceCount(),\n prepared.traceParseFailureCount(),\n prepared.malformedPointCount(),\n prepared.acceptedPointCount(),\n prepared.rejectedPointCount(),\n prepared.finalizedSegmentCount(),\n null,\n loadedAt);\n var request = new HistoricalBootstrapPostgresStreamingPersistence.StreamingLoadRequest(\n args.stateDatabase(),\n datasetId,\n manifest,\n prepared.semanticFingerprint(),\n domain.coverageVersion(),\n domain.evidenceCoverageHash(),\n GateBSemanticFingerprint.DAY_DELTA_VERSION,\n Map.of(day, metrics),\n loadedAt,\n loadedAt.plus(Duration.ofDays(1)),\n loadedAt,\n loadedAt.plus(args.scoringRetention()));\n var loaded = HistoricalBootstrapPostgresStreamingPersistence.load(dataSource, request);\n if (loaded.dayCount() != 1 || !loaded.datasetId().equals(datasetId)) {\n throw new IllegalStateException(\"regional historical PostgreSQL load did not produce one deterministic day\");\n }\n\n var seal = authorityStore.sealQualified(authority, manifest, datasetId, day);\n System.out.println(\"datasetId=\" + datasetId);\n System.out.println(\"datasetState=READY\");\n System.out.println(\"terminalQuality=QUALIFIED\");\n System.out.println(\"patternCount=\" + loaded.patternCount());\n System.out.println(\"hourBinCount=\" + loaded.hourBinCount());\n System.out.println(\"corridorCount=\" + loaded.corridorCount());\n System.out.println(\"sealSha256=\" + seal.sealSha256());\n System.out.println(\"rawScratchRetentionRequired=false\");\n System.out.println(\"REGIONAL_HISTORICAL_DAY=QUALIFIED\");\n\n if (!args.retainCompactState()) cleanupCompactState(args.stateDatabase());\n preparedMetrics.delete(day);\n }\n\n static GateBSourceDayRunner.PreparedDayMetrics recoverCommittedPreparedMetrics(\n GateBExecutionPlan plan,\n GateDPreparedDayMetricsStore preparedMetrics\n ) throws Exception {\n Objects.requireNonNull(plan, \"plan\");\n Objects.requireNonNull(preparedMetrics, \"preparedMetrics\");\n try (LocalDayCommitStore state = LocalDayCommitStore.open(\n plan.stateDatabase(), ArchiveSafetyLimits.defaults())) {\n LocalDayCommitStore.CommittedDay committed = state.committedDay(plan.dayIdentity());\n if (committed == null) return null;\n LocalDayCommitStore.ExecutionState execution = state.executionState(plan.datasetId());\n if (execution.committedDayCount() != 1) {\n throw new IllegalStateException(\n \"one-day regional recovery requires exactly one committed local day\");\n }\n GateBSourceDayRunner.PreparedDayMetrics metrics = preparedMetrics.requireMatches(\n plan.sourceDate(),\n plan.datasetId(),\n plan.manifestSha256(),\n plan.dayManifestSha256(),\n plan.semanticFingerprint(),\n plan.coverageVersion(),\n plan.coverageHash());\n if (committed.quality() != LocalDayCommitStore.EvidenceQualityStatus.QUALIFIED\n || metrics.quality() != LocalDayCommitStore.EvidenceQualityStatus.QUALIFIED\n || metrics.reasonCode() != null\n || !committed.deltaChecksum().equals(metrics.dayDeltaChecksum())) {\n throw new IllegalStateException(\n \"committed regional day and durable prepared metrics disagree\");\n }\n return metrics;\n }\n }\n\n private static Path preparedMetricsDirectory(Path stateDatabase) {\n Path normalized = stateDatabase.toAbsolutePath().normalize();\n Path parent = normalized.getParent();\n if (parent == null) {\n throw new IllegalArgumentException(\"regional state database requires a parent directory\");\n }\n return parent.resolve(\"prepared\");\n }\n\n private static void requireOneDayScoringManifest(BootstrapSourceManifest manifest, LocalDate day) {\n if (manifest.window().purpose() != DatasetWindow.DatasetPurpose.BOOTSTRAP_SCORING\n || manifest.window().calendarDayCount() != 1\n || !manifest.window().fromDate().equals(day)\n || !manifest.window().toDate().equals(day)\n || manifest.days().size() != 1\n || manifest.days().getFirst().role() != DatasetWindow.DateRole.SCORING) {\n throw new IllegalArgumentException(\n \"regional daily authority requires one immutable BOOTSTRAP_SCORING manifest day\");\n }\n if (!\"ADSB_LOL\".equals(manifest.provider())) {\n throw new IllegalArgumentException(\"regional daily source provider must be ADSB_LOL\");\n }\n }\n\n private static void validateSourceContract(ResolvedSourceDay sourceDay) {\n for (SourceRelease release : sourceDay.candidateReleases()) {\n if (!RegionalHistoricalTruthAuthorityStore.repositoryAllowed(\n RegionalHistoricalTruthAuthority.ADSB_LOL_REPOSITORY_POLICY, release.repository())) {\n throw new IllegalArgumentException(\"manifest contains repository outside regional authority policy\");\n }\n }\n }\n\n private static boolean gapStatus(SourcePreflightStatus status) {\n return status == SourcePreflightStatus.NO_RELEASE\n || status == SourcePreflightStatus.NO_ARCHIVE_ASSETS\n || status == SourcePreflightStatus.MLAT_ONLY_RELEASE\n || status == SourcePreflightStatus.MANUAL_EXCLUSION;\n }\n\n static UUID deterministicRegionalDatasetId(\n BootstrapSourceManifest manifest,\n LocalDate sourceDate,\n BootstrapCoverageSpec coverage,\n GateBContentPolicy contentPolicy\n ) {\n UUID evidenceIdentity = HistoricalBootstrapGateBMain.deterministicDatasetId(\n manifest, sourceDate, coverage, contentPolicy);\n String canonical = \"regional-historical-dataset-v2|\"\n + HistoricalBootstrapPostgresStreamingPersistence.IMPORTER_VERSION\n + \"|\" + evidenceIdentity;\n return UUID.nameUUIDFromBytes(canonical.getBytes(StandardCharsets.UTF_8));\n }\n\n private static String datasetState(DataSource dataSource, UUID datasetId) throws SQLException {\n try (Connection connection = dataSource.getConnection();\n var statement = connection.prepareStatement(\n \"select state from historical_bootstrap_datasets where dataset_id=?\")) {\n statement.setObject(1, datasetId);\n try (var rs = statement.executeQuery()) {\n if (!rs.next()) return null;\n String state = rs.getString(1);\n if (rs.next()) throw new IllegalStateException(\"duplicate historical dataset identity\");\n return state;\n }\n }\n }\n\n private static void cleanupCompactState(Path stateDatabase) throws Exception {\n Path normalized = stateDatabase.toAbsolutePath().normalize();\n Files.deleteIfExists(Path.of(normalized + \".mv.db\"));\n Files.deleteIfExists(Path.of(normalized + \".trace.db\"));\n Files.deleteIfExists(normalized);\n }\n\n record Args(\n CoverageRegionId regionId,\n Path manifest,\n LocalDate sourceDate,\n Path scratchRoot,\n Path stateDatabase,\n Path assetCache,\n long assetCacheMaximumBytes,\n long maximumTransferBytes,\n long scratchReserveBytes,\n int maximumAssetAttempts,\n int maximumTraceParseFailures,\n long minimumAcceptedPoints,\n Duration requestTimeout,\n Duration maturityAfterDayEnd,\n Duration scoringRetention,\n boolean retainCompactState,\n String jdbcUrl,\n String jdbcUser,\n String jdbcPassword\n ) {\n private static final Set KNOWN = Set.of(\n \"--region-id\", \"--manifest\", \"--source-date\", \"--scratch\", \"--state-db\", \"--asset-cache\",\n \"--asset-cache-max-bytes\", \"--max-transfer-bytes\", \"--scratch-reserve-bytes\",\n \"--max-asset-attempts\", \"--max-trace-parse-failures\", \"--min-accepted-points\",\n \"--request-timeout-minutes\", \"--maturity-hours\", \"--scoring-retention-days\",\n \"--retain-state\", \"--jdbc-url\", \"--jdbc-user\", \"--jdbc-password\");\n\n Args {\n Objects.requireNonNull(regionId, \"regionId\");\n Objects.requireNonNull(manifest, \"manifest\");\n Objects.requireNonNull(sourceDate, \"sourceDate\");\n Objects.requireNonNull(scratchRoot, \"scratchRoot\");\n Objects.requireNonNull(stateDatabase, \"stateDatabase\");\n Objects.requireNonNull(assetCache, \"assetCache\");\n Objects.requireNonNull(requestTimeout, \"requestTimeout\");\n Objects.requireNonNull(maturityAfterDayEnd, \"maturityAfterDayEnd\");\n Objects.requireNonNull(scoringRetention, \"scoringRetention\");\n if (assetCacheMaximumBytes < 1L || maximumTransferBytes < 1L || scratchReserveBytes < 0L\n || maximumAssetAttempts < 1 || maximumAssetAttempts > 3\n || maximumTraceParseFailures < 0 || minimumAcceptedPoints < 1L\n || requestTimeout.isZero() || requestTimeout.isNegative()\n || maturityAfterDayEnd.isNegative() || maturityAfterDayEnd.compareTo(Duration.ofDays(14)) > 0\n || scoringRetention.compareTo(Duration.ofDays(90)) < 0\n || scoringRetention.compareTo(Duration.ofDays(400)) > 0\n || jdbcUrl == null || !jdbcUrl.startsWith(\"jdbc:postgresql:\")\n || jdbcUser == null || jdbcUser.isBlank() || jdbcPassword == null) {\n throw new IllegalArgumentException(\"invalid bounded regional historical daily job arguments\");\n }\n }\n\n static Args parse(String[] raw) {\n java.util.LinkedHashMap values = new java.util.LinkedHashMap<>();\n for (int i = 0; i < raw.length; i++) {\n String key = raw[i];\n if (!KNOWN.contains(key) || i + 1 >= raw.length) {\n throw new IllegalArgumentException(\"invalid/missing regional daily option near \" + key);\n }\n if (values.put(key, raw[++i]) != null) {\n throw new IllegalArgumentException(\"duplicate regional daily option \" + key);\n }\n }\n return new Args(\n CoverageRegionId.of(required(values, \"--region-id\")),\n Path.of(required(values, \"--manifest\")),\n LocalDate.parse(required(values, \"--source-date\")),\n Path.of(required(values, \"--scratch\")),\n Path.of(required(values, \"--state-db\")),\n Path.of(required(values, \"--asset-cache\")),\n Long.parseLong(required(values, \"--asset-cache-max-bytes\")),\n Long.parseLong(required(values, \"--max-transfer-bytes\")),\n Long.parseLong(required(values, \"--scratch-reserve-bytes\")),\n Integer.parseInt(values.getOrDefault(\"--max-asset-attempts\", \"2\")),\n Integer.parseInt(values.getOrDefault(\"--max-trace-parse-failures\", \"0\")),\n Long.parseLong(values.getOrDefault(\"--min-accepted-points\", \"1\")),\n Duration.ofMinutes(Long.parseLong(values.getOrDefault(\"--request-timeout-minutes\", \"120\"))),\n Duration.ofHours(Long.parseLong(values.getOrDefault(\"--maturity-hours\", \"6\"))),\n Duration.ofDays(Long.parseLong(values.getOrDefault(\"--scoring-retention-days\", \"90\"))),\n Boolean.parseBoolean(values.getOrDefault(\"--retain-state\", \"false\")),\n required(values, \"--jdbc-url\"),\n required(values, \"--jdbc-user\"),\n required(values, \"--jdbc-password\"));\n }\n\n private static String required(Map values, String key) {\n String value = values.get(key);\n if (value == null || value.isBlank()) throw new IllegalArgumentException(\"missing required \" + key);\n return value;\n }\n }\n\n private static final class DriverManagerDataSource implements DataSource {\n private final String url;\n private final Properties properties;\n\n private DriverManagerDataSource(String url, String user, String password) {\n this.url = url;\n this.properties = new Properties();\n this.properties.setProperty(\"user\", user);\n this.properties.setProperty(\"password\", password);\n // PostgreSQL rewrites compatible JDBC batches into multi-values INSERT statements.\n // This DataSource is used only by the offline historical loader.\n this.properties.setProperty(\"reWriteBatchedInserts\", \"true\");\n }\n\n @Override public Connection getConnection() throws SQLException {\n return DriverManager.getConnection(url, properties);\n }\n @Override public Connection getConnection(String username, String password) throws SQLException {\n Properties override = new Properties();\n override.putAll(properties);\n override.setProperty(\"user\", username);\n override.setProperty(\"password\", password);\n return DriverManager.getConnection(url, override);\n }\n @Override public PrintWriter getLogWriter() { return DriverManager.getLogWriter(); }\n @Override public void setLogWriter(PrintWriter out) { DriverManager.setLogWriter(out); }\n @Override public void setLoginTimeout(int seconds) { DriverManager.setLoginTimeout(seconds); }\n @Override public int getLoginTimeout() { return DriverManager.getLoginTimeout(); }\n @Override public Logger getParentLogger() throws SQLFeatureNotSupportedException {\n throw new SQLFeatureNotSupportedException();\n }\n @Override public T unwrap(Class iface) throws SQLException {\n if (iface.isInstance(this)) return iface.cast(this);\n throw new SQLException(\"not a wrapper for \" + iface.getName());\n }\n @Override public boolean isWrapperFor(Class iface) { return iface.isInstance(this); }\n }\n}\n" }, { "path": "backend/historical-bootstrap-tool/src/main/java/hu/gombos/flightspotter/historicalbootstrap/RegionalHistoricalFeedMain.java", "encoding": "UTF-8", "sha256": "49500cf27850d72fab103a122b0bf7099c16b790d68d9182f3387772cf723c61", "content": "package hu.gombos.flightspotter.historicalbootstrap;\n\nimport hu.gombos.flightspotter.common.geo.UberH3Service;\nimport hu.gombos.flightspotter.forecastmodel.CoverageDomain;\nimport hu.gombos.flightspotter.forecastmodel.CoverageDomainRegistry;\nimport hu.gombos.flightspotter.forecastmodel.CoverageRegionId;\nimport hu.gombos.flightspotter.forecastmodel.RegionalHistoricalTruthAuthority;\nimport tools.jackson.databind.ObjectMapper;\nimport tools.jackson.databind.json.JsonMapper;\n\nimport java.nio.file.Files;\nimport java.nio.file.Path;\nimport java.nio.file.StandardCopyOption;\nimport java.sql.Connection;\nimport java.sql.DriverManager;\nimport java.time.Clock;\nimport java.time.Duration;\nimport java.time.Instant;\nimport java.time.LocalDate;\nimport java.time.ZoneOffset;\nimport java.util.ArrayList;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Objects;\nimport java.util.Properties;\n\n/**\n * One bounded production feed cycle for regional historical T-24 evidence.\n *\n *

This is deliberately a separate worker boundary from every serving JVM. It performs metadata discovery once\n * for a bounded backlog, processes at most a configured number of unsealed source days, and relies on the immutable\n * regional day seal for replay safety. A PostgreSQL advisory lock prevents concurrent workers across hosts.

\n */\npublic final class RegionalHistoricalFeedMain {\n private static final String LOCK_NAME = \"flighttrove-regional-historical-feed-v1\";\n\n private RegionalHistoricalFeedMain() { }\n\n public static void main(String[] args) throws Exception {\n if (args.length != 0) throw new IllegalArgumentException(\"RegionalHistoricalFeedMain uses environment configuration only\");\n run(Config.fromEnvironment(System.getenv()), Clock.systemUTC());\n }\n\n static FeedResult run(Config config, Clock clock) throws Exception {\n Objects.requireNonNull(config, \"config\");\n Objects.requireNonNull(clock, \"clock\");\n Instant now = clock.instant();\n LocalDate today = now.atZone(ZoneOffset.UTC).toLocalDate();\n LocalDate newest = today.minusDays(config.sourceLagDays());\n LocalDate oldest = newest.minusDays(config.backlogDays() - 1L);\n\n UberH3Service h3 = new UberH3Service();\n CoverageDomain domain = CoverageDomainRegistry.sourceControlled(h3).require(config.regionId());\n RegionalHistoricalTruthAuthority authority =\n RegionalHistoricalTruthAuthorityStore.authority(domain, config.maturityAfterDayEnd());\n\n Files.createDirectories(config.manifestDirectory());\n Files.createDirectories(config.scratchRoot());\n Files.createDirectories(config.stateRoot());\n Files.createDirectories(config.assetCache());\n\n try (Connection lockConnection = connection(config)) {\n if (!tryLock(lockConnection)) {\n System.out.println(\"REGIONAL_HISTORICAL_FEED=SKIPPED_LOCK_HELD\");\n return new FeedResult(0, 0, 0, true);\n }\n try {\n return runLocked(config, now, oldest, newest, authority, domain);\n } finally {\n unlock(lockConnection);\n }\n }\n }\n\n private static FeedResult runLocked(\n Config config,\n Instant now,\n LocalDate oldest,\n LocalDate newest,\n RegionalHistoricalTruthAuthority authority,\n CoverageDomain domain\n ) throws Exception {\n try (Connection retentionConnection = connection(config)) {\n var retention = HistoricalBootstrapRetentionMaintenance.purgeExpired(\n retentionConnection,\n now,\n config.retentionPurgeBatchSize());\n System.out.println(\"REGIONAL_HISTORICAL_RETENTION\"\n + \" selected=\" + retention.selectedDatasets()\n + \" purged=\" + retention.purgedDatasets()\n + \" deletedRows=\" + retention.deletedRows());\n }\n\n long filesystemUsableBytes = Files.getFileStore(config.stateRoot()).getUsableSpace();\n if (filesystemUsableBytes < config.minimumFilesystemHeadroomBytes()) {\n System.out.println(\"REGIONAL_HISTORICAL_FEED=DEFERRED_DISK_HEADROOM\"\n + \" usableBytes=\" + filesystemUsableBytes\n + \" requiredBytes=\" + config.minimumFilesystemHeadroomBytes());\n return new FeedResult(0, 0, 0, false);\n }\n\n DatasetWindow discoveryWindow = DatasetWindow.fixedScoring(oldest, newest);\n String token = discoveryToken(config);\n List releases = new GitHubReleaseDiscoveryClient(token).discover(discoveryWindow);\n Map> releasesByDay = new HashMap<>();\n for (SourceRelease release : releases) {\n releasesByDay.computeIfAbsent(release.sourceDate(), ignored -> new ArrayList<>()).add(release);\n }\n\n int examined = 0;\n int processed = 0;\n int deferred = 0;\n int retryableFailures = 0;\n CanonicalSourceResolver resolver = new CanonicalSourceResolver();\n PreflightPolicy preflight = new PreflightPolicy(config.minimumArchiveBytes(), config.maximumArchiveBytes());\n\n for (LocalDate day = oldest;\n !day.isAfter(newest) && examined < config.maximumDaysPerRun();\n day = day.plusDays(1)) {\n if (sealExists(config, authority.fingerprint(), day)) {\n cleanupSealedLocalArtifacts(config, day);\n continue;\n }\n // A D-1 source archive may be published before its authoritative 6-hour\n // maturity boundary. Never process or seal it until source-day maturity.\n Instant matureAt = sourceDayMatureAt(day, config.maturityAfterDayEnd());\n if (now.isBefore(matureAt)) {\n deferred++;\n System.out.println(\"sourceDate=\" + day + \" terminalQuality=DEFERRED\"\n + \" reason=AWAIT_SOURCE_MATURITY matureAt=\" + matureAt);\n continue;\n }\n examined++;\n\n ManifestSelection selectedManifest = selectManifest(\n config,\n day,\n releasesByDay.getOrDefault(day, List.of()),\n resolver,\n preflight);\n ResolvedSourceDay sourceDay = selectedManifest.manifest().days().getFirst();\n Path manifestPath = selectedManifest.path();\n if (selectedManifest.recoveryPinned()) {\n System.out.println(\"sourceDate=\" + day + \" recoveryManifestPinned=true\"\n + \" manifestSha256=\" + selectedManifest.manifest().manifestSha256());\n }\n\n if (sourceDay.preflightStatus() != SourcePreflightStatus.PREFLIGHT_OK\n && now.isBefore(finalizationAt(day, config.missingSourceFinalizationDelay()))) {\n deferred++;\n System.out.println(\"sourceDate=\" + day\n + \" terminalQuality=DEFERRED\"\n + \" sourcePreflightStatus=\" + sourceDay.preflightStatus()\n + \" finalizationAt=\" + finalizationAt(day, config.missingSourceFinalizationDelay()));\n continue;\n }\n\n try {\n if (cleanupAbandonedRawScratch(config, day)) {\n System.out.println(\"sourceDate=\" + day + \" abandonedRawScratchCleaned=true\");\n }\n RegionalHistoricalDailyMain.main(dailyArgs(config, domain, day, manifestPath));\n processed++;\n } catch (HistoricalSourceTransportException retryableSourceFailure) {\n // Source transport failure is not evidence about the source day and must not be sealed.\n // The one-day datasets are independent, so this day remains retryable while the bounded feed\n // continues to later days instead of creating a permanent head-of-line block.\n retryableFailures++;\n System.out.println(\"sourceDate=\" + day\n + \" terminalQuality=RETRYABLE\"\n + \" failureType=\" + retryableSourceFailure.getClass().getSimpleName());\n }\n }\n\n System.out.println(\"REGIONAL_HISTORICAL_FEED=COMPLETE\");\n System.out.println(\"regionId=\" + domain.regionId());\n System.out.println(\"authorityFingerprint=\" + authority.fingerprint());\n System.out.println(\"range=\" + oldest + \"..\" + newest);\n System.out.println(\"examined=\" + examined + \" processed=\" + processed + \" deferred=\" + deferred\n + \" retryableFailures=\" + retryableFailures);\n return new FeedResult(examined, processed, deferred, false);\n }\n\n private static String discoveryToken(Config config) throws Exception {\n if (config.githubTokenFile() != null) {\n Path path = config.githubTokenFile().toAbsolutePath().normalize();\n if (!Files.isRegularFile(path) || !Files.isReadable(path)) {\n throw new IllegalStateException(\"regional historical GitHub token file is unavailable\");\n }\n String token = Files.readString(path).trim();\n if (token.isBlank()) throw new IllegalStateException(\"regional historical GitHub token file is empty\");\n return token;\n }\n String token = System.getenv(config.githubTokenEnvironmentVariable());\n return token == null || token.isBlank() ? null : token.trim();\n }\n\n static Instant finalizationAt(LocalDate day, Duration delay) {\n return day.plusDays(1).atStartOfDay(ZoneOffset.UTC).toInstant().plus(delay);\n }\n\n /**\n * The upstream full-production archive is normally published around 03:25 UTC\n * on D+1, but regional historical truth is not mature until D+1 06:00 UTC.\n * A D-1 discovery horizon is safe only with this processing admission guard.\n */\n static Instant sourceDayMatureAt(LocalDate sourceDay, Duration maturityAfterDayEnd) {\n return finalizationAt(sourceDay, maturityAfterDayEnd);\n }\n\n static ManifestSelection selectManifest(\n Config config,\n LocalDate day,\n List releases,\n CanonicalSourceResolver resolver,\n PreflightPolicy preflight\n ) throws Exception {\n Objects.requireNonNull(config, \"config\");\n Objects.requireNonNull(day, \"day\");\n Objects.requireNonNull(releases, \"releases\");\n Objects.requireNonNull(resolver, \"resolver\");\n Objects.requireNonNull(preflight, \"preflight\");\n\n Path manifestPath = config.manifestDirectory().resolve(day + \".json\").toAbsolutePath().normalize();\n Path dayState = config.stateRoot().resolve(day.toString()).toAbsolutePath().normalize();\n if (Files.exists(dayState)) {\n if (!Files.isRegularFile(manifestPath)) {\n throw new IllegalStateException(\n \"regional historical recovery state exists without its immutable manifest: \" + day);\n }\n BootstrapSourceManifest persisted = BootstrapSourceManifestIO.read(manifestPath);\n if (persisted.days().size() != 1\n || !persisted.window().fromDate().equals(day)\n || !persisted.window().toDate().equals(day)\n || !persisted.days().getFirst().sourceDate().equals(day)) {\n throw new IllegalStateException(\n \"regional historical recovery manifest does not match source day: \" + day);\n }\n return new ManifestSelection(persisted, manifestPath, true);\n }\n\n DatasetWindow oneDayWindow = DatasetWindow.fixedScoring(day, day);\n ResolvedSourceDay sourceDay = resolver.resolve(\n oneDayWindow,\n day,\n releases,\n java.util.Optional.empty(),\n preflight);\n BootstrapSourceManifest manifest = BootstrapSourceManifest.create(\n oneDayWindow, preflight, List.of(sourceDay));\n return new ManifestSelection(\n manifest,\n persistManifest(config.manifestDirectory(), manifest, day),\n false);\n }\n\n private static Path persistManifest(\n Path directory, BootstrapSourceManifest manifest, LocalDate day\n ) throws Exception {\n ObjectMapper mapper = JsonMapper.builder().build();\n Path target = directory.resolve(day + \".json\").toAbsolutePath().normalize();\n Path temporary = directory.resolve(\".\" + day + \".json.tmp\").toAbsolutePath().normalize();\n mapper.writerWithDefaultPrettyPrinter().writeValue(temporary.toFile(), manifest.toJsonModel());\n try {\n Files.move(temporary, target,\n StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);\n } catch (java.nio.file.AtomicMoveNotSupportedException unsupported) {\n Files.move(temporary, target, StandardCopyOption.REPLACE_EXISTING);\n }\n return target;\n }\n\n static boolean cleanupAbandonedRawScratch(Config config, LocalDate day) throws Exception {\n Path dayScratch = config.scratchRoot().resolve(day.toString());\n boolean existed = Files.exists(dayScratch);\n deleteTree(dayScratch, config.scratchRoot());\n return existed;\n }\n\n private static void cleanupSealedLocalArtifacts(Config config, LocalDate day) throws Exception {\n deleteTree(config.scratchRoot().resolve(day.toString()), config.scratchRoot());\n deleteTree(config.stateRoot().resolve(day.toString()), config.stateRoot());\n }\n\n private static void deleteTree(Path target, Path allowedRoot) throws Exception {\n Path root = allowedRoot.toAbsolutePath().normalize();\n Path normalized = target.toAbsolutePath().normalize();\n if (!normalized.startsWith(root) || normalized.equals(root) || !Files.exists(normalized)) return;\n try (var stream = Files.walk(normalized)) {\n for (Path path : stream.sorted(java.util.Comparator.reverseOrder()).toList()) {\n Files.deleteIfExists(path);\n }\n }\n }\n\n private static boolean sealExists(Config config, String authorityFingerprint, LocalDate day) throws Exception {\n try (Connection connection = connection(config);\n var statement = connection.prepareStatement(\"\"\"\n select count(*) from regional_historical_truth_day_seals\n where authority_fingerprint=? and target_day=?\n \"\"\")) {\n statement.setString(1, authorityFingerprint);\n statement.setDate(2, java.sql.Date.valueOf(day));\n try (var result = statement.executeQuery()) {\n result.next();\n return result.getLong(1) == 1L;\n }\n }\n }\n\n private static boolean tryLock(Connection connection) throws Exception {\n try (var statement = connection.prepareStatement(\"select pg_try_advisory_lock(hashtext(?))\")) {\n statement.setString(1, LOCK_NAME);\n try (var result = statement.executeQuery()) {\n result.next();\n return result.getBoolean(1);\n }\n }\n }\n\n private static void unlock(Connection connection) {\n try (var statement = connection.prepareStatement(\"select pg_advisory_unlock(hashtext(?))\")) {\n statement.setString(1, LOCK_NAME);\n statement.execute();\n } catch (Exception ignored) {\n // Connection close also releases session-scoped advisory locks.\n }\n }\n\n private static Connection connection(Config config) throws Exception {\n Properties properties = new Properties();\n properties.setProperty(\"user\", config.jdbcUser());\n properties.setProperty(\"password\", config.jdbcPassword());\n return DriverManager.getConnection(config.jdbcUrl(), properties);\n }\n\n private static String[] dailyArgs(Config config, CoverageDomain domain, LocalDate day, Path manifest) {\n Path dayScratch = config.scratchRoot().resolve(day.toString());\n Path dayState = config.stateRoot().resolve(day.toString()).resolve(\"regional\");\n return new String[] {\n \"--region-id\", domain.regionId().value(),\n \"--manifest\", manifest.toString(),\n \"--source-date\", day.toString(),\n \"--scratch\", dayScratch.toString(),\n \"--state-db\", dayState.toString(),\n \"--asset-cache\", config.assetCache().toString(),\n \"--asset-cache-max-bytes\", Long.toString(config.assetCacheMaximumBytes()),\n \"--max-transfer-bytes\", Long.toString(config.maximumTransferBytes()),\n \"--scratch-reserve-bytes\", Long.toString(config.scratchReserveBytes()),\n \"--max-asset-attempts\", Integer.toString(config.maximumAssetAttempts()),\n \"--max-trace-parse-failures\", Integer.toString(config.maximumTraceParseFailures()),\n \"--min-accepted-points\", Long.toString(config.minimumAcceptedPoints()),\n \"--request-timeout-minutes\", Long.toString(config.requestTimeout().toMinutes()),\n \"--maturity-hours\", Long.toString(config.maturityAfterDayEnd().toHours()),\n \"--scoring-retention-days\", Long.toString(config.scoringRetention().toDays()),\n \"--retain-state\", \"false\",\n \"--jdbc-url\", config.jdbcUrl(),\n \"--jdbc-user\", config.jdbcUser(),\n \"--jdbc-password\", config.jdbcPassword()\n };\n }\n\n record FeedResult(int examined, int processed, int deferred, boolean lockHeld) { }\n\n record ManifestSelection(\n BootstrapSourceManifest manifest,\n Path path,\n boolean recoveryPinned\n ) {\n ManifestSelection {\n Objects.requireNonNull(manifest, \"manifest\");\n Objects.requireNonNull(path, \"path\");\n }\n }\n\n record Config(\n CoverageRegionId regionId,\n int sourceLagDays,\n int backlogDays,\n int maximumDaysPerRun,\n Duration missingSourceFinalizationDelay,\n Path manifestDirectory,\n Path scratchRoot,\n Path stateRoot,\n Path assetCache,\n long assetCacheMaximumBytes,\n long maximumTransferBytes,\n long scratchReserveBytes,\n long minimumFilesystemHeadroomBytes,\n int maximumAssetAttempts,\n int maximumTraceParseFailures,\n long minimumAcceptedPoints,\n Long minimumArchiveBytes,\n Long maximumArchiveBytes,\n Duration requestTimeout,\n Duration maturityAfterDayEnd,\n Duration scoringRetention,\n int retentionPurgeBatchSize,\n String githubTokenEnvironmentVariable,\n Path githubTokenFile,\n String jdbcUrl,\n String jdbcUser,\n String jdbcPassword\n ) {\n Config {\n Objects.requireNonNull(regionId, \"regionId\");\n Objects.requireNonNull(missingSourceFinalizationDelay, \"missingSourceFinalizationDelay\");\n Objects.requireNonNull(manifestDirectory, \"manifestDirectory\");\n Objects.requireNonNull(scratchRoot, \"scratchRoot\");\n Objects.requireNonNull(stateRoot, \"stateRoot\");\n Objects.requireNonNull(assetCache, \"assetCache\");\n Objects.requireNonNull(requestTimeout, \"requestTimeout\");\n Objects.requireNonNull(maturityAfterDayEnd, \"maturityAfterDayEnd\");\n Objects.requireNonNull(scoringRetention, \"scoringRetention\");\n if (sourceLagDays < 1 || sourceLagDays > 14\n || backlogDays < 1 || backlogDays > 120\n || maximumDaysPerRun < 1 || maximumDaysPerRun > 7\n || missingSourceFinalizationDelay.compareTo(Duration.ofHours(24)) < 0\n || missingSourceFinalizationDelay.compareTo(Duration.ofDays(14)) > 0\n || assetCacheMaximumBytes < 1L || maximumTransferBytes < 1L\n || scratchReserveBytes < 0L\n || minimumFilesystemHeadroomBytes < 32_000_000_000L\n || minimumFilesystemHeadroomBytes > 1_000_000_000_000L\n || maximumAssetAttempts < 1 || maximumAssetAttempts > 3\n || maximumTraceParseFailures < 0 || minimumAcceptedPoints < 1L\n || requestTimeout.isZero() || requestTimeout.isNegative()\n || maturityAfterDayEnd.isNegative() || maturityAfterDayEnd.compareTo(Duration.ofDays(14)) > 0\n || scoringRetention.compareTo(Duration.ofDays(90)) < 0\n || scoringRetention.compareTo(Duration.ofDays(400)) > 0\n || retentionPurgeBatchSize < 1 || retentionPurgeBatchSize > 8\n || githubTokenEnvironmentVariable == null || githubTokenEnvironmentVariable.isBlank()\n || jdbcUrl == null || !jdbcUrl.startsWith(\"jdbc:postgresql:\")\n || jdbcUser == null || jdbcUser.isBlank() || jdbcPassword == null || jdbcPassword.isBlank()) {\n throw new IllegalArgumentException(\"invalid bounded regional historical feed configuration\");\n }\n if (maximumArchiveBytes != null && maximumArchiveBytes > maximumTransferBytes) {\n throw new IllegalArgumentException(\"maximumArchiveBytes cannot exceed maximumTransferBytes\");\n }\n }\n\n static Config fromEnvironment(Map environment) {\n Objects.requireNonNull(environment, \"environment\");\n Path root = Path.of(required(environment, \"REGIONAL_HISTORICAL_WORK_ROOT\"));\n long maxTransfer = longValue(environment, \"REGIONAL_HISTORICAL_MAX_TRANSFER_BYTES\", 12_000_000_000L);\n return new Config(\n CoverageRegionId.of(environment.getOrDefault(\"REGIONAL_HISTORICAL_REGION_ID\", \"EUROPE_V1\")),\n intValue(environment, \"REGIONAL_HISTORICAL_SOURCE_LAG_DAYS\", 2),\n intValue(environment, \"REGIONAL_HISTORICAL_BACKLOG_DAYS\", 70),\n intValue(environment, \"REGIONAL_HISTORICAL_MAX_DAYS_PER_RUN\", 2),\n Duration.ofHours(longValue(environment, \"REGIONAL_HISTORICAL_FINALIZE_MISSING_AFTER_HOURS\", 72L)),\n root.resolve(\"manifests\"),\n root.resolve(\"scratch\"),\n root.resolve(\"state\"),\n Path.of(required(environment, \"REGIONAL_HISTORICAL_ASSET_CACHE\")),\n longValue(environment, \"REGIONAL_HISTORICAL_ASSET_CACHE_MAX_BYTES\", 12_000_000_000L),\n maxTransfer,\n longValue(environment, \"REGIONAL_HISTORICAL_SCRATCH_RESERVE_BYTES\", 8_000_000_000L),\n longValue(environment, \"REGIONAL_HISTORICAL_MIN_FILESYSTEM_HEADROOM_BYTES\", 32_000_000_000L),\n intValue(environment, \"REGIONAL_HISTORICAL_MAX_ASSET_ATTEMPTS\", 2),\n intValue(environment, \"REGIONAL_HISTORICAL_MAX_TRACE_PARSE_FAILURES\", 0),\n longValue(environment, \"REGIONAL_HISTORICAL_MIN_ACCEPTED_POINTS\", 1L),\n nullableLong(environment.get(\"REGIONAL_HISTORICAL_MIN_SOURCE_BYTES\")),\n nullableLong(environment.getOrDefault(\n \"REGIONAL_HISTORICAL_MAX_SOURCE_BYTES\", Long.toString(maxTransfer))),\n Duration.ofMinutes(longValue(environment, \"REGIONAL_HISTORICAL_REQUEST_TIMEOUT_MINUTES\", 120L)),\n Duration.ofHours(longValue(environment, \"REGIONAL_HISTORICAL_MATURITY_HOURS\", 6L)),\n Duration.ofDays(longValue(environment, \"REGIONAL_HISTORICAL_SCORING_RETENTION_DAYS\", 90L)),\n intValue(environment, \"REGIONAL_HISTORICAL_RETENTION_PURGE_BATCH_SIZE\", 2),\n environment.getOrDefault(\"REGIONAL_HISTORICAL_GITHUB_TOKEN_ENV\", \"GITHUB_TOKEN\"),\n optionalPath(environment.get(\"REGIONAL_HISTORICAL_GITHUB_TOKEN_FILE\")),\n required(environment, \"REGIONAL_HISTORICAL_JDBC_URL\"),\n required(environment, \"REGIONAL_HISTORICAL_JDBC_USER\"),\n required(environment, \"REGIONAL_HISTORICAL_JDBC_PASSWORD\"));\n }\n\n private static String required(Map env, String key) {\n String value = env.get(key);\n if (value == null || value.isBlank()) throw new IllegalArgumentException(\"missing environment \" + key);\n return value.trim();\n }\n\n private static int intValue(Map env, String key, int fallback) {\n return Integer.parseInt(env.getOrDefault(key, Integer.toString(fallback)));\n }\n\n private static long longValue(Map env, String key, long fallback) {\n return Long.parseLong(env.getOrDefault(key, Long.toString(fallback)));\n }\n\n private static Path optionalPath(String value) {\n return value == null || value.isBlank() ? null : Path.of(value.trim());\n }\n\n private static Long nullableLong(String value) {\n return value == null || value.isBlank() ? null : Long.parseLong(value.trim());\n }\n }\n}\n" }, { "path": "backend/historical-bootstrap-tool/src/main/java/hu/gombos/flightspotter/historicalbootstrap/RegionalHistoricalTruthAuthorityStore.java", "encoding": "UTF-8", "sha256": "884e43374f7fb726cb163aa60eb85873a007d46ae4e7cb33573bcde55c0a6dd0", "content": "package hu.gombos.flightspotter.historicalbootstrap;\n\nimport hu.gombos.flightspotter.forecastmodel.CoverageDomain;\nimport hu.gombos.flightspotter.forecastmodel.RegionalHistoricalTruthAuthority;\n\nimport javax.sql.DataSource;\nimport java.nio.charset.StandardCharsets;\nimport java.security.MessageDigest;\nimport java.sql.Connection;\nimport java.sql.PreparedStatement;\nimport java.sql.ResultSet;\nimport java.sql.SQLException;\nimport java.sql.Timestamp;\nimport java.time.Duration;\nimport java.time.Instant;\nimport java.time.LocalDate;\nimport java.time.ZoneOffset;\nimport java.util.Arrays;\nimport java.util.HexFormat;\nimport java.util.Objects;\nimport java.util.Set;\nimport java.util.UUID;\n\n/**\n * Immutable JDBC boundary for {@link RegionalHistoricalTruthAuthority#GENERATION}.\n *\n *

Every source day is sealed exactly once per authority fingerprint. Replaying an identical day is\n * idempotent; replaying different evidence for an already-sealed day fails closed in {@link #verify}.

\n */\nfinal class RegionalHistoricalTruthAuthorityStore {\n\n static final String PROVIDER = \"ADSB_LOL\";\n static final String LICENSE = \"ODBL-1.0\";\n\n private final DataSource dataSource;\n\n RegionalHistoricalTruthAuthorityStore(DataSource dataSource) {\n this.dataSource = Objects.requireNonNull(dataSource, \"dataSource\");\n }\n\n static RegionalHistoricalTruthAuthority authority(CoverageDomain domain, Duration maturity) {\n Objects.requireNonNull(domain, \"domain\");\n Objects.requireNonNull(maturity, \"maturity\");\n return new RegionalHistoricalTruthAuthority(\n domain.regionId(),\n domain.canonicalSha256(),\n domain.coverageVersion(),\n domain.evidenceCoverageHash(),\n PROVIDER,\n RegionalHistoricalTruthAuthority.ADSB_LOL_REPOSITORY_POLICY,\n LICENSE,\n RegionalHistoricalTruthAuthority.DEFAULT_MANIFEST_POLICY,\n HistoricalBootstrapPostgresStreamingPersistence.IMPORTER_VERSION,\n GateBSemanticFingerprint.PARSER_VERSION,\n ArchiveSafetyLimits.defaults().semanticFingerprintComponent(),\n HistoricalEligibilityV1.VERSION,\n HistoricalSegmenterV1.VERSION,\n GateBSemanticFingerprint.DAY_DELTA_VERSION,\n RegionalHistoricalTruthAuthority.DEFAULT_COMPLETENESS,\n maturity,\n RegionalHistoricalTruthAuthority.DEFAULT_SEAL);\n }\n\n DaySeal sealQualified(\n RegionalHistoricalTruthAuthority authority,\n BootstrapSourceManifest manifest,\n UUID datasetId,\n LocalDate day\n ) throws Exception {\n Objects.requireNonNull(authority, \"authority\");\n Objects.requireNonNull(manifest, \"manifest\");\n Objects.requireNonNull(datasetId, \"datasetId\");\n Objects.requireNonNull(day, \"day\");\n\n try (Connection connection = dataSource.getConnection()) {\n connection.setAutoCommit(false);\n try {\n register(connection, authority);\n DatasetRow dataset = dataset(connection, datasetId, day);\n Instant readyAt = max(\n dataset.readyAt(),\n incompleteReadyAt(authority, day));\n DaySeal seal = new DaySeal(\n authority.fingerprint(),\n day,\n datasetId,\n \"AUTHORITATIVE_COMPLETE\",\n readyAt,\n manifest.manifestSha256(),\n manifest.dayManifestSha256(day),\n dataset.delta(),\n dataset.semantic(),\n \"QUALIFIED\",\n null,\n hash(\n authority.fingerprint(),\n day,\n datasetId,\n \"AUTHORITATIVE_COMPLETE\",\n readyAt,\n manifest.manifestSha256(),\n manifest.dayManifestSha256(day),\n dataset.delta(),\n dataset.semantic(),\n \"QUALIFIED\",\n null));\n insert(connection, seal);\n verify(connection, seal);\n connection.commit();\n return seal;\n } catch (Exception | Error failure) {\n connection.rollback();\n throw failure;\n }\n }\n }\n\n DaySeal sealIncomplete(\n RegionalHistoricalTruthAuthority authority,\n BootstrapSourceManifest manifest,\n LocalDate day,\n String quality,\n String reason\n ) throws Exception {\n Objects.requireNonNull(authority, \"authority\");\n Objects.requireNonNull(manifest, \"manifest\");\n Objects.requireNonNull(day, \"day\");\n if (!Set.of(\"GAP\", \"DEGRADED\").contains(quality)\n || reason == null\n || !reason.matches(\"[A-Z0-9_]{1,64}\")) {\n throw new IllegalArgumentException(\"invalid incomplete regional truth\");\n }\n\n Instant readyAt = incompleteReadyAt(authority, day);\n String status = quality.equals(\"GAP\") ? \"UNKNOWN_GAP\" : \"UNKNOWN_DEGRADED\";\n DaySeal seal = new DaySeal(\n authority.fingerprint(),\n day,\n null,\n status,\n readyAt,\n manifest.manifestSha256(),\n manifest.dayManifestSha256(day),\n null,\n null,\n quality,\n reason,\n hash(\n authority.fingerprint(),\n day,\n null,\n status,\n readyAt,\n manifest.manifestSha256(),\n manifest.dayManifestSha256(day),\n null,\n null,\n quality,\n reason));\n\n try (Connection connection = dataSource.getConnection()) {\n connection.setAutoCommit(false);\n try {\n register(connection, authority);\n insert(connection, seal);\n verify(connection, seal);\n connection.commit();\n return seal;\n } catch (Exception | Error failure) {\n connection.rollback();\n throw failure;\n }\n }\n }\n\n private void register(\n Connection connection,\n RegionalHistoricalTruthAuthority authority\n ) throws SQLException {\n try (PreparedStatement statement = connection.prepareStatement(\"\"\"\n insert into regional_historical_truth_authorities(\n authority_fingerprint,\n authority_generation,\n region_id,\n region_canonical_sha256,\n coverage_version,\n evidence_coverage_hash,\n provider,\n source_repository_policy,\n license_identifier,\n source_manifest_policy_version,\n importer_version,\n parser_version,\n archive_safety_profile,\n eligibility_version,\n segmentation_version,\n derivation_version,\n completeness_semantics_version,\n maturity_seconds,\n seal_semantics_version\n )\n values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)\n on conflict(authority_fingerprint) do nothing\n \"\"\")) {\n int index = 1;\n statement.setString(index++, authority.fingerprint());\n statement.setString(index++, RegionalHistoricalTruthAuthority.GENERATION);\n statement.setString(index++, authority.regionId().value());\n statement.setString(index++, authority.regionCanonicalSha256());\n statement.setString(index++, authority.coverageVersion());\n statement.setString(index++, authority.evidenceCoverageHash());\n statement.setString(index++, authority.provider());\n statement.setString(index++, authority.sourceRepositoryPolicy());\n statement.setString(index++, authority.licenseIdentifier());\n statement.setString(index++, authority.sourceManifestPolicyVersion());\n statement.setString(index++, authority.importerVersion());\n statement.setString(index++, authority.parserVersion());\n statement.setString(index++, authority.archiveSafetyProfile());\n statement.setString(index++, authority.eligibilityVersion());\n statement.setString(index++, authority.segmentationVersion());\n statement.setString(index++, authority.derivationVersion());\n statement.setString(index++, authority.completenessSemanticsVersion());\n statement.setLong(index++, authority.maturityAfterDayEnd().toSeconds());\n statement.setString(index, authority.sealSemanticsVersion());\n statement.executeUpdate();\n }\n }\n\n private DatasetRow dataset(Connection connection, UUID datasetId, LocalDate day)\n throws SQLException {\n try (PreparedStatement statement = connection.prepareStatement(\"\"\"\n select d.ready_at,\n d.semantic_fingerprint,\n day.day_delta_checksum\n from historical_bootstrap_datasets d\n join historical_bootstrap_days day\n on day.dataset_id = d.dataset_id\n and day.source_date = ?\n where d.dataset_id = ?\n and d.state = 'READY'\n and d.dataset_purpose = 'BOOTSTRAP_SCORING'\n and d.from_date = ?\n and d.to_date = ?\n and day.experiment_role = 'SCORING'\n and day.terminal_quality = 'QUALIFIED'\n \"\"\")) {\n statement.setDate(1, java.sql.Date.valueOf(day));\n statement.setObject(2, datasetId);\n statement.setDate(3, java.sql.Date.valueOf(day));\n statement.setDate(4, java.sql.Date.valueOf(day));\n\n try (ResultSet result = statement.executeQuery()) {\n if (!result.next()) {\n throw new IllegalStateException(\n \"qualified one-day regional dataset is unavailable\");\n }\n DatasetRow dataset = new DatasetRow(\n result.getTimestamp(1).toInstant(),\n result.getString(2),\n result.getString(3));\n if (result.next()) {\n throw new IllegalStateException(\n \"regional dataset/day identity is not unique\");\n }\n return dataset;\n }\n }\n }\n\n private void insert(Connection connection, DaySeal seal) throws SQLException {\n try (PreparedStatement statement = connection.prepareStatement(\"\"\"\n insert into regional_historical_truth_day_seals(\n authority_fingerprint,\n target_day,\n dataset_id,\n status,\n authoritative_ready_at,\n source_manifest_sha256,\n day_manifest_sha256,\n day_delta_checksum,\n dataset_semantic_fingerprint,\n terminal_quality,\n reason_code,\n seal_sha256\n )\n values(?,?,?,?,?,?,?,?,?,?,?,?)\n on conflict(authority_fingerprint,target_day) do nothing\n \"\"\")) {\n statement.setString(1, seal.authorityFingerprint());\n statement.setDate(2, java.sql.Date.valueOf(seal.targetDay()));\n statement.setObject(3, seal.datasetId());\n statement.setString(4, seal.status());\n statement.setTimestamp(5, Timestamp.from(seal.authoritativeReadyAt()));\n statement.setString(6, seal.sourceManifestSha256());\n statement.setString(7, seal.dayManifestSha256());\n statement.setString(8, seal.dayDeltaChecksum());\n statement.setString(9, seal.datasetSemanticFingerprint());\n statement.setString(10, seal.terminalQuality());\n statement.setString(11, seal.reasonCode());\n statement.setString(12, seal.sealSha256());\n statement.executeUpdate();\n }\n }\n\n private void verify(Connection connection, DaySeal expected) throws SQLException {\n try (PreparedStatement statement = connection.prepareStatement(\"\"\"\n select seal_sha256\n from regional_historical_truth_day_seals\n where authority_fingerprint = ?\n and target_day = ?\n \"\"\")) {\n statement.setString(1, expected.authorityFingerprint());\n statement.setDate(2, java.sql.Date.valueOf(expected.targetDay()));\n\n try (ResultSet result = statement.executeQuery()) {\n if (!result.next()\n || !result.getString(1).trim().equals(expected.sealSha256())\n || result.next()) {\n throw new IllegalStateException(\n \"regional historical seal idempotency mismatch\");\n }\n }\n }\n }\n\n static Instant incompleteReadyAt(\n RegionalHistoricalTruthAuthority authority,\n LocalDate day\n ) {\n Objects.requireNonNull(authority, \"authority\");\n Objects.requireNonNull(day, \"day\");\n return day.plusDays(1)\n .atStartOfDay(ZoneOffset.UTC)\n .toInstant()\n .plus(authority.maturityAfterDayEnd());\n }\n\n static boolean repositoryAllowed(String policy, String repository) {\n return RegionalHistoricalTruthAuthority.ADSB_LOL_REPOSITORY_POLICY.equals(policy)\n && repository != null\n && repository.matches(\"adsblol/globe_history_\\\\d{4}\");\n }\n\n private static Instant max(Instant first, Instant second) {\n return first.isAfter(second) ? first : second;\n }\n\n private static String hash(Object... values) {\n String canonical = String.join(\n \"|\",\n Arrays.stream(values)\n .map(value -> value == null ? \"-\" : value.toString())\n .toList());\n try {\n return HexFormat.of().formatHex(\n MessageDigest.getInstance(\"SHA-256\")\n .digest(canonical.getBytes(StandardCharsets.UTF_8)));\n } catch (Exception failure) {\n throw new IllegalStateException(\"SHA-256 unavailable\", failure);\n }\n }\n\n record DaySeal(\n String authorityFingerprint,\n LocalDate targetDay,\n UUID datasetId,\n String status,\n Instant authoritativeReadyAt,\n String sourceManifestSha256,\n String dayManifestSha256,\n String dayDeltaChecksum,\n String datasetSemanticFingerprint,\n String terminalQuality,\n String reasonCode,\n String sealSha256\n ) { }\n\n private record DatasetRow(\n Instant readyAt,\n String semantic,\n String delta\n ) { }\n}\n" }, { "path": "backend/historical-bootstrap-tool/src/main/java/hu/gombos/flightspotter/historicalbootstrap/SourceMetadata.java", "encoding": "UTF-8", "sha256": "e58e9ab2445069e994a00d2990af0694d470812c570353d06cd54812fabe5a86", "content": "package hu.gombos.flightspotter.historicalbootstrap;\n\nimport java.time.LocalDate;\nimport java.util.ArrayList;\nimport java.util.Comparator;\nimport java.util.LinkedHashSet;\nimport java.util.List;\nimport java.util.Locale;\nimport java.util.Objects;\n\nrecord SourceAsset(long assetId, String name, long byteSize, String sha256) {\n SourceAsset {\n if (assetId < 1) throw new IllegalArgumentException(\"assetId must be positive\");\n if (name == null || name.isBlank()) throw new IllegalArgumentException(\"asset name is required\");\n if (byteSize < 1) throw new IllegalArgumentException(\"archive asset byteSize must be positive\");\n sha256 = normalizeDigest(sha256);\n }\n\n private static String normalizeDigest(String digest) {\n if (digest == null || digest.isBlank()) return null;\n String value = digest.trim().toLowerCase(Locale.ROOT);\n if (value.startsWith(\"sha256:\")) value = value.substring(\"sha256:\".length());\n return value.matches(\"[0-9a-f]{64}\") ? value : null;\n }\n}\n\nrecord SourceRelease(\n String repository,\n long releaseId,\n String tag,\n LocalDate sourceDate,\n boolean draft,\n boolean prerelease,\n ReleaseVariant variant,\n List assets\n) {\n SourceRelease {\n if (repository == null || repository.isBlank()) throw new IllegalArgumentException(\"repository is required\");\n if (releaseId < 1) throw new IllegalArgumentException(\"releaseId must be positive\");\n if (tag == null || tag.isBlank()) throw new IllegalArgumentException(\"tag is required\");\n Objects.requireNonNull(sourceDate, \"sourceDate\");\n Objects.requireNonNull(variant, \"variant\");\n assets = assets == null ? List.of() : assets.stream()\n .sorted(Comparator.comparing(SourceAsset::name).thenComparingLong(SourceAsset::assetId))\n .toList();\n }\n}\n\nenum ReleaseVariant {\n PROD,\n STAGING,\n MLAT_ONLY,\n OTHER;\n\n static ReleaseVariant fromTag(String tag) {\n String lower = tag == null ? \"\" : tag.toLowerCase(Locale.ROOT);\n if (lower.contains(\"mlatonly\") || lower.contains(\"mlat-only\")) return MLAT_ONLY;\n if (lower.contains(\"staging\")) return STAGING;\n if (lower.contains(\"prod\")) return PROD;\n return OTHER;\n }\n}\n\nenum ArchiveFamilyKind {\n UNSPLIT,\n SPLIT\n}\n\nrecord ResolvedArchiveFamily(ArchiveFamilyKind kind, List assets) {\n ResolvedArchiveFamily {\n Objects.requireNonNull(kind, \"kind\");\n if (assets == null || assets.isEmpty()) throw new IllegalArgumentException(\"archive family must have assets\");\n assets = assets.stream()\n .sorted(Comparator.comparing(SourceAsset::name).thenComparingLong(SourceAsset::assetId))\n .toList();\n }\n\n long totalBytes() {\n long total = 0L;\n for (SourceAsset asset : assets) total = Math.addExact(total, asset.byteSize());\n return total;\n }\n}\n\nenum SourcePreflightStatus {\n PREFLIGHT_OK,\n NO_RELEASE,\n MLAT_ONLY_RELEASE,\n ASSET_FAMILY_AMBIGUOUS,\n SPLIT_SEQUENCE_INCOMPLETE,\n UPSTREAM_DIGEST_MISSING,\n SOURCE_VOLUME_ANOMALY,\n MANUAL_EXCLUSION,\n UNSUPPORTED_RELEASE_STATE,\n MULTIPLE_RELEASE_CANDIDATES,\n OVERRIDE_INVALID,\n NO_ARCHIVE_ASSETS\n}\n\nrecord ManualSourceOverride(\n LocalDate sourceDate,\n long releaseId,\n List assetIds,\n String reason,\n boolean allowDraftOrPrerelease,\n boolean exclude\n) {\n ManualSourceOverride {\n Objects.requireNonNull(sourceDate, \"sourceDate\");\n if (reason == null || reason.isBlank()) throw new IllegalArgumentException(\"override reason is required\");\n List values = assetIds == null ? List.of() : new ArrayList<>(new LinkedHashSet<>(assetIds));\n values.sort(Long::compareTo);\n assetIds = List.copyOf(values);\n if (exclude) {\n if (releaseId < 0) throw new IllegalArgumentException(\"excluded override releaseId cannot be negative\");\n } else {\n if (releaseId < 1) throw new IllegalArgumentException(\"override releaseId must be positive\");\n if (assetIds.isEmpty()) throw new IllegalArgumentException(\"override assetIds are required\");\n }\n }\n}\n\nrecord PreflightPolicy(Long minimumArchiveBytes, Long maximumArchiveBytes) {\n PreflightPolicy {\n if (minimumArchiveBytes != null && minimumArchiveBytes < 0) {\n throw new IllegalArgumentException(\"minimumArchiveBytes cannot be negative\");\n }\n if (maximumArchiveBytes != null && maximumArchiveBytes < 1) {\n throw new IllegalArgumentException(\"maximumArchiveBytes must be positive\");\n }\n if (minimumArchiveBytes != null && maximumArchiveBytes != null\n && minimumArchiveBytes > maximumArchiveBytes) {\n throw new IllegalArgumentException(\"minimumArchiveBytes cannot exceed maximumArchiveBytes\");\n }\n }\n\n static PreflightPolicy noVolumeBounds() {\n return new PreflightPolicy(null, null);\n }\n\n boolean volumeAllowed(long totalBytes) {\n return (minimumArchiveBytes == null || totalBytes >= minimumArchiveBytes)\n && (maximumArchiveBytes == null || totalBytes <= maximumArchiveBytes);\n }\n}\n\nrecord ResolvedSourceDay(\n LocalDate sourceDate,\n DatasetWindow.DateRole role,\n SourcePreflightStatus preflightStatus,\n List candidateReleases,\n SourceRelease selectedRelease,\n ResolvedArchiveFamily selectedFamily,\n String overrideReason\n) {\n ResolvedSourceDay {\n Objects.requireNonNull(sourceDate, \"sourceDate\");\n Objects.requireNonNull(role, \"role\");\n Objects.requireNonNull(preflightStatus, \"preflightStatus\");\n candidateReleases = candidateReleases == null ? List.of() : candidateReleases.stream()\n .sorted(Comparator.comparingLong(SourceRelease::releaseId))\n .toList();\n if (candidateReleases.stream().anyMatch(release -> !release.sourceDate().equals(sourceDate))) {\n throw new IllegalArgumentException(\"candidate release sourceDate must match day\");\n }\n if ((selectedRelease == null) != (selectedFamily == null)) {\n throw new IllegalArgumentException(\"selected release and family must be present together\");\n }\n if (selectedRelease != null && candidateReleases.stream()\n .noneMatch(candidate -> candidate.releaseId() == selectedRelease.releaseId())) {\n throw new IllegalArgumentException(\"selected release must exist in candidate snapshot\");\n }\n }\n}\n" }, { "path": "backend/truth-model/src/main/java/hu/gombos/flightspotter/truthmodel/forecast/ForecastProductMaterializationCoordinator.java", "encoding": "UTF-8", "sha256": "42294af54d0e4edecc1938b45b5ecb21b1a607a958af85cd3a847f906ec741d0", "content": "package hu.gombos.flightspotter.truthmodel.forecast;\n\nimport io.micrometer.core.instrument.Gauge;\nimport io.micrometer.core.instrument.MeterRegistry;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\nimport org.springframework.scheduling.annotation.Scheduled;\nimport org.springframework.stereotype.Component;\n\nimport java.time.Clock;\nimport java.time.Instant;\nimport java.time.temporal.ChronoUnit;\nimport java.util.Set;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicBoolean;\nimport java.util.concurrent.atomic.AtomicInteger;\n\n/** Maintains the rolling shared-anchor next-24h product cache without mutating P862 prospective evidence. */\n@Component\nfinal class ForecastProductMaterializationCoordinator {\n\n private static final Logger log = LoggerFactory.getLogger(ForecastProductMaterializationCoordinator.class);\n\n private final ForecastProductMaterializationProperties controls;\n private final P862ShadowProperties authority;\n private final ForecastProductMaterializationRepository repository;\n private final MeterRegistry metrics;\n private final Clock clock;\n private final AtomicBoolean running = new AtomicBoolean();\n private final AtomicInteger activeAnchorsGauge = new AtomicInteger();\n private final AtomicInteger pendingBatchGauge = new AtomicInteger();\n\n ForecastProductMaterializationCoordinator(\n ForecastProductMaterializationProperties controls,\n P862ShadowProperties authority,\n ForecastProductMaterializationRepository repository,\n MeterRegistry metrics,\n Clock clock\n ) {\n this.controls = controls;\n this.authority = authority;\n this.repository = repository;\n this.metrics = metrics;\n this.clock = clock;\n Gauge.builder(\"flighttrove.forecast.product.active.anchors\", activeAnchorsGauge, AtomicInteger::get)\n .description(\"Current entitlement-active shared T24 anchor cells\")\n .register(metrics);\n Gauge.builder(\"flighttrove.forecast.product.reconciliation.pending_batch\",\n pendingBatchGauge, AtomicInteger::get)\n .description(\"Missing shared-anchor T24 snapshots selected in the latest bounded reconciliation batch\")\n .register(metrics);\n }\n\n @Scheduled(fixedDelayString = \"${flighttrove.forecast-product-materialization.reconciliation-interval:PT1M}\")\n void reconcile() {\n if (!controls.enabled() || !running.compareAndSet(false, true)) return;\n try {\n Instant now = clock.instant();\n Instant nextWholeHour = now.truncatedTo(ChronoUnit.HOURS).plus(1, ChronoUnit.HOURS);\n Instant endExclusive = nextWholeHour.plus(24, ChronoUnit.HOURS);\n\n Set anchors = repository.activeAnchorCells();\n activeAnchorsGauge.set(anchors.size());\n pendingBatchGauge.set(0);\n if (!anchors.isEmpty()) {\n var tasks = repository.missingSnapshots(\n nextWholeHour, endExclusive, anchors, controls, controls.reconciliationBatchSize());\n pendingBatchGauge.set(tasks.size());\n\n for (ForecastProductMaterializationRepository.SnapshotTask task : tasks) {\n // A task selected just before its target-hour boundary is no longer actionable if execution\n // crosses the boundary. Leave it missing; the read path remains fail-closed.\n Instant attemptNow = clock.instant();\n if (!attemptNow.isBefore(task.targetHour())) continue;\n long started = System.nanoTime();\n try {\n record(repository.materialize(\n authority, controls, task.targetHour(), task.demandAnchorH3R5(), attemptNow));\n } catch (P862ProductReadinessService.P862ProductNotReadyException notReady) {\n metrics.counter(\"flighttrove.forecast.product.materialization\", \"result\", \"not_ready\")\n .increment();\n log.info(\"Forecast product anchor materialization waiting for regional historical/P862 readiness\");\n log.debug(\"Forecast product readiness detail\", notReady);\n // Readiness is authority-wide; retrying more anchors in the same cycle cannot succeed.\n break;\n } catch (RuntimeException transientFailure) {\n metrics.counter(\"flighttrove.forecast.product.materialization\", \"result\", \"retryable_failure\")\n .increment();\n log.warn(\"Forecast product anchor materialization left open for retry anchor={} target={} reason={}\",\n task.demandAnchorH3R5(), task.targetHour(),\n transientFailure.getClass().getSimpleName());\n log.debug(\"Forecast product retryable failure detail\", transientFailure);\n // Infrastructure failures are commonly shared (PostgreSQL/network). Avoid a retry storm;\n // no immutable row was written, so the next reconciliation cycle will retry safely.\n break;\n } finally {\n metrics.timer(\"flighttrove.forecast.product.materialization.duration\",\n \"operation\", \"anchor_snapshot\")\n .record(System.nanoTime() - started, TimeUnit.NANOSECONDS);\n }\n }\n }\n\n int cleaned = repository.cleanupBefore(now.minus(controls.retention()), 1_000);\n if (cleaned > 0) {\n metrics.counter(\"flighttrove.forecast.product.materialization\", \"result\", \"cleaned\")\n .increment(cleaned);\n }\n } catch (ForecastProductZoneRelevantCandidateSource.ProductDemandCapacityExceededException capacity) {\n metrics.counter(\"flighttrove.forecast.product.materialization\", \"result\", \"demand_capacity_exceeded\")\n .increment();\n log.error(\"Forecast product active-anchor capacity exceeded; product remains fail-closed\", capacity);\n } catch (RuntimeException failure) {\n metrics.counter(\"flighttrove.forecast.product.materialization\", \"result\", \"failed\").increment();\n log.warn(\"Forecast product reconciliation failed closed reason={}\", failure.getClass().getSimpleName());\n log.debug(\"Forecast product reconciliation failure detail\", failure);\n } finally {\n running.set(false);\n }\n }\n\n private void record(ForecastProductMaterializationRepository.Result result) {\n metrics.counter(\"flighttrove.forecast.product.materialization\", \"result\", result.name().toLowerCase())\n .increment();\n log.info(\"Forecast product anchor snapshot transition result={}\", result);\n }\n}\n" }, { "path": "backend/truth-model/src/main/java/hu/gombos/flightspotter/truthmodel/forecast/ForecastProductMaterializationProperties.java", "encoding": "UTF-8", "sha256": "2e308f87828ccb451caf839cdb7321c582bd51e6c5009efc3c5f778cf6bc34ee", "content": "package hu.gombos.flightspotter.truthmodel.forecast;\n\nimport org.springframework.boot.context.properties.ConfigurationProperties;\nimport org.springframework.boot.context.properties.bind.ConstructorBinding;\n\nimport java.nio.charset.StandardCharsets;\nimport java.security.MessageDigest;\nimport java.time.Duration;\nimport java.util.HexFormat;\n\n/** Bounded operational controls for the shared-anchor continuous T+24 product projection. */\n@ConfigurationProperties(\"flighttrove.forecast-product-materialization\")\npublic record ForecastProductMaterializationProperties(\n boolean enabled,\n Duration invocationLateness,\n Duration retention,\n int reconciliationBatchSize,\n int maximumCandidatesPerAnchor,\n int maximumActiveZones,\n int maximumActiveAnchors\n) {\n public static final String SELECTION_VERSION = \"forecast-product-shared-anchor-r5-v4\";\n public static final int ZONE_DEMAND_R5_RING = 8;\n public static final double MAX_SUPPORTED_DISCOVERY_RADIUS_KM = 60d;\n public static final double CORRIDOR_CENTER_COVER_KM = 2.5d;\n\n private static final int DEFAULT_RECONCILIATION_BATCH_SIZE = 240;\n private static final int DEFAULT_MAXIMUM_CANDIDATES_PER_ANCHOR = 4_096;\n private static final int DEFAULT_MAXIMUM_ACTIVE_ZONES = 100_000;\n private static final int DEFAULT_MAXIMUM_ACTIVE_ANCHORS = 100_000;\n\n public ForecastProductMaterializationProperties(\n boolean enabled,\n Duration invocationLateness,\n Duration retention,\n int reconciliationBatchSize\n ) {\n this(enabled, invocationLateness, retention, reconciliationBatchSize,\n DEFAULT_MAXIMUM_CANDIDATES_PER_ANCHOR,\n DEFAULT_MAXIMUM_ACTIVE_ZONES,\n DEFAULT_MAXIMUM_ACTIVE_ANCHORS);\n }\n\n @ConstructorBinding\n public ForecastProductMaterializationProperties {\n invocationLateness = invocationLateness == null ? Duration.ofMinutes(5) : invocationLateness;\n retention = retention == null ? Duration.ofDays(3) : retention;\n reconciliationBatchSize = reconciliationBatchSize == 0\n ? DEFAULT_RECONCILIATION_BATCH_SIZE : reconciliationBatchSize;\n maximumCandidatesPerAnchor = maximumCandidatesPerAnchor == 0\n ? DEFAULT_MAXIMUM_CANDIDATES_PER_ANCHOR : maximumCandidatesPerAnchor;\n maximumActiveZones = maximumActiveZones == 0\n ? DEFAULT_MAXIMUM_ACTIVE_ZONES : maximumActiveZones;\n maximumActiveAnchors = maximumActiveAnchors == 0\n ? DEFAULT_MAXIMUM_ACTIVE_ANCHORS : maximumActiveAnchors;\n\n if (invocationLateness.isZero() || invocationLateness.isNegative()\n || invocationLateness.compareTo(Duration.ofMinutes(15)) > 0\n || retention.compareTo(Duration.ofDays(2)) < 0\n || retention.compareTo(Duration.ofDays(30)) > 0\n || reconciliationBatchSize < 24 || reconciliationBatchSize > 10_000\n || maximumCandidatesPerAnchor < 64 || maximumCandidatesPerAnchor > 4_096\n || maximumActiveZones < 1 || maximumActiveZones > 250_000\n || maximumActiveAnchors < 1 || maximumActiveAnchors > 200_000) {\n throw new IllegalArgumentException(\"Invalid bounded forecast product materialization configuration\");\n }\n }\n\n public String selectionFingerprint() {\n String canonical = SELECTION_VERSION\n + \"|r5Ring=\" + ZONE_DEMAND_R5_RING\n + \"|maxSupportedRadiusKm=\" + MAX_SUPPORTED_DISCOVERY_RADIUS_KM\n + \"|corridorCenterCoverKm=\" + CORRIDOR_CENTER_COVER_KM\n + \"|maxCandidatesPerAnchor=\" + maximumCandidatesPerAnchor;\n try {\n return HexFormat.of().formatHex(MessageDigest.getInstance(\"SHA-256\")\n .digest(canonical.getBytes(StandardCharsets.UTF_8)));\n } catch (Exception failure) {\n throw new IllegalStateException(\"SHA-256 unavailable\", failure);\n }\n }\n}\n" }, { "path": "backend/truth-model/src/main/java/hu/gombos/flightspotter/truthmodel/forecast/ForecastProductMaterializationRepository.java", "encoding": "UTF-8", "sha256": "4baa8d859ba26155458acecce09661767a0e16c118d3429c49192574b2104c03", "content": "package hu.gombos.flightspotter.truthmodel.forecast;\n\nimport hu.gombos.flightspotter.forecastmodel.P862FrozenPipeline;\nimport org.springframework.jdbc.core.JdbcTemplate;\nimport org.springframework.stereotype.Repository;\nimport org.springframework.transaction.support.TransactionTemplate;\n\nimport java.nio.charset.StandardCharsets;\nimport java.security.MessageDigest;\nimport java.sql.Timestamp;\nimport java.time.Instant;\nimport java.util.ArrayList;\nimport java.util.HexFormat;\nimport java.util.List;\nimport java.util.Set;\nimport java.util.UUID;\n\n/**\n * Idempotent persistence boundary for shared product T+24 anchor-cell snapshots.\n *\n *

The scientific P862 prospective evidence remains untouched. Product materialization may be computed on time\n * or later as a causal backfill, but every feature read is still frozen at {@code targetHour - 24h}. Therefore a\n * newly created zone can warm its shared anchor cache without using hindsight data.

\n *\n *

Transient infrastructure exceptions never create an immutable invocation row. The coordinator retries them.\n * Only a deterministic candidate-capacity overflow is persisted as a terminal snapshot state for the current\n * selection fingerprint.

\n */\n@Repository\npublic class ForecastProductMaterializationRepository {\n\n enum Result { FORECASTED, CAPACITY_EXCEEDED, DUPLICATE }\n\n record SnapshotTask(Instant targetHour, String demandAnchorH3R5) {}\n\n private final JdbcTemplate jdbc;\n private final TransactionTemplate transactions;\n private final ForecastProductZoneRelevantCandidateSource candidates;\n private final P862ProductReadinessService readiness;\n\n public ForecastProductMaterializationRepository(\n JdbcTemplate jdbc,\n TransactionTemplate transactions,\n ForecastProductZoneRelevantCandidateSource candidates,\n P862ProductReadinessService readiness\n ) {\n this.jdbc = jdbc;\n this.transactions = transactions;\n this.candidates = candidates;\n this.readiness = readiness;\n }\n\n Set activeAnchorCells() {\n return candidates.activeAnchorCells();\n }\n\n Result materialize(\n P862ShadowProperties authority,\n ForecastProductMaterializationProperties controls,\n Instant targetHour,\n String demandAnchorH3R5,\n Instant generatedAt\n ) {\n requireAuthority(authority);\n Instant asOf = P862FrozenPipeline.asOf(targetHour);\n if (generatedAt == null || generatedAt.isBefore(asOf) || !generatedAt.isBefore(targetHour)) {\n throw new IllegalStateException(\n \"Forecast product generation must occur after exact T-24 as-of and before the target hour\");\n }\n String selectionFingerprint = controls.selectionFingerprint();\n if (exists(targetHour, demandAnchorH3R5, selectionFingerprint)) return Result.DUPLICATE;\n\n readiness.requireReady(authority, asOf);\n\n List selected;\n try {\n selected = candidates.load(asOf, targetHour, demandAnchorH3R5);\n } catch (P862RegionalFeatureSource.ProductCandidateCapacityExceededException capacityExceeded) {\n return persistCapacityExceeded(\n authority, controls, targetHour, demandAnchorH3R5, asOf, generatedAt,\n capacityExceeded.getClass().getSimpleName());\n }\n\n return transactions.execute(status -> {\n lock(targetHour, demandAnchorH3R5, selectionFingerprint);\n if (exists(targetHour, demandAnchorH3R5, selectionFingerprint)) return Result.DUPLICATE;\n insertInvocation(\n authority, controls, targetHour, demandAnchorH3R5, asOf, generatedAt,\n generationMode(authority, asOf, generatedAt), \"FORECASTED\", selected.size(), null);\n for (int index = 0; index < selected.size(); index++) {\n insertCandidate(\n authority, controls, targetHour, demandAnchorH3R5, asOf, generatedAt,\n selected.get(index), index + 1);\n }\n return Result.FORECASTED;\n });\n }\n\n private Result persistCapacityExceeded(\n P862ShadowProperties authority,\n ForecastProductMaterializationProperties controls,\n Instant targetHour,\n String demandAnchorH3R5,\n Instant asOf,\n Instant generatedAt,\n String failureCode\n ) {\n return transactions.execute(status -> {\n lock(targetHour, demandAnchorH3R5, controls.selectionFingerprint());\n if (exists(targetHour, demandAnchorH3R5, controls.selectionFingerprint())) return Result.DUPLICATE;\n insertInvocation(\n authority, controls, targetHour, demandAnchorH3R5, asOf, generatedAt,\n generationMode(authority, asOf, generatedAt), \"CAPACITY_EXCEEDED\", 0, failureCode);\n return Result.CAPACITY_EXCEEDED;\n });\n }\n\n /**\n * Returns bounded missing snapshots for active anchors. The caller chooses a future-only target range, allowing\n * newly created demand to warm all still-actionable hours without inventing a post-target forecast.\n */\n List missingSnapshots(\n Instant fromInclusive,\n Instant toExclusive,\n Set demandAnchors,\n ForecastProductMaterializationProperties controls,\n int limit\n ) {\n if (fromInclusive == null || toExclusive == null || !fromInclusive.isBefore(toExclusive)\n || demandAnchors == null || demandAnchors.isEmpty()\n || demandAnchors.size() > controls.maximumActiveAnchors()\n || limit < 1 || limit > 10_000) {\n throw new IllegalArgumentException(\"Invalid forecast product snapshot reconciliation range\");\n }\n String anchorsCsv = demandAnchors.stream().sorted()\n .collect(java.util.stream.Collectors.joining(\",\"));\n return jdbc.query(\"\"\"\n with anchors as materialized (\n select unnest(string_to_array(?, ','))::varchar(15) demand_anchor_h3_r5\n ), targets as materialized (\n select generate_series(?::timestamptz, (?::timestamptz - interval '1 hour'), interval '1 hour')\n target_hour\n )\n select t.target_hour,a.demand_anchor_h3_r5\n from targets t cross join anchors a\n left join forecast_product_invocations i\n on i.target_hour=t.target_hour\n and i.demand_anchor_h3_r5=a.demand_anchor_h3_r5\n and i.selection_fingerprint=?\n where i.target_hour is null\n -- Complete one anchor's full horizon before spending the bounded batch on the next.\n -- Product reads require all 24 anchor-hours, so target-first ordering can keep every user\n -- UNAVAILABLE during a large cold start even though the same work could make some anchors usable.\n order by a.demand_anchor_h3_r5,t.target_hour\n limit ?\n \"\"\", (rs, ignored) -> new SnapshotTask(\n rs.getTimestamp(\"target_hour\").toInstant(),\n rs.getString(\"demand_anchor_h3_r5\")),\n anchorsCsv, Timestamp.from(fromInclusive), Timestamp.from(toExclusive),\n controls.selectionFingerprint(), limit);\n }\n\n int cleanupBefore(Instant cutoff, int limit) {\n if (cutoff == null || limit < 1 || limit > 10_000) {\n throw new IllegalArgumentException(\"Invalid forecast product cleanup boundary\");\n }\n return jdbc.update(\"\"\"\n with expired as (\n select target_hour,demand_anchor_h3_r5,selection_fingerprint\n from forecast_product_invocations\n where target_hour 0;\n }\n\n private void lock(Instant targetHour, String demandAnchorH3R5, String selectionFingerprint) {\n jdbc.query(\"select pg_advisory_xact_lock(hashtextextended(?,0))\", rs -> null,\n \"forecast-product:\" + targetHour + \":\" + demandAnchorH3R5 + \":\" + selectionFingerprint);\n }\n\n private static String generationMode(P862ShadowProperties authority, Instant asOf, Instant generatedAt) {\n return generatedAt.isAfter(asOf.plus(authority.invocationLateness()))\n ? \"CAUSAL_BACKFILL\" : \"ON_TIME\";\n }\n\n private static void requireAuthority(P862ShadowProperties authority) {\n if (!authority.enabled() || !authority.fingerprintMatches()\n || !authority.truthAuthorityFingerprint().matches(\"[0-9a-f]{64}\")) {\n throw new IllegalStateException(\"Frozen P862 product authority is unavailable\");\n }\n }\n\n private static UUID uuid(String canonical) {\n return UUID.nameUUIDFromBytes(canonical.getBytes(StandardCharsets.UTF_8));\n }\n\n private static String sha(String canonical) {\n try {\n return HexFormat.of().formatHex(MessageDigest.getInstance(\"SHA-256\")\n .digest(canonical.getBytes(StandardCharsets.UTF_8)));\n } catch (Exception failure) {\n throw new IllegalStateException(failure);\n }\n }\n}\n" }, { "path": "backend/truth-model/src/main/java/hu/gombos/flightspotter/truthmodel/forecast/ForecastProductZoneRelevantCandidateSource.java", "encoding": "UTF-8", "sha256": "ec3ff96c0b4e2a04243973e44833a08234a6900cc812315bc61a31241d69e06f", "content": "package hu.gombos.flightspotter.truthmodel.forecast;\n\nimport com.uber.h3core.H3Core;\nimport hu.gombos.flightspotter.common.geo.H3CoreFactory;\nimport hu.gombos.flightspotter.truthmodel.subscription.SubscriptionEntitlementPolicy;\nimport org.springframework.beans.factory.annotation.Autowired;\nimport org.springframework.jdbc.core.JdbcTemplate;\nimport org.springframework.stereotype.Component;\n\n\nimport java.sql.ResultSet;\nimport java.sql.SQLException;\nimport java.util.ArrayList;\nimport java.util.LinkedHashMap;\nimport java.util.LinkedHashSet;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\n\n/**\n * Shared product relevance adapter over immutable regional P862 evidence.\n *\n *

The materialization unit is an H3 R5 anchor cell, not a user or saved-zone identity. Every entitlement-active\n * saved zone contributes only its anchor identity. All users whose zones fall into the same anchor reuse one immutable\n * hourly snapshot. Candidate discovery covers the conservative R5 ring around the anchor and persists the complete\n * bounded candidate universe; exact user/tier distance is applied only by {@code TomorrowForecastReadService}.

\n *\n *

This separation is a correctness boundary: one user's demand can never rank, truncate or otherwise change\n * another user's T+24 candidate set. Capacity overflow fails closed for the affected anchor instead of silently\n * materializing a partial AVAILABLE snapshot.

\n */\n@Component\nfinal class ForecastProductZoneRelevantCandidateSource {\n\n private final JdbcTemplate jdbc;\n private final P862RegionalFeatureSource features;\n private final ForecastProductMaterializationProperties controls;\n private final SubscriptionEntitlementPolicy entitlements;\n private final H3Core h3;\n\n @Autowired\n ForecastProductZoneRelevantCandidateSource(\n JdbcTemplate jdbc,\n P862RegionalFeatureSource features,\n ForecastProductMaterializationProperties controls,\n SubscriptionEntitlementPolicy entitlements\n ) {\n this(jdbc, features, controls, entitlements, H3CoreFactory.create());\n }\n\n ForecastProductZoneRelevantCandidateSource(\n JdbcTemplate jdbc,\n P862RegionalFeatureSource features,\n ForecastProductMaterializationProperties controls,\n SubscriptionEntitlementPolicy entitlements,\n H3Core h3\n ) {\n this.jdbc = java.util.Objects.requireNonNull(jdbc, \"jdbc\");\n this.features = java.util.Objects.requireNonNull(features, \"features\");\n this.controls = java.util.Objects.requireNonNull(controls, \"controls\");\n this.entitlements = java.util.Objects.requireNonNull(entitlements, \"entitlements\");\n this.h3 = java.util.Objects.requireNonNull(h3, \"h3\");\n validateSupportedEntitlementRadius();\n }\n\n Set activeAnchorCells() {\n List zones = jdbc.query(\"\"\"\n select id::text as zone_id,latitude,longitude\n from saved_alert_locations\n where entitlement_active=true\n order by id\n limit ?\n \"\"\", ForecastProductZoneRelevantCandidateSource::zonePoint,\n controls.maximumActiveZones() + 1);\n if (zones.size() > controls.maximumActiveZones()) {\n throw new ProductDemandCapacityExceededException(\n \"entitlement-active spotting-zone count exceeded configured T24 bound\");\n }\n\n LinkedHashSet anchors = new LinkedHashSet<>();\n for (ZonePoint zone : zones) {\n if (zone.id() == null || zone.id().isBlank()\n || !validCoordinate(zone.latitude(), zone.longitude())) {\n throw new ProductDemandCapacityExceededException(\n \"entitlement-active spotting zone has invalid identity or coordinates\");\n }\n anchors.add(h3.latLngToCellAddress(zone.latitude(), zone.longitude(), 5));\n if (anchors.size() > controls.maximumActiveAnchors()) {\n throw new ProductDemandCapacityExceededException(\n \"active T24 anchor-cell count exceeded configured bound\");\n }\n }\n return Set.copyOf(anchors);\n }\n\n List load(java.time.Instant asOf, java.time.Instant target, String demandAnchorH3R5) {\n requireAnchor(demandAnchorH3R5);\n Set coveredCells = Set.copyOf(\n h3.gridDisk(demandAnchorH3R5, ForecastProductMaterializationProperties.ZONE_DEMAND_R5_RING));\n if (coveredCells.isEmpty()) {\n throw new IllegalStateException(\"T24 anchor coverage unexpectedly empty\");\n }\n\n // The feature source itself asks for maximum+1 and fails closed on overflow, therefore every successful\n // snapshot is complete up to the explicit per-anchor product capacity. No ranking/truncation is allowed here.\n List universe =\n features.loadProductCandidateUniverse(\n asOf,\n target,\n coveredCells,\n controls.maximumCandidatesPerAnchor());\n\n Map unique = new LinkedHashMap<>();\n for (P862RegionalFeatureSource.Candidate candidate : universe) {\n if (candidate == null || candidate.candidateKey() == null || candidate.candidateKey().isBlank()) {\n throw new IllegalStateException(\"regional P862 product candidate has invalid identity\");\n }\n String parent = parentR5(candidate.corridorH3R7());\n if (!coveredCells.contains(parent)) {\n throw new IllegalStateException(\"regional P862 product candidate escaped requested anchor coverage\");\n }\n P862RegionalFeatureSource.Candidate previous = unique.putIfAbsent(candidate.candidateKey(), candidate);\n if (previous != null && !previous.equals(candidate)) {\n throw new IllegalStateException(\"regional P862 product candidate identity collision\");\n }\n }\n if (unique.size() > controls.maximumCandidatesPerAnchor()) {\n throw new P862RegionalFeatureSource.ProductCandidateCapacityExceededException(\n \"anchor T24 candidate universe exceeded configured materialization bound\");\n }\n\n List selected = new ArrayList<>(unique.size());\n unique.values().forEach(candidate -> selected.add(new SelectedCandidate(candidate, demandAnchorH3R5)));\n return List.copyOf(selected);\n }\n\n private String parentR5(String corridorR7) {\n if (corridorR7 == null || corridorR7.isBlank() || h3.getResolution(corridorR7) != 7) {\n throw new IllegalStateException(\"regional P862 candidate contains invalid R7 corridor cell\");\n }\n var center = h3.cellToLatLng(corridorR7);\n if (center == null || !validCoordinate(center.lat, center.lng)) {\n throw new IllegalStateException(\"regional P862 candidate corridor center is invalid\");\n }\n return h3.latLngToCellAddress(center.lat, center.lng, 5);\n }\n\n private void requireAnchor(String anchor) {\n if (anchor == null || anchor.isBlank() || !anchor.matches(\"^85[0-9a-f]{13}$\")\n || h3.getResolution(anchor) != 5) {\n throw new IllegalArgumentException(\"T24 demand anchor must be a valid H3 R5 cell\");\n }\n }\n\n private void validateSupportedEntitlementRadius() {\n double maximum = entitlements.visibleTiers().stream()\n .map(entitlements::templateFor)\n .mapToDouble(value -> value.discoveryRadiusKm())\n .max()\n .orElse(0d);\n if (!Double.isFinite(maximum)\n || maximum > ForecastProductMaterializationProperties.MAX_SUPPORTED_DISCOVERY_RADIUS_KM) {\n throw new IllegalStateException(\n \"T24 anchor coverage must be reviewed when entitlement discovery radius exceeds \"\n + ForecastProductMaterializationProperties.MAX_SUPPORTED_DISCOVERY_RADIUS_KM + \" km\");\n }\n }\n\n private static ZonePoint zonePoint(ResultSet rs, int ignored) throws SQLException {\n return new ZonePoint(rs.getString(\"zone_id\"), rs.getDouble(\"latitude\"), rs.getDouble(\"longitude\"));\n }\n\n private static boolean validCoordinate(double latitude, double longitude) {\n return Double.isFinite(latitude) && Double.isFinite(longitude)\n && latitude >= -90d && latitude <= 90d\n && longitude >= -180d && longitude <= 180d;\n }\n\n\n record SelectedCandidate(P862RegionalFeatureSource.Candidate candidate, String demandAnchorH3R5) {\n SelectedCandidate {\n if (candidate == null || demandAnchorH3R5 == null || demandAnchorH3R5.isBlank()) {\n throw new IllegalArgumentException(\"selected T24 product candidate requires anchor-cell identity\");\n }\n }\n }\n\n private record ZonePoint(String id, double latitude, double longitude) {}\n\n static final class ProductDemandCapacityExceededException extends IllegalStateException {\n ProductDemandCapacityExceededException(String message) {\n super(message);\n }\n }\n}\n" }, { "path": "backend/truth-model/src/main/java/hu/gombos/flightspotter/truthmodel/forecast/P862ProductReadinessService.java", "encoding": "UTF-8", "sha256": "51b7bdce2e387edd9e6566be0c5df3905db3aa543836b4115836085650c68236", "content": "package hu.gombos.flightspotter.truthmodel.forecast;\n\nimport hu.gombos.flightspotter.forecastmodel.P862FrozenPipeline;\nimport org.springframework.jdbc.core.JdbcTemplate;\nimport org.springframework.stereotype.Component;\n\nimport java.sql.Timestamp;\nimport java.time.Instant;\nimport java.time.LocalDate;\nimport java.time.ZoneOffset;\nimport java.util.Objects;\n\n/**\n * Regional historical authority readiness for continuous exact-T24 materialization.\n *\n *

The 56-day warmup must consist exclusively of immutable READY + QUALIFIED source-day datasets\n * from the exact configured regional authority, all known by the causal as-of.

\n */\n@Component\nfinal class P862ProductReadinessService {\n private final JdbcTemplate jdbc;\n\n P862ProductReadinessService(JdbcTemplate jdbc) {\n this.jdbc = Objects.requireNonNull(jdbc, \"jdbc\");\n }\n\n Readiness evaluate(P862ShadowProperties authority, Instant asOf) {\n Objects.requireNonNull(authority, \"authority\");\n Objects.requireNonNull(asOf, \"asOf\");\n\n LocalDate asOfDay = asOf.atZone(ZoneOffset.UTC).toLocalDate();\n LocalDate from = asOfDay.minusDays(P862FrozenPipeline.WARMUP_COMPLETE_DAYS + 1L);\n LocalDate toExclusive = asOfDay.minusDays(1L);\n\n Integer registeredAuthority = jdbc.queryForObject(\"\"\"\n select count(*) from regional_historical_truth_authorities\n where authority_fingerprint=? and authority_generation=? and created_at<=?\n \"\"\", Integer.class, authority.truthAuthorityFingerprint(),\n authority.authorityGeneration(), Timestamp.from(asOf));\n if (registeredAuthority == null || registeredAuthority != 1) {\n return new Readiness(false, 0, 0);\n }\n\n Integer qualifiedDays = jdbc.queryForObject(\"\"\"\n select count(*)\n from p862_effective_truth_day_seals d\n join historical_bootstrap_ready_datasets dataset\n on dataset.dataset_id=d.historical_dataset_id\n join historical_bootstrap_days source_day\n on source_day.dataset_id=dataset.dataset_id\n and source_day.source_date=d.target_day\n where d.target_day>=? and d.target_day rows = jdbc.query(\"\"\"\n select (p.target_hour at time zone 'UTC')::date as target_day,c.route_key,o.outcome\n from p862_shadow_predictions p join p862_hourly_candidates c using(candidate_id)\n left join p862_shadow_outcomes o using(prediction_id)\n where p.window_id=? order by p.target_hour,p.daily_rank\n \"\"\", (rs, ignored) -> new P862ProspectiveEvaluator.Observation(\n rs.getObject(\"target_day\", LocalDate.class), rs.getString(\"route_key\"), map(rs.getString(\"outcome\"))),\n properties.windowId());\n var coverage = new P862ProspectiveEvaluator.Coverage(expectedHours, forecasted, missed, failures,\n maturedHours, unknown, (long) P862FrozenPipeline.WINDOW_DAYS\n * P862FrozenPipeline.DAILY_CAP, selected);\n P862ProspectiveEvaluator.Report budget = evaluator.evaluate(coverage, rows);\n long top2Selected = diagnosticCount(\"hourly_rank<=2\", \"1=1\");\n long top2Positive = diagnosticCount(\"hourly_rank<=2\", \"o.outcome='POSITIVE'\");\n long top2Unknown = diagnosticCount(\"hourly_rank<=2\",\n \"(o.outcome is null or o.outcome in ('UNKNOWN_INCOMPLETE_TRUTH','INELIGIBLE_OVERFLOW'))\");\n long top3Selected = diagnosticCount(\"hourly_rank<=3\", \"1=1\");\n long top3Positive = diagnosticCount(\"hourly_rank<=3\", \"o.outcome='POSITIVE'\");\n long top3Unknown = diagnosticCount(\"hourly_rank<=3\",\n \"(o.outcome is null or o.outcome in ('UNKNOWN_INCOMPLETE_TRUTH','INELIGIBLE_OVERFLOW'))\");\n return new EvidenceReport(budget, new HourlyDiagnostics(top2Selected, top2Positive, top2Unknown,\n ratio(top2Positive, top2Selected - top2Unknown), top3Selected, top3Positive, top3Unknown,\n ratio(top3Positive, top3Selected - top3Unknown), \"833/1080=0.771296296296\"));\n }\n\n private long count(String sql, Object... additionalArguments) {\n Object[] arguments = new Object[additionalArguments.length + 1];\n arguments[0] = properties.windowId();\n System.arraycopy(additionalArguments, 0, arguments, 1, additionalArguments.length);\n Long value = jdbc.queryForObject(sql, Long.class, arguments);\n return value == null ? 0 : value;\n }\n private long diagnosticCount(String rankPredicate, String outcomePredicate) {\n String sql = \"select count(*) from p862_hourly_diagnostics h left join p862_hourly_diagnostic_outcomes o \"\n + \"using(diagnostic_id) where h.window_id=? and \" + rankPredicate + \" and \" + outcomePredicate;\n Long value = jdbc.queryForObject(sql, Long.class, properties.windowId());\n return value == null ? 0 : value;\n }\n private static double ratio(long numerator, long denominator) {\n return denominator <= 0 ? 0d : (double) numerator / denominator;\n }\n private static P862ProspectiveEvaluator.Outcome map(String value) {\n if (value == null) return P862ProspectiveEvaluator.Outcome.UNMATURED;\n return P862ProspectiveEvaluator.Outcome.valueOf(value);\n }\n}\n" }, { "path": "backend/truth-model/src/main/java/hu/gombos/flightspotter/truthmodel/forecast/P862RegionalFeatureSource.java", "encoding": "UTF-8", "sha256": "1ea97bbe0261573cda6938bd429b8bc84082af39c745e9ab51ac161ee3f4215a", "content": "package hu.gombos.flightspotter.truthmodel.forecast;\n\nimport hu.gombos.flightspotter.forecastmodel.P862FrozenPipeline;\nimport hu.gombos.flightspotter.forecastmodel.RouteRateScore;\nimport org.springframework.jdbc.core.JdbcTemplate;\nimport org.springframework.stereotype.Component;\n\nimport java.math.BigDecimal;\nimport java.sql.Timestamp;\nimport java.time.Instant;\nimport java.time.ZoneOffset;\nimport java.util.ArrayList;\nimport java.util.Comparator;\nimport java.util.List;\n\n/**\n * Bounded regional historical feature source for exact causal T-24 prediction.\n * No user-demand LIVE observation can contribute to this source.\n */\n@Component\nfinal class P862RegionalFeatureSource {\n private static final Comparator ORDER = Comparator.comparing(Candidate::score).reversed()\n .thenComparing(Comparator.comparingLong(Candidate::hourSupport).reversed())\n .thenComparing(Comparator.comparingLong(Candidate::corridorSupport).reversed())\n .thenComparing(Candidate::candidateKey);\n private final JdbcTemplate jdbc;\n private final P862ShadowProperties properties;\n\n P862RegionalFeatureSource(JdbcTemplate jdbc, P862ShadowProperties properties) {\n this.jdbc = jdbc; this.properties = properties;\n }\n\n List load(Instant asOf, Instant target) {\n return loadRegionalHistorical(asOf, target);\n }\n\n /**\n * Product-only candidate universe. Unlike the frozen prospective evidence path this must not\n * globally top-K the region before spotting-zone relevance is applied. It returns the complete\n * regionally qualified/scored universe up to an explicit hard bound; overflow fails closed.\n */\n List loadProductCandidateUniverse(\n Instant asOf,\n Instant target,\n java.util.Set demandH3R5,\n int maximumRows\n ) {\n if (maximumRows < 1 || maximumRows > 250_000) {\n throw new IllegalArgumentException(\"product candidate universe bound must be in 1..250000\");\n }\n if (demandH3R5 == null || demandH3R5.isEmpty() || demandH3R5.size() > 200_000\n || demandH3R5.stream().anyMatch(value -> value == null || !value.matches(\"^85[0-9a-f]{13}$\"))) {\n throw new IllegalArgumentException(\"product candidate demand cells are missing or invalid\");\n }\n String demandCsv = demandH3R5.stream().sorted().collect(java.util.stream.Collectors.joining(\",\"));\n List ranked = loadRegionalHistoricalUniverse(asOf, target, maximumRows + 1, demandCsv);\n if (ranked.size() > maximumRows) {\n throw new ProductCandidateCapacityExceededException(\n \"regional P862 product candidate universe exceeded configured bound \" + maximumRows);\n }\n return ranked;\n }\n\n /**\n * Region-wide historical evidence source. Every contributing day is bound to the configured\n * immutable historical authority and had to be READY + QUALIFIED before the exact as-of.\n * No live coverage demand or observed-flight row participates in this branch.\n */\n private List loadRegionalHistorical(Instant asOf, Instant target) {\n List ranked = loadRegionalHistoricalUniverse(asOf, target, 0, null);\n return List.copyOf(ranked.subList(0, Math.min(3, ranked.size())));\n }\n\n static final class ProductCandidateCapacityExceededException extends IllegalStateException {\n ProductCandidateCapacityExceededException(String message) {\n super(message);\n }\n }\n\n private List loadRegionalHistoricalUniverse(\n Instant asOf, Instant target, int rowLimit, String demandH3R5Csv) {\n P862FrozenPipeline.requireExactT24(target, asOf);\n short hourOfWeek = (short) ((target.atZone(ZoneOffset.UTC).getDayOfWeek().getValue() - 1) * 24\n + target.atZone(ZoneOffset.UTC).getHour());\n int isoDow = target.atZone(ZoneOffset.UTC).getDayOfWeek().getValue();\n Instant from = asOf.minus(properties.evidenceRetention());\n\n Integer exposures = jdbc.queryForObject(\"\"\"\n select count(*)\n from p862_effective_truth_day_seals d\n join historical_bootstrap_ready_datasets dataset\n on dataset.dataset_id=d.historical_dataset_id\n where d.status='AUTHORITATIVE_COMPLETE'\n and d.source_fingerprint=?\n and d.authority_generation=?\n and d.historical_dataset_id is not null\n and d.authoritative_ready_at<=? and d.created_at<=?\n and d.target_day>=?::timestamptz::date\n and extract(isodow from d.target_day)=?\n \"\"\", Integer.class, properties.truthAuthorityFingerprint(), properties.authorityGeneration(),\n Timestamp.from(asOf), Timestamp.from(asOf), Timestamp.from(from), isoDow);\n if (exposures == null || exposures < 1) return List.of();\n if (demandH3R5Csv != null) {\n return loadDemandFilteredProductUniverse(\n asOf, from, hourOfWeek, rowLimit, demandH3R5Csv, exposures);\n }\n\n List values = jdbc.query(\"\"\"\n with qualified_days as materialized (\n select d.target_day,d.historical_dataset_id\n from p862_effective_truth_day_seals d\n join historical_bootstrap_ready_datasets dataset\n on dataset.dataset_id=d.historical_dataset_id\n where d.status='AUTHORITATIVE_COMPLETE'\n and d.source_fingerprint=?\n and d.authority_generation=?\n and d.historical_dataset_id is not null\n and d.authoritative_ready_at<=? and d.created_at<=?\n and d.target_day>=?::timestamptz::date\n ), route_totals as materialized (\n select p.start_h3_r5,p.end_h3_r5,\n sum(p.evidence_count) route_support,\n sum(p.qualified_evidence_count) qualified_support\n from qualified_days q\n join historical_bootstrap_ready_pattern_aggregates p\n on p.dataset_id=q.historical_dataset_id\n where p.pattern_kind='ROUTE' and p.subject_key='-'\n group by p.start_h3_r5,p.end_h3_r5\n ), hour_totals as materialized (\n select p.start_h3_r5,p.end_h3_r5,sum(h.evidence_count) hour_support\n from qualified_days q\n join historical_bootstrap_ready_pattern_aggregates p\n on p.dataset_id=q.historical_dataset_id\n join historical_bootstrap_ready_hour_bins h on h.pattern_id=p.pattern_id\n where p.pattern_kind='ROUTE' and p.subject_key='-' and h.hour_of_week=?\n group by p.start_h3_r5,p.end_h3_r5\n ), routes as materialized (\n select r.start_h3_r5,r.end_h3_r5,r.route_support,r.qualified_support,h.hour_support\n from route_totals r join hour_totals h using(start_h3_r5,end_h3_r5)\n where r.route_support>=2\n and r.qualified_support=r.route_support\n and h.hour_support>=1\n order by h.hour_support desc,r.route_support desc,r.start_h3_r5,r.end_h3_r5\n \"\"\" + (rowLimit == 0 ? \" limit 5000\\n\" : \"\") + \"\"\"\n ), cells as (\n select r.*,c.corridor_h3_r7,sum(c.traversal_count) corridor_support,\n row_number() over(partition by r.start_h3_r5,r.end_h3_r5\n order by sum(c.traversal_count) desc,c.corridor_h3_r7) cell_rank\n from routes r\n join qualified_days q on true\n join historical_bootstrap_ready_route_corridor_aggregates c\n on c.dataset_id=q.historical_dataset_id\n and c.start_h3_r5=r.start_h3_r5\n and c.end_h3_r5=r.end_h3_r5\n \"\"\" + (demandH3R5Csv == null\n ? \"\"\n : \" and c.corridor_h3_r5 = any(string_to_array(?, ','))\\n\") + \"\"\"\n group by r.start_h3_r5,r.end_h3_r5,r.route_support,r.qualified_support,\n r.hour_support,c.corridor_h3_r7\n )\n select * from cells\n where cell_rank<=8 and corridor_support>=1\n order by hour_support desc,route_support desc,start_h3_r5,end_h3_r5,cell_rank\n \"\"\" + (rowLimit > 0 ? \" limit ?\" : \"\"), (rs, ignored) -> {\n long route = rs.getLong(\"route_support\");\n long qualified = rs.getLong(\"qualified_support\");\n long hourSupport = rs.getLong(\"hour_support\");\n long corridor = rs.getLong(\"corridor_support\");\n BigDecimal score = RouteRateScore.calculate(RouteRateScore.Variant.THINNED, route, qualified,\n hourSupport, corridor, exposures).score();\n String startCell = rs.getString(\"start_h3_r5\");\n String endCell = rs.getString(\"end_h3_r5\");\n String corridorCell = rs.getString(\"corridor_h3_r7\");\n String routeKey = startCell + \">\" + endCell;\n return new Candidate(startCell, endCell, corridorCell, routeKey, routeKey + \">\" + corridorCell,\n score, route, qualified, hourSupport, corridor, exposures);\n }, queryArguments(\n rowLimit,\n demandH3R5Csv,\n properties.truthAuthorityFingerprint(),\n properties.authorityGeneration(),\n Timestamp.from(asOf),\n Timestamp.from(asOf),\n Timestamp.from(from),\n hourOfWeek));\n\n List ranked = new ArrayList<>(values);\n ranked.sort(ORDER);\n return List.copyOf(ranked);\n }\n\n private List loadDemandFilteredProductUniverse(\n Instant asOf,\n Instant from,\n short hourOfWeek,\n int rowLimit,\n String demandH3R5Csv,\n int exposures\n ) {\n List values = jdbc.query(\"\"\"\n with qualified_days as materialized (\n select d.target_day,d.historical_dataset_id\n from p862_effective_truth_day_seals d\n join historical_bootstrap_ready_datasets dataset\n on dataset.dataset_id=d.historical_dataset_id\n where d.status='AUTHORITATIVE_COMPLETE'\n and d.source_fingerprint=?\n and d.authority_generation=?\n and d.historical_dataset_id is not null\n and d.authoritative_ready_at<=? and d.created_at<=?\n and d.target_day>=?::timestamptz::date\n ), demand_cells as materialized (\n select unnest(string_to_array(?, ','))::varchar(15) corridor_h3_r5\n ), demand_corridors as materialized (\n select c.start_h3_r5,c.end_h3_r5,c.corridor_h3_r7,\n sum(c.traversal_count) corridor_support\n from qualified_days q\n join historical_bootstrap_ready_route_corridor_aggregates c\n on c.dataset_id=q.historical_dataset_id\n join demand_cells dc on dc.corridor_h3_r5=c.corridor_h3_r5\n group by c.start_h3_r5,c.end_h3_r5,c.corridor_h3_r7\n ), relevant_routes as materialized (\n select distinct start_h3_r5,end_h3_r5 from demand_corridors\n ), route_totals as materialized (\n select p.start_h3_r5,p.end_h3_r5,\n sum(p.evidence_count) route_support,\n sum(p.qualified_evidence_count) qualified_support\n from qualified_days q\n join historical_bootstrap_ready_pattern_aggregates p\n on p.dataset_id=q.historical_dataset_id\n join relevant_routes rr\n on rr.start_h3_r5=p.start_h3_r5\n and rr.end_h3_r5=p.end_h3_r5\n where p.pattern_kind='ROUTE' and p.subject_key='-'\n group by p.start_h3_r5,p.end_h3_r5\n ), hour_totals as materialized (\n select p.start_h3_r5,p.end_h3_r5,sum(h.evidence_count) hour_support\n from qualified_days q\n join historical_bootstrap_ready_pattern_aggregates p\n on p.dataset_id=q.historical_dataset_id\n join relevant_routes rr\n on rr.start_h3_r5=p.start_h3_r5\n and rr.end_h3_r5=p.end_h3_r5\n join historical_bootstrap_ready_hour_bins h on h.pattern_id=p.pattern_id\n where p.pattern_kind='ROUTE' and p.subject_key='-' and h.hour_of_week=?\n group by p.start_h3_r5,p.end_h3_r5\n ), routes as materialized (\n select r.start_h3_r5,r.end_h3_r5,r.route_support,r.qualified_support,h.hour_support\n from route_totals r join hour_totals h using(start_h3_r5,end_h3_r5)\n where r.route_support>=2\n and r.qualified_support=r.route_support\n and h.hour_support>=1\n )\n select r.*,c.corridor_h3_r7,c.corridor_support\n from routes r\n join demand_corridors c using(start_h3_r5,end_h3_r5)\n where c.corridor_support>=1\n order by r.hour_support desc,r.route_support desc,c.corridor_support desc,\n r.start_h3_r5,r.end_h3_r5,c.corridor_h3_r7\n limit ?\n \"\"\", (rs, ignored) -> {\n long route = rs.getLong(\"route_support\");\n long qualified = rs.getLong(\"qualified_support\");\n long hourSupport = rs.getLong(\"hour_support\");\n long corridor = rs.getLong(\"corridor_support\");\n BigDecimal score = RouteRateScore.calculate(RouteRateScore.Variant.THINNED, route, qualified,\n hourSupport, corridor, exposures).score();\n String startCell = rs.getString(\"start_h3_r5\");\n String endCell = rs.getString(\"end_h3_r5\");\n String corridorCell = rs.getString(\"corridor_h3_r7\");\n String routeKey = startCell + \">\" + endCell;\n return new Candidate(startCell, endCell, corridorCell, routeKey, routeKey + \">\" + corridorCell,\n score, route, qualified, hourSupport, corridor, exposures);\n },\n properties.truthAuthorityFingerprint(),\n properties.authorityGeneration(),\n Timestamp.from(asOf),\n Timestamp.from(asOf),\n Timestamp.from(from),\n demandH3R5Csv,\n hourOfWeek,\n rowLimit);\n\n List ranked = new ArrayList<>(values);\n ranked.sort(ORDER);\n return List.copyOf(ranked);\n }\n\n private static Object[] queryArguments(\n int rowLimit,\n String demandH3R5Csv,\n Object... base\n ) {\n List values = new ArrayList<>(java.util.Arrays.asList(base));\n if (demandH3R5Csv != null) values.add(demandH3R5Csv);\n if (rowLimit > 0) values.add(rowLimit);\n return values.toArray();\n }\n\n record Candidate(String startH3R5, String endH3R5, String corridorH3R7, String routeKey,\n String candidateKey, BigDecimal score, long routeSupport, long qualifiedSupport,\n long hourSupport, long corridorSupport, int qualifiedWeekdayExposures) {}\n}\n" }, { "path": "backend/truth-model/src/main/java/hu/gombos/flightspotter/truthmodel/forecast/P862ShadowMetrics.java", "encoding": "UTF-8", "sha256": "e891997d8e3d2351a564ce9f38a13dca3458aad8e4d838449f2878610b2d979d", "content": "package hu.gombos.flightspotter.truthmodel.forecast;\n\nimport io.micrometer.core.instrument.Counter;\nimport io.micrometer.core.instrument.MeterRegistry;\nimport org.springframework.stereotype.Component;\n\nimport java.util.Map;\n\n@Component\nfinal class P862ShadowMetrics {\n private final Map counters;\n P862ShadowMetrics(MeterRegistry registry) {\n counters = Map.of(\n \"attempt\", counter(registry, \"shadow_invocation_attempt_total\"),\n \"created\", counter(registry, \"shadow_prediction_created_total\"),\n \"duplicate\", counter(registry, \"shadow_prediction_duplicate_total\"),\n \"missed\", counter(registry, \"shadow_invocation_missed_total\"),\n \"generation_failure\", counter(registry, \"shadow_candidate_generation_failure_total\"),\n \"joined\", counter(registry, \"shadow_outcome_join_total\"),\n \"unknown\", counter(registry, \"shadow_outcome_unknown_total\"),\n \"invalidated\", counter(registry, \"shadow_window_invalidated_total\"));\n }\n void increment(String name) { counters.get(name).increment(); }\n void increment(String name, long amount) { counters.get(name).increment(amount); }\n private static Counter counter(MeterRegistry registry, String name) {\n return Counter.builder(name).description(\"P8.6.2 prospective shadow evidence counter\").register(registry);\n }\n}\n" }, { "path": "backend/truth-model/src/main/java/hu/gombos/flightspotter/truthmodel/forecast/P862ShadowProperties.java", "encoding": "UTF-8", "sha256": "cdca947867a63ba2050f6078557444c99bf6894341386720f3be5ad42fbf196d", "content": "package hu.gombos.flightspotter.truthmodel.forecast;\n\nimport hu.gombos.flightspotter.forecastmodel.P862FrozenPipeline;\nimport hu.gombos.flightspotter.forecastmodel.P862RegionalHistoricalPipeline;\nimport hu.gombos.flightspotter.forecastmodel.RegionalHistoricalTruthAuthority;\nimport org.springframework.boot.context.properties.ConfigurationProperties;\nimport org.springframework.boot.context.properties.bind.ConstructorBinding;\n\nimport java.time.Duration;\nimport java.time.LocalDate;\nimport java.util.Map;\n\n@ConfigurationProperties(\"p862-shadow\")\npublic record P862ShadowProperties(\n boolean enabled,\n String windowId,\n LocalDate startDate,\n String pipelineFingerprint,\n Duration invocationLateness,\n Duration evidenceRetention,\n Duration coordinatorInterval,\n Duration outcomeJoinInterval,\n int catchupBatchSize,\n String truthAuthorityFingerprint,\n String authorityGeneration\n) {\n @ConstructorBinding\n public P862ShadowProperties {\n windowId = windowId == null ? \"\" : windowId.trim();\n pipelineFingerprint = pipelineFingerprint == null ? \"\" : pipelineFingerprint.trim();\n truthAuthorityFingerprint = truthAuthorityFingerprint == null ? \"\" : truthAuthorityFingerprint.trim();\n authorityGeneration = authorityGeneration == null || authorityGeneration.isBlank()\n ? RegionalHistoricalTruthAuthority.GENERATION : authorityGeneration.trim();\n startDate = startDate == null ? LocalDate.of(1970, 1, 1) : startDate;\n invocationLateness = invocationLateness == null ? Duration.ofMinutes(5) : invocationLateness;\n evidenceRetention = evidenceRetention == null ? P862FrozenPipeline.MIN_EVIDENCE_RETENTION : evidenceRetention;\n coordinatorInterval = coordinatorInterval == null ? Duration.ofMinutes(1) : coordinatorInterval;\n outcomeJoinInterval = outcomeJoinInterval == null ? Duration.ofMinutes(5) : outcomeJoinInterval;\n catchupBatchSize = catchupBatchSize == 0 ? 48 : catchupBatchSize;\n\n if (!authorityGeneration.equals(RegionalHistoricalTruthAuthority.GENERATION)) {\n throw new IllegalArgumentException(\n \"P862 supports only REGIONAL_HISTORICAL_TRUTH_AUTHORITY_V1\");\n }\n if (invocationLateness.isNegative() || invocationLateness.isZero()\n || invocationLateness.compareTo(Duration.ofMinutes(15)) > 0\n || evidenceRetention.compareTo(P862FrozenPipeline.MIN_EVIDENCE_RETENTION) < 0\n || evidenceRetention.compareTo(Duration.ofDays(400)) > 0\n || coordinatorInterval.isNegative() || coordinatorInterval.isZero()\n || outcomeJoinInterval.isNegative() || outcomeJoinInterval.isZero()\n || catchupBatchSize < 1 || catchupBatchSize > 672) {\n throw new IllegalArgumentException(\"Invalid bounded P862 configuration\");\n }\n if (enabled && !truthAuthorityFingerprint.matches(\"[0-9a-f]{64}\")) {\n throw new IllegalArgumentException(\"Enabled P862 requires a reviewed truth-authority fingerprint\");\n }\n }\n\n P862ShadowProperties(\n boolean enabled,\n String windowId,\n LocalDate startDate,\n String pipelineFingerprint,\n Duration invocationLateness,\n Duration evidenceRetention,\n Duration coordinatorInterval,\n Duration outcomeJoinInterval,\n int catchupBatchSize,\n String truthAuthorityFingerprint\n ) {\n this(enabled, windowId, startDate, pipelineFingerprint, invocationLateness, evidenceRetention,\n coordinatorInterval, outcomeJoinInterval, catchupBatchSize, truthAuthorityFingerprint,\n RegionalHistoricalTruthAuthority.GENERATION);\n }\n\n public LocalDate endDateExclusive() {\n return startDate.plusDays(P862FrozenPipeline.WINDOW_DAYS);\n }\n\n public String experimentId() {\n return P862RegionalHistoricalPipeline.EXPERIMENT_ID;\n }\n\n public Map fingerprints() {\n return P862RegionalHistoricalPipeline.fingerprints();\n }\n\n public String expectedPipelineFingerprint() {\n return P862RegionalHistoricalPipeline.combinedFingerprint();\n }\n\n public String candidateGeneratorVersion() {\n return P862RegionalHistoricalPipeline.CANDIDATE_GENERATOR_VERSION;\n }\n\n public String featureSourceVersion() {\n return P862RegionalHistoricalPipeline.FEATURE_SOURCE_VERSION;\n }\n\n public boolean fingerprintMatches() {\n return pipelineFingerprint.equals(expectedPipelineFingerprint());\n }\n}\n" }, { "path": "backend/truth-model/src/main/java/hu/gombos/flightspotter/truthmodel/forecast/P862ShadowRepository.java", "encoding": "UTF-8", "sha256": "979079d2d85f7ead6a95c58a01c20d7d9b16b13db4d3f3760e4ee4f468fd71b0", "content": "package hu.gombos.flightspotter.truthmodel.forecast;\n\nimport hu.gombos.flightspotter.forecastmodel.P861DailySelectionPolicy;\nimport hu.gombos.flightspotter.forecastmodel.P862FrozenPipeline;\nimport org.springframework.jdbc.core.JdbcTemplate;\nimport org.springframework.stereotype.Repository;\nimport org.springframework.transaction.support.TransactionTemplate;\n\nimport java.nio.charset.StandardCharsets;\nimport java.security.MessageDigest;\nimport java.sql.Timestamp;\nimport java.time.Instant;\nimport java.time.LocalDate;\nimport java.time.ZoneOffset;\nimport java.util.ArrayList;\nimport java.util.HexFormat;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.UUID;\n\n@Repository\npublic class P862ShadowRepository {\n enum HourResult { FORECASTED, GENERATION_FAILURE, CREATED_MISSED, DUPLICATE }\n private final JdbcTemplate jdbc;\n private final TransactionTemplate transactions;\n private final P862RegionalFeatureSource features;\n private final P862ShadowMetrics metrics;\n\n public P862ShadowRepository(JdbcTemplate jdbc, TransactionTemplate transactions, P862RegionalFeatureSource features,\n P862ShadowMetrics metrics) {\n this.jdbc = jdbc; this.transactions = transactions; this.features = features; this.metrics = metrics;\n }\n\n void ensureWindow(P862ShadowProperties p) {\n Map f = p.fingerprints();\n int inserted = jdbc.update(\"\"\"\n insert into p862_prospective_windows(window_id,experiment_id,authority_generation,started_at,planned_end_at,\n pipeline_fingerprint,truth_authority_fingerprint,thinned_fingerprint,candidate_generator_fingerprint,\n hourly_selector_fingerprint,daily_selector_fingerprint,daily_decision_fingerprint,\n feature_source_fingerprint,outcome_maturity_fingerprint,metrics_fingerprint,\n selection_contract,metrics_contract,status)\n values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?, 'PLANNED') on conflict(window_id) do nothing\n \"\"\", p.windowId(), p.experimentId(), p.authorityGeneration(),\n Timestamp.from(p.startDate().atStartOfDay(ZoneOffset.UTC).toInstant()),\n Timestamp.from(p.endDateExclusive().atStartOfDay(ZoneOffset.UTC).toInstant()), p.pipelineFingerprint(),\n p.truthAuthorityFingerprint(), f.get(\"thinned_scorer\"), f.get(\"candidate_generator\"), f.get(\"hourly_top2\"), f.get(\"daily_selector\"),\n f.get(\"daily_decision\"), f.get(\"causal_feature\"), f.get(\"outcome_maturity\"), f.get(\"prospective_metrics\"),\n \"24 exact-T24 hourly snapshots -> end-of-day portfolio route round robin -> 5/day\",\n \"precision,wilson95,halves,rolling5,daily,coverage,concentration,hourly-top2-top3;activation=false\");\n Map stored = jdbc.queryForMap(\"\"\"\n select experiment_id,authority_generation,started_at,planned_end_at,pipeline_fingerprint,\n truth_authority_fingerprint,thinned_fingerprint,candidate_generator_fingerprint,\n hourly_selector_fingerprint,daily_selector_fingerprint,daily_decision_fingerprint,\n feature_source_fingerprint,outcome_maturity_fingerprint,metrics_fingerprint\n from p862_prospective_windows where window_id=?\n \"\"\", p.windowId());\n Instant expectedStart = p.startDate().atStartOfDay(ZoneOffset.UTC).toInstant();\n Instant expectedEnd = p.endDateExclusive().atStartOfDay(ZoneOffset.UTC).toInstant();\n boolean mismatch = !((String) stored.get(\"experiment_id\")).trim().equals(p.experimentId())\n || !((String) stored.get(\"authority_generation\")).trim().equals(p.authorityGeneration())\n || !((String) stored.get(\"pipeline_fingerprint\")).trim().equals(p.expectedPipelineFingerprint())\n || !((String) stored.get(\"truth_authority_fingerprint\")).trim().equals(p.truthAuthorityFingerprint())\n || !((Timestamp) stored.get(\"started_at\")).toInstant().equals(expectedStart)\n || !((Timestamp) stored.get(\"planned_end_at\")).toInstant().equals(expectedEnd)\n || fingerprintMismatch(stored, \"thinned_fingerprint\", f.get(\"thinned_scorer\"))\n || fingerprintMismatch(stored, \"candidate_generator_fingerprint\", f.get(\"candidate_generator\"))\n || fingerprintMismatch(stored, \"hourly_selector_fingerprint\", f.get(\"hourly_top2\"))\n || fingerprintMismatch(stored, \"daily_selector_fingerprint\", f.get(\"daily_selector\"))\n || fingerprintMismatch(stored, \"daily_decision_fingerprint\", f.get(\"daily_decision\"))\n || fingerprintMismatch(stored, \"feature_source_fingerprint\", f.get(\"causal_feature\"))\n || fingerprintMismatch(stored, \"outcome_maturity_fingerprint\", f.get(\"outcome_maturity\"))\n || fingerprintMismatch(stored, \"metrics_fingerprint\", f.get(\"prospective_metrics\"));\n if (mismatch) {\n int invalidated = jdbc.update(\"\"\"\n update p862_prospective_windows set status='INVALIDATED',invalidation_reason='FROZEN_WINDOW_CONTRACT_MISMATCH'\n where window_id=? and status in ('PLANNED','ACTIVE')\n \"\"\", p.windowId());\n if (invalidated > 0) metrics.increment(\"invalidated\");\n throw new IllegalStateException(\"P862 window fingerprint mismatch\");\n }\n }\n\n WarmupReadiness warmupReadiness(P862ShadowProperties p) {\n LocalDate from = p.startDate().minusDays(P862FrozenPipeline.WARMUP_COMPLETE_DAYS + 2L);\n LocalDate toExclusive = p.startDate().minusDays(2);\n Instant firstAsOf = p.startDate().minusDays(1).atStartOfDay(ZoneOffset.UTC).toInstant();\n\n Integer authoritativeDays = jdbc.queryForObject(\"\"\"\n select count(*)\n from p862_effective_truth_day_seals d\n where d.target_day>=? and d.target_day=? and d.target_day 256) {\n throw new IllegalArgumentException(\"P862 invalidation reason must be bounded\");\n }\n int invalidated = jdbc.update(\"\"\"\n update p862_prospective_windows set status='INVALIDATED',invalidation_reason=?\n where window_id=? and status in ('PLANNED','ACTIVE')\n \"\"\", reason, p.windowId());\n if (invalidated > 0) metrics.increment(\"invalidated\");\n }\n\n HourResult generateHour(P862ShadowProperties p, Instant target, Instant generatedAt) {\n requireTargetInWindow(p, target);\n Instant asOfBoundary = P862FrozenPipeline.asOf(target);\n Integer alreadyPresent = jdbc.queryForObject(\n \"select count(*) from p862_shadow_invocations where window_id=? and target_hour=?\", Integer.class,\n p.windowId(), Timestamp.from(target));\n if (alreadyPresent != null && alreadyPresent > 0) return HourResult.DUPLICATE;\n if (generatedAt == null || generatedAt.isBefore(asOfBoundary)\n || generatedAt.isAfter(asOfBoundary.plus(p.invocationLateness()))) {\n throw new IllegalStateException(\"P862 forecast generation is outside exact causal invocation window\");\n }\n List loaded = List.of();\n RuntimeException generationFailure = null;\n try { loaded = features.load(asOfBoundary, target); }\n catch (RuntimeException failure) { generationFailure = failure; }\n List candidates = loaded;\n RuntimeException capturedFailure = generationFailure;\n return transactions.execute(status -> {\n jdbc.query(\"select pg_advisory_xact_lock(hashtextextended(?,0))\", rs -> null,\n \"p862:\" + p.windowId() + ':' + target);\n Integer existing = jdbc.queryForObject(\"select count(*) from p862_shadow_invocations where window_id=? and target_hour=?\",\n Integer.class, p.windowId(), Timestamp.from(target));\n if (existing != null && existing > 0) return HourResult.DUPLICATE;\n requireActiveFingerprint(p);\n Instant asOf = P862FrozenPipeline.asOf(target);\n if (capturedFailure != null) {\n insertInvocation(p, target, generatedAt, \"CANDIDATE_GENERATION_FAILURE\", 0, safe(capturedFailure));\n return HourResult.GENERATION_FAILURE;\n }\n List topTwo = candidates.subList(0,\n Math.min(P862FrozenPipeline.HOURLY_CAP, candidates.size()));\n insertInvocation(p, target, generatedAt, \"FORECASTED\", topTwo.size(), null);\n Map fingerprints = p.fingerprints();\n for (int i = 0; i < candidates.size(); i++) insertDiagnostic(p, target, asOf, generatedAt,\n candidates.get(i), i + 1);\n for (int i = 0; i < topTwo.size(); i++) insertCandidate(p, target, asOf, generatedAt,\n topTwo.get(i), i + 1, fingerprints);\n return HourResult.FORECASTED;\n });\n }\n\n HourResult markMissed(P862ShadowProperties p, Instant target, Instant generatedAt) {\n requireTargetInWindow(p, target);\n Instant deadline = P862FrozenPipeline.asOf(target).plus(p.invocationLateness());\n return transactions.execute(status -> {\n jdbc.query(\"select pg_advisory_xact_lock(hashtextextended(?,0))\", rs -> null,\n \"p862:\" + p.windowId() + ':' + target);\n Integer existing = jdbc.queryForObject(\"select count(*) from p862_shadow_invocations where window_id=? and target_hour=?\",\n Integer.class, p.windowId(), Timestamp.from(target));\n if (existing != null && existing > 0) return HourResult.DUPLICATE;\n if (generatedAt == null || !generatedAt.isAfter(deadline)) {\n throw new IllegalStateException(\"P862 cannot mark an invocation missed before its exact deadline\");\n }\n requireActiveFingerprint(p);\n insertInvocation(p, target, generatedAt, \"MISSED_CAUSAL_INVOCATION\", 0,\n \"Exact as-of deadline elapsed; reconstruction prohibited\");\n return HourResult.CREATED_MISSED;\n });\n }\n\n List missingPastTargets(P862ShadowProperties p, Instant currentAsOf) {\n Instant first = p.startDate().atStartOfDay(ZoneOffset.UTC).toInstant();\n Instant windowLast = p.endDateExclusive().atStartOfDay(ZoneOffset.UTC).toInstant().minusSeconds(3600);\n Instant lastTarget = currentAsOf.plusSeconds(24 * 3600L).minusSeconds(3600);\n if (lastTarget.isAfter(windowLast)) lastTarget = windowLast;\n if (lastTarget.isBefore(first)) return List.of();\n return jdbc.query(\"\"\"\n select expected from generate_series(?::timestamptz,?::timestamptz,interval '1 hour') expected\n left join p862_shadow_invocations i on i.window_id=? and i.target_hour=expected\n where i.invocation_id is null order by expected limit ?\n \"\"\", (rs, ignored) -> rs.getTimestamp(1).toInstant(), Timestamp.from(first), Timestamp.from(lastTarget),\n p.windowId(), p.catchupBatchSize());\n }\n\n List readyUnfinalizedDays(P862ShadowProperties p, int limit) {\n if (limit < 1 || limit > 28) throw new IllegalArgumentException(\"P862 invalid daily-finalization batch\");\n return jdbc.query(\"\"\"\n select d::date\n from generate_series(?::date,(?::date-1),interval '1 day') d\n where not exists (\n select 1 from p862_daily_selections s where s.window_id=? and s.target_day=d::date)\n and 24=(select count(*) from p862_shadow_invocations i\n where i.window_id=? and i.target_hour>=d\n and i.target_hour rs.getObject(1, LocalDate.class), p.startDate(), p.endDateExclusive(),\n p.windowId(), p.windowId(), limit);\n }\n\n int finalizeDay(P862ShadowProperties p, LocalDate targetDay, Instant generatedAt) {\n return transactions.execute(status -> {\n jdbc.query(\"select pg_advisory_xact_lock(hashtextextended(?,0))\", rs -> null,\n \"p862-day:\" + p.windowId() + ':' + targetDay);\n Integer existing = jdbc.queryForObject(\"\"\"\n select count(*) from p862_daily_selections where window_id=? and target_day=?\n \"\"\", Integer.class, p.windowId(), targetDay);\n if (existing != null && existing > 0) return 0;\n Integer terminal = jdbc.queryForObject(\"\"\"\n select count(*) from p862_shadow_invocations where window_id=?\n and target_hour>=? and target_hour rows = readDayCandidates(p.windowId(), targetDay);\n List selectorInput = rows.stream().map(CandidateRow::selector).toList();\n List selected = new P861DailySelectionPolicy(\n P861DailySelectionPolicy.Family.ROUTE_ROUND_ROBIN).selectDay(selectorInput);\n for (int rank = 0; rank < selected.size(); rank++) {\n P861DailySelectionPolicy.Candidate value = selected.get(rank);\n CandidateRow row = rows.stream().filter(r -> r.candidateId().toString().equals(value.identity())).findFirst().orElseThrow();\n insertPrediction(p, row, generatedAt, rank + 1);\n }\n jdbc.update(\"\"\"\n insert into p862_daily_selections(window_id,target_day,generated_at,hourly_candidate_count,\n selected_count,daily_selector_fingerprint) values (?,?,?,?,?,?)\n \"\"\", p.windowId(), targetDay, Timestamp.from(generatedAt), rows.size(), selected.size(),\n p.fingerprints().get(\"daily_selector\"));\n return selected.size();\n });\n }\n\n void completeWindowIfMature(P862ShadowProperties p, Instant now) {\n Instant maturity = p.endDateExclusive().plusDays(1).atStartOfDay(ZoneOffset.UTC).toInstant();\n if (now.isBefore(maturity)) return;\n Integer invocations = jdbc.queryForObject(\n \"select count(*) from p862_shadow_invocations where window_id=?\", Integer.class, p.windowId());\n Integer dailySelections = jdbc.queryForObject(\n \"select count(*) from p862_daily_selections where window_id=?\", Integer.class, p.windowId());\n Integer matureDays = jdbc.queryForObject(\"\"\"\n select count(*) from p862_effective_truth_day_seals\n where target_day>=? and target_day readDayCandidates(String windowId, LocalDate day) {\n return jdbc.query(\"\"\"\n select candidate_id,target_hour,as_of,generated_at,candidate_key,route_key,thinned_score,hour_support,\n corridor_support,hourly_rank from p862_hourly_candidates where window_id=?\n and target_hour>=? and target_hour new CandidateRow(rs.getObject(\"candidate_id\", UUID.class),\n rs.getTimestamp(\"target_hour\").toInstant(), rs.getTimestamp(\"as_of\").toInstant(),\n rs.getTimestamp(\"generated_at\").toInstant(), rs.getString(\"candidate_key\"), rs.getString(\"route_key\"),\n rs.getBigDecimal(\"thinned_score\"), rs.getLong(\"hour_support\"), rs.getLong(\"corridor_support\"),\n rs.getInt(\"hourly_rank\")), windowId, Timestamp.from(day.atStartOfDay(ZoneOffset.UTC).toInstant()),\n Timestamp.from(day.plusDays(1).atStartOfDay(ZoneOffset.UTC).toInstant()));\n }\n\n private void insertInvocation(P862ShadowProperties p, Instant target, Instant generatedAt, String state,\n int count, String failure) {\n String canonical = p.windowId() + '|' + target + '|' + state;\n jdbc.update(\"\"\"\n insert into p862_shadow_invocations(invocation_id,window_id,target_hour,as_of,generated_at,status,\n candidate_count,failure_reason,pipeline_fingerprint,truth_authority_fingerprint) values (?,?,?,?,?,?,?,?,?,?)\n \"\"\", uuid(canonical), p.windowId(), Timestamp.from(target), Timestamp.from(P862FrozenPipeline.asOf(target)),\n Timestamp.from(generatedAt), state, count, failure, p.pipelineFingerprint(), p.truthAuthorityFingerprint());\n jdbc.update(\"update p862_prospective_windows set status='ACTIVE' where window_id=? and status='PLANNED'\",\n p.windowId());\n }\n\n private void insertCandidate(P862ShadowProperties p, Instant target, Instant asOf, Instant generatedAt,\n P862RegionalFeatureSource.Candidate c, int rank, Map f) {\n String canonical = p.windowId() + '|' + target + '|' + c.candidateKey() + '|' + c.score() + '|' + rank;\n jdbc.update(\"\"\"\n insert into p862_hourly_candidates(candidate_id,window_id,target_hour,as_of,generated_at,start_h3_r5,\n end_h3_r5,corridor_h3_r7,route_key,candidate_key,hourly_rank,thinned_score,route_support,\n qualified_support,hour_support,corridor_support,qualified_weekday_exposures,scorer_version,\n scorer_fingerprint,candidate_generator_version,candidate_generator_fingerprint,hourly_selector_version,\n hourly_selector_fingerprint,feature_source_version,feature_source_fingerprint,snapshot_sha256)\n values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)\n \"\"\", uuid(canonical), p.windowId(), Timestamp.from(target), Timestamp.from(asOf), Timestamp.from(generatedAt),\n c.startH3R5(), c.endH3R5(), c.corridorH3R7(), c.routeKey(), c.candidateKey(), rank, c.score(),\n c.routeSupport(), c.qualifiedSupport(), c.hourSupport(), c.corridorSupport(), c.qualifiedWeekdayExposures(),\n RouteRateScoreVersion.VERSION, f.get(\"thinned_scorer\"), p.candidateGeneratorVersion(),\n f.get(\"candidate_generator\"), P862FrozenPipeline.HOURLY_SELECTOR_VERSION, f.get(\"hourly_top2\"),\n p.featureSourceVersion(), f.get(\"causal_feature\"), sha(canonical));\n }\n\n private void insertDiagnostic(P862ShadowProperties p, Instant target, Instant asOf, Instant generatedAt,\n P862RegionalFeatureSource.Candidate c, int rank) {\n String canonical = \"diagnostic|\" + p.windowId() + '|' + target + '|' + c.candidateKey() + '|' + c.score() + '|' + rank;\n jdbc.update(\"\"\"\n insert into p862_hourly_diagnostics(diagnostic_id,window_id,target_hour,as_of,generated_at,start_h3_r5,\n end_h3_r5,corridor_h3_r7,route_key,candidate_key,hourly_rank,thinned_score,route_support,\n qualified_support,hour_support,corridor_support,snapshot_sha256)\n values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)\n \"\"\", uuid(canonical), p.windowId(), Timestamp.from(target), Timestamp.from(asOf), Timestamp.from(generatedAt),\n c.startH3R5(), c.endH3R5(), c.corridorH3R7(), c.routeKey(), c.candidateKey(), rank, c.score(),\n c.routeSupport(), c.qualifiedSupport(), c.hourSupport(), c.corridorSupport(), sha(canonical));\n }\n\n private void insertPrediction(P862ShadowProperties p, CandidateRow row, Instant generatedAt, int rank) {\n String canonical = p.windowId() + '|' + row.candidateId() + '|' + rank + '|' + generatedAt;\n jdbc.update(\"\"\"\n insert into p862_shadow_predictions(prediction_id,candidate_id,window_id,target_hour,as_of,generated_at,\n daily_rank,daily_selector_version,daily_selector_fingerprint,experiment_id,prediction_sha256)\n values (?,?,?,?,?,?,?,?,?,?,?)\n \"\"\", uuid(canonical), row.candidateId(), p.windowId(), Timestamp.from(row.target()), Timestamp.from(row.asOf()),\n Timestamp.from(generatedAt), rank, P861DailySelectionPolicy.VERSION,\n p.fingerprints().get(\"daily_selector\"), p.experimentId(), sha(canonical));\n }\n\n private void requireActiveFingerprint(P862ShadowProperties p) {\n String state = jdbc.queryForObject(\"select status||':'||pipeline_fingerprint from p862_prospective_windows where window_id=?\",\n String.class, p.windowId());\n if (state == null || state.startsWith(\"INVALIDATED:\") || state.startsWith(\"COMPLETE:\")\n || !state.endsWith(':' + p.expectedPipelineFingerprint())) {\n throw new IllegalStateException(\"P862 window is terminal or fingerprint mismatched\");\n }\n }\n\n private static boolean fingerprintMismatch(Map stored, String column, String expected) {\n Object value = stored.get(column);\n return value == null || !value.toString().trim().equals(expected);\n }\n\n private static void requireTargetInWindow(P862ShadowProperties p, Instant target) {\n P862FrozenPipeline.requireExactT24(target, P862FrozenPipeline.asOf(target));\n LocalDate day = target.atZone(ZoneOffset.UTC).toLocalDate();\n if (day.isBefore(p.startDate()) || !day.isBefore(p.endDateExclusive())) {\n throw new IllegalArgumentException(\"P862 target is outside frozen prospective window\");\n }\n }\n\n private static String safe(RuntimeException failure) {\n String value = failure.getClass().getSimpleName() + \":\" + String.valueOf(failure.getMessage());\n return value.length() <= 256 ? value : value.substring(0, 256);\n }\n private static UUID uuid(String canonical) { return UUID.nameUUIDFromBytes(canonical.getBytes(StandardCharsets.UTF_8)); }\n private static String sha(String canonical) {\n try { return HexFormat.of().formatHex(MessageDigest.getInstance(\"SHA-256\").digest(canonical.getBytes(StandardCharsets.UTF_8))); }\n catch (Exception failure) { throw new IllegalStateException(failure); }\n }\n private static final class RouteRateScoreVersion { static final String VERSION = \"p853-route-thinned-rate-v1\"; }\n\n record WarmupReadiness(int authoritativeDays, int schemaV2Days, long expiringRows) {\n boolean ready() {\n return authoritativeDays == P862FrozenPipeline.WARMUP_COMPLETE_DAYS\n && schemaV2Days == P862FrozenPipeline.WARMUP_COMPLETE_DAYS && expiringRows == 0;\n }\n\n String failureReason() {\n if (ready()) return \"\";\n return \"WARMUP_NOT_READY_AT_FIRST_ASOF:authoritativeDays=\" + authoritativeDays\n + \",schemaV2Days=\" + schemaV2Days + \",expiringRows=\" + expiringRows;\n }\n }\n\n private record CandidateRow(UUID candidateId, Instant target, Instant asOf, Instant generatedAt, String candidateKey,\n String routeKey, java.math.BigDecimal score, long hourSupport, long corridorSupport,\n int hourlyRank) {\n P861DailySelectionPolicy.Candidate selector() {\n return new P861DailySelectionPolicy.Candidate(candidateId.toString(), candidateKey, routeKey, target,\n asOf, score, hourSupport, corridorSupport, hourlyRank);\n }\n }\n}\n" }, { "path": "backend/truth-model/src/main/java/hu/gombos/flightspotter/truthmodel/forecast/P862TruthJoinService.java", "encoding": "UTF-8", "sha256": "6c47fc85a586211ba8e9be40a9225dd35da26b692a83900f01f4c64399b2a671", "content": "package hu.gombos.flightspotter.truthmodel.forecast;\n\nimport org.springframework.jdbc.core.JdbcTemplate;\nimport org.springframework.stereotype.Service;\nimport org.springframework.transaction.support.TransactionTemplate;\n\nimport java.nio.charset.StandardCharsets;\nimport java.security.MessageDigest;\nimport java.sql.Timestamp;\nimport java.time.Instant;\nimport java.util.HexFormat;\nimport java.util.List;\nimport java.util.UUID;\n\n@Service\nfinal class P862TruthJoinService {\n private final JdbcTemplate jdbc;\n private final TransactionTemplate transactions;\n private final P862ShadowProperties properties;\n private final P862ShadowMetrics metrics;\n\n P862TruthJoinService(JdbcTemplate jdbc, TransactionTemplate transactions, P862ShadowProperties properties,\n P862ShadowMetrics metrics) {\n this.jdbc = jdbc; this.transactions = transactions; this.properties = properties; this.metrics = metrics;\n }\n\n /**\n * P862 is a consumer of authoritative day completeness, not its producer.\n *\n * A process-local/global enable flag cannot prove that a UTC day had complete ingest coverage.\n * Day seals must therefore be written by a separately reviewed coverage authority. Until such a\n * seal exists the outcome remains unmatured and prospective evidence is incomplete.\n */\n void sealMatureDays(Instant now) {\n if (now == null) throw new IllegalArgumentException(\"P862 truth join time is required\");\n // Intentionally no-op. Never manufacture AUTHORITATIVE_COMPLETE from configuration alone.\n }\n\n int joinBatch(Instant now) {\n List pending = jdbc.query(\"\"\"\n select p.prediction_id,p.prediction_sha256,p.target_hour,c.route_key,c.corridor_h3_r7,\n d.status,d.authoritative_ready_at,d.authority_generation,d.historical_dataset_id\n from p862_shadow_predictions p join p862_hourly_candidates c using(candidate_id)\n join p862_effective_truth_day_seals d\n on d.target_day=(p.target_hour at time zone 'UTC')::date\n left join p862_shadow_outcomes o on o.prediction_id=p.prediction_id\n where p.window_id=? and o.prediction_id is null and d.source_fingerprint=?\n and d.authority_generation=? and d.authoritative_ready_at<=?\n order by p.target_hour,p.daily_rank limit 200\n \"\"\", (rs, ignored) -> pending(rs), properties.windowId(),\n properties.truthAuthorityFingerprint(), properties.authorityGeneration(), Timestamp.from(now));\n int joined = 0;\n for (Pending value : pending) if (append(value)) joined++;\n\n List diagnostics = jdbc.query(\"\"\"\n select h.diagnostic_id,h.snapshot_sha256,h.target_hour,h.route_key,h.corridor_h3_r7,\n d.status,d.authoritative_ready_at,d.authority_generation,d.historical_dataset_id\n from p862_hourly_diagnostics h\n join p862_effective_truth_day_seals d\n on d.target_day=(h.target_hour at time zone 'UTC')::date\n left join p862_hourly_diagnostic_outcomes o using(diagnostic_id)\n where h.window_id=? and o.diagnostic_id is null and d.source_fingerprint=?\n and d.authority_generation=? and d.authoritative_ready_at<=?\n order by h.target_hour,h.hourly_rank limit 600\n \"\"\", (rs, ignored) -> pending(rs), properties.windowId(),\n properties.truthAuthorityFingerprint(), properties.authorityGeneration(), Timestamp.from(now));\n for (Pending value : diagnostics) if (appendDiagnostic(value)) joined++;\n return joined;\n }\n\n private Pending pending(java.sql.ResultSet rs) throws java.sql.SQLException {\n return new Pending(\n rs.getObject(1, UUID.class),\n rs.getString(2).trim(),\n rs.getTimestamp(3).toInstant(),\n rs.getString(4),\n rs.getString(5),\n rs.getString(6),\n rs.getTimestamp(7).toInstant(),\n rs.getString(8),\n rs.getObject(9, UUID.class));\n }\n\n private boolean append(Pending p) {\n Boolean created = transactions.execute(status -> {\n jdbc.query(\"select pg_advisory_xact_lock(hashtextextended(?,0))\", rs -> null, \"p862-outcome:\" + p.id());\n Integer existing = jdbc.queryForObject(\"select count(*) from p862_shadow_outcomes where prediction_id=?\",\n Integer.class, p.id());\n if (existing != null && existing > 0) return false;\n Classification classification = classify(p);\n String outcome = classification.outcome(), reason = classification.reason();\n String canonical = p.id() + \"|\" + p.snapshot() + \"|\" + outcome + \"|\" + p.readyAt();\n jdbc.update(\"\"\"\n insert into p862_shadow_outcomes(outcome_id,prediction_id,prediction_sha256,outcome,matched_at,\n reason,outcome_sha256) values (?,?,?,?,?,?,?)\n \"\"\", uuid(\"outcome:\" + p.id()), p.id(), p.snapshot(), outcome, Timestamp.from(p.readyAt()), reason,\n sha(canonical));\n return true;\n });\n if (Boolean.TRUE.equals(created)) {\n metrics.increment(\"joined\");\n if (p.sealStatus().equals(\"UNKNOWN_INCOMPLETE_TRUTH\")) metrics.increment(\"unknown\");\n return true;\n }\n return false;\n }\n\n private boolean appendDiagnostic(Pending p) {\n Boolean created = transactions.execute(status -> {\n jdbc.query(\"select pg_advisory_xact_lock(hashtextextended(?,0))\", rs -> null,\n \"p862-diagnostic-outcome:\" + p.id());\n Integer existing = jdbc.queryForObject(\"select count(*) from p862_hourly_diagnostic_outcomes where diagnostic_id=?\",\n Integer.class, p.id());\n if (existing != null && existing > 0) return false;\n Classification classification = classify(p);\n String canonical = p.id() + \"|\" + p.snapshot() + \"|\" + classification.outcome() + \"|\" + p.readyAt();\n jdbc.update(\"\"\"\n insert into p862_hourly_diagnostic_outcomes(outcome_id,diagnostic_id,diagnostic_sha256,outcome,\n matched_at,reason,outcome_sha256) values (?,?,?,?,?,?,?)\n \"\"\", uuid(\"diagnostic-outcome:\" + p.id()), p.id(), p.snapshot(), classification.outcome(),\n Timestamp.from(p.readyAt()), classification.reason(), sha(canonical));\n return true;\n });\n if (Boolean.TRUE.equals(created)) {\n metrics.increment(\"joined\");\n if (p.sealStatus().equals(\"UNKNOWN_INCOMPLETE_TRUTH\")) metrics.increment(\"unknown\");\n return true;\n }\n return false;\n }\n\n private Classification classify(Pending p) {\n if (p.sealStatus().equals(\"UNKNOWN_INCOMPLETE_TRUTH\")) {\n return new Classification(\"UNKNOWN_INCOMPLETE_TRUTH\",\n \"Authoritative day seal reports incomplete regional truth\");\n }\n return classifyRegionalHistorical(p);\n }\n\n private Classification classifyRegionalHistorical(Pending p) {\n int utcHour = p.target().atZone(java.time.ZoneOffset.UTC).getHour();\n if (utcHour == 0 || utcHour == 23) {\n return new Classification(\"INELIGIBLE_OVERFLOW\",\n \"Regional one-day evidence excludes UTC boundary hours because cross-day carry is cut\");\n }\n if (p.historicalDatasetId() == null) {\n return new Classification(\"UNKNOWN_INCOMPLETE_TRUTH\",\n \"Regional historical authority seal is missing its immutable source dataset\");\n }\n String[] route = p.routeKey().split(\">\", -1);\n if (route.length != 2) {\n return new Classification(\"INELIGIBLE_OVERFLOW\", \"Invalid regional route identity\");\n }\n short hourOfWeek = (short) ((p.target().atZone(java.time.ZoneOffset.UTC).getDayOfWeek().getValue() - 1) * 24\n + p.target().atZone(java.time.ZoneOffset.UTC).getHour());\n\n List rows = jdbc.query(\"\"\"\n select pattern.evidence_count,pattern.qualified_evidence_count,\n coalesce(hour_bin.evidence_count,0) as hour_support,\n coalesce(corridor.traversal_count,0) as corridor_support\n from historical_bootstrap_ready_pattern_aggregates pattern\n left join historical_bootstrap_ready_hour_bins hour_bin\n on hour_bin.pattern_id=pattern.pattern_id and hour_bin.hour_of_week=?\n left join historical_bootstrap_ready_route_corridor_aggregates corridor\n on corridor.dataset_id=pattern.dataset_id\n and corridor.derivation_version=pattern.derivation_version\n and corridor.start_h3_r5=pattern.start_h3_r5\n and corridor.end_h3_r5=pattern.end_h3_r5\n and corridor.corridor_h3_r7=?\n where pattern.dataset_id=?\n and pattern.pattern_kind='ROUTE' and pattern.subject_key='-'\n and pattern.start_h3_r5=? and pattern.end_h3_r5=?\n \"\"\", (rs, ignored) -> new RegionalTruth(\n rs.getLong(\"evidence_count\"),\n rs.getLong(\"qualified_evidence_count\"),\n rs.getLong(\"hour_support\"),\n rs.getLong(\"corridor_support\")),\n hourOfWeek, p.cell(), p.historicalDatasetId(), route[0], route[1]);\n\n if (rows.isEmpty()) {\n return new Classification(\"NEGATIVE\",\n \"Qualified regional historical day contains no matching route\");\n }\n if (rows.size() != 1) {\n return new Classification(\"INELIGIBLE_OVERFLOW\",\n \"Regional historical route aggregate identity is ambiguous\");\n }\n RegionalTruth truth = rows.getFirst();\n if (truth.routeSupport() < 1L\n || truth.qualifiedSupport() != truth.routeSupport()\n || truth.hourSupport() < 0L || truth.hourSupport() > truth.routeSupport()\n || truth.corridorSupport() < 0L || truth.corridorSupport() > truth.routeSupport()) {\n return new Classification(\"INELIGIBLE_OVERFLOW\",\n \"Regional historical aggregate is not clean enough for deterministic truth\");\n }\n if (truth.hourSupport() == 0L) {\n return new Classification(\"NEGATIVE\",\n \"Qualified regional historical day has no matching route in target UTC hour\");\n }\n if (truth.corridorSupport() == 0L) {\n // Corridor aggregate absence cannot prove a negative because bounded corridor overflow\n // intentionally drops route-shape cells while preserving the coarse route aggregate.\n return new Classification(\"INELIGIBLE_OVERFLOW\",\n \"Matching route/hour exists but corridor absence may be bounded-overflow ambiguity\");\n }\n if (truth.routeSupport() == 1L\n || truth.hourSupport() == truth.routeSupport()\n || truth.corridorSupport() == truth.routeSupport()) {\n return new Classification(\"POSITIVE\",\n \"Qualified regional historical aggregate deterministically proves route/hour/corridor match\");\n }\n return new Classification(\"INELIGIBLE_OVERFLOW\",\n \"Multiple regional route traversals make hour-to-corridor pairing ambiguous\");\n }\n\n private static UUID uuid(String value) { return UUID.nameUUIDFromBytes(value.getBytes(StandardCharsets.UTF_8)); }\n private static String sha(String value) {\n try { return HexFormat.of().formatHex(MessageDigest.getInstance(\"SHA-256\").digest(value.getBytes(StandardCharsets.UTF_8))); }\n catch (Exception failure) { throw new IllegalStateException(failure); }\n }\n private record Pending(\n UUID id,\n String snapshot,\n Instant target,\n String routeKey,\n String cell,\n String sealStatus,\n Instant readyAt,\n String authorityGeneration,\n UUID historicalDatasetId\n ) {}\n private record RegionalTruth(long routeSupport, long qualifiedSupport, long hourSupport, long corridorSupport) {}\n private record Classification(String outcome, String reason) {}\n}\n" }, { "path": "backend/truth-model/src/main/resources/db/migration/V102__historical_corridor_packs.sql", "encoding": "UTF-8", "sha256": "88457f4628e74f6bb982ac67975c84289fdd30c7f881b728763f5a89fdf0313d", "content": "-- Compact immutable route-corridor persistence for regional historical evidence.\n-- Existing V71 row-per-R7 evidence remains readable during migration; new loaders write bounded\n-- route x R5-parent packs. The READY compatibility view preserves the exact legacy row contract.\n\nCREATE TABLE historical_bootstrap_route_corridor_packs (\n dataset_id UUID NOT NULL REFERENCES historical_bootstrap_datasets(dataset_id) ON DELETE CASCADE,\n start_h3_r5 VARCHAR(15) NOT NULL CHECK (start_h3_r5 ~ '^85[0-9a-f]{13}$'),\n end_h3_r5 VARCHAR(15) NOT NULL CHECK (end_h3_r5 ~ '^85[0-9a-f]{13}$'),\n corridor_h3_r5 VARCHAR(15) NOT NULL CHECK (corridor_h3_r5 ~ '^85[0-9a-f]{13}$'),\n corridor_h3_r7 VARCHAR(15)[] NOT NULL,\n traversal_count BIGINT[] NOT NULL,\n progress_permille_sum BIGINT[] NOT NULL,\n minimum_altitude_ft INTEGER[] NOT NULL,\n maximum_altitude_ft INTEGER[] NOT NULL,\n altitude_sum_ft BIGINT[] NOT NULL,\n altitude_evidence_count BIGINT[] NOT NULL,\n created_at TIMESTAMPTZ NOT NULL,\n PRIMARY KEY (dataset_id,start_h3_r5,end_h3_r5,corridor_h3_r5),\n CONSTRAINT ck_hb_corridor_pack_cardinality CHECK (\n cardinality(corridor_h3_r7) BETWEEN 1 AND 49\n AND cardinality(traversal_count) = cardinality(corridor_h3_r7)\n AND cardinality(progress_permille_sum) = cardinality(corridor_h3_r7)\n AND cardinality(minimum_altitude_ft) = cardinality(corridor_h3_r7)\n AND cardinality(maximum_altitude_ft) = cardinality(corridor_h3_r7)\n AND cardinality(altitude_sum_ft) = cardinality(corridor_h3_r7)\n AND cardinality(altitude_evidence_count) = cardinality(corridor_h3_r7)\n ),\n CONSTRAINT ck_hb_corridor_pack_no_nulls CHECK (\n array_position(corridor_h3_r7,NULL) IS NULL\n AND array_position(traversal_count,NULL) IS NULL\n AND array_position(progress_permille_sum,NULL) IS NULL\n AND array_position(minimum_altitude_ft,NULL) IS NULL\n AND array_position(maximum_altitude_ft,NULL) IS NULL\n AND array_position(altitude_sum_ft,NULL) IS NULL\n AND array_position(altitude_evidence_count,NULL) IS NULL\n )\n);\n\nCREATE INDEX ix_hb_corridor_pack_parent_lookup\n ON historical_bootstrap_route_corridor_packs(\n dataset_id,corridor_h3_r5,start_h3_r5,end_h3_r5);\n\nCREATE OR REPLACE FUNCTION historical_bootstrap_corridor_pack_mutation_guard()\nRETURNS trigger LANGUAGE plpgsql AS $$\nDECLARE\n dataset_state VARCHAR(20);\n dataset_derivation VARCHAR(80);\n dataset_importer VARCHAR(80);\n route_evidence BIGINT;\n cell_count INTEGER;\n i INTEGER;\nBEGIN\n IF TG_OP='UPDATE' THEN\n RAISE EXCEPTION 'historical bootstrap corridor packs are immutable after insert';\n END IF;\n\n SELECT state,derivation_version,importer_version\n INTO dataset_state,dataset_derivation,dataset_importer\n FROM historical_bootstrap_datasets\n WHERE dataset_id=CASE WHEN TG_OP='DELETE' THEN OLD.dataset_id ELSE NEW.dataset_id END;\n\n IF dataset_state IS NULL THEN\n IF TG_OP='DELETE' THEN RETURN OLD; END IF;\n RAISE EXCEPTION 'historical bootstrap corridor pack requires existing dataset';\n END IF;\n\n IF TG_OP='DELETE' THEN\n IF dataset_state='EXPIRED' THEN RETURN OLD; END IF;\n RAISE EXCEPTION 'historical bootstrap corridor packs can only be deleted with an EXPIRED dataset';\n END IF;\n\n IF dataset_state NOT IN ('MANIFEST_LOCKED','DERIVING','STAGED') THEN\n RAISE EXCEPTION 'historical bootstrap corridor pack cannot be staged in dataset state %', dataset_state;\n END IF;\n\n IF dataset_importer !~ '^p7-postgres-loader-v[0-9]+-packed-corridors$' THEN\n RAISE EXCEPTION 'historical bootstrap corridor pack requires packed-corridor importer generation';\n END IF;\n\n SELECT evidence_count INTO route_evidence\n FROM historical_bootstrap_pattern_aggregates\n WHERE dataset_id=NEW.dataset_id\n AND derivation_version=dataset_derivation\n AND pattern_kind='ROUTE'\n AND subject_key='-'\n AND start_h3_r5=NEW.start_h3_r5\n AND end_h3_r5=NEW.end_h3_r5;\n\n IF route_evidence IS NULL THEN\n RAISE EXCEPTION 'historical bootstrap corridor pack requires corresponding coarse ROUTE aggregate';\n END IF;\n\n cell_count := cardinality(NEW.corridor_h3_r7);\n IF (SELECT count(DISTINCT value) FROM unnest(NEW.corridor_h3_r7) AS value) <> cell_count THEN\n RAISE EXCEPTION 'historical bootstrap corridor pack contains duplicate R7 cells';\n END IF;\n\n FOR i IN 1..cell_count LOOP\n IF NEW.corridor_h3_r7[i] !~ '^87[0-9a-f]{13}$'\n OR NEW.traversal_count[i] < 1\n OR NEW.traversal_count[i] > route_evidence\n OR NEW.progress_permille_sum[i] < 0\n OR NEW.progress_permille_sum[i] > NEW.traversal_count[i] * (1000::BIGINT)\n OR NEW.minimum_altitude_ft[i] < -2000\n OR NEW.maximum_altitude_ft[i] < NEW.minimum_altitude_ft[i]\n OR NEW.maximum_altitude_ft[i] > 100000\n OR NEW.altitude_evidence_count[i] < NEW.traversal_count[i]\n OR NEW.altitude_sum_ft[i] < NEW.altitude_evidence_count[i] * (-2000::BIGINT)\n OR NEW.altitude_sum_ft[i] > NEW.altitude_evidence_count[i] * (100000::BIGINT) THEN\n RAISE EXCEPTION 'invalid historical bootstrap corridor pack cell at index %', i;\n END IF;\n END LOOP;\n\n RETURN NEW;\nEND $$;\n\nCREATE TRIGGER trg_hb_corridor_pack_mutation_guard\nBEFORE INSERT OR UPDATE OR DELETE ON historical_bootstrap_route_corridor_packs\nFOR EACH ROW EXECUTE FUNCTION historical_bootstrap_corridor_pack_mutation_guard();\n\n-- Storage-generation boundary: a packed importer can never write the legacy row-per-R7 table.\n-- This prevents duplicate logical evidence through the UNION compatibility view even if a writer is misconfigured.\nCREATE OR REPLACE FUNCTION historical_bootstrap_corridor_mutation_guard()\nRETURNS trigger LANGUAGE plpgsql AS $$\nDECLARE\n dataset_state VARCHAR(20);\n dataset_importer VARCHAR(80);\n route_evidence BIGINT;\nBEGIN\n IF TG_OP='UPDATE' THEN\n RAISE EXCEPTION 'historical bootstrap corridor aggregates are immutable after insert';\n END IF;\n\n SELECT state,importer_version INTO dataset_state,dataset_importer\n FROM historical_bootstrap_datasets\n WHERE dataset_id=CASE WHEN TG_OP='DELETE' THEN OLD.dataset_id ELSE NEW.dataset_id END;\n\n IF dataset_state IS NULL THEN\n IF TG_OP='DELETE' THEN RETURN OLD; END IF;\n RAISE EXCEPTION 'historical bootstrap corridor aggregate requires existing dataset';\n END IF;\n\n IF TG_OP='DELETE' THEN\n IF dataset_state='EXPIRED' THEN RETURN OLD; END IF;\n RAISE EXCEPTION 'historical bootstrap corridor aggregates can only be deleted with an EXPIRED dataset';\n END IF;\n\n IF dataset_state NOT IN ('MANIFEST_LOCKED','DERIVING','STAGED') THEN\n RAISE EXCEPTION 'historical bootstrap corridor cannot be staged in dataset state %', dataset_state;\n END IF;\n\n IF dataset_importer ~ '^p7-postgres-loader-v[0-9]+-packed-corridors$' THEN\n RAISE EXCEPTION 'packed-corridor importer cannot write legacy corridor aggregates';\n END IF;\n\n SELECT evidence_count INTO route_evidence\n FROM historical_bootstrap_pattern_aggregates\n WHERE dataset_id=NEW.dataset_id\n AND derivation_version=NEW.derivation_version\n AND pattern_kind='ROUTE'\n AND subject_key='-'\n AND start_h3_r5=NEW.start_h3_r5\n AND end_h3_r5=NEW.end_h3_r5;\n\n IF route_evidence IS NULL THEN\n RAISE EXCEPTION 'historical bootstrap corridor requires corresponding coarse ROUTE aggregate';\n END IF;\n IF NEW.traversal_count > route_evidence THEN\n RAISE EXCEPTION 'historical bootstrap corridor traversal count % exceeds ROUTE evidence %',\n NEW.traversal_count,route_evidence;\n END IF;\n RETURN NEW;\nEND $$;\n\nCREATE OR REPLACE VIEW historical_bootstrap_ready_route_corridor_aggregates AS\nSELECT c.dataset_id,\n c.derivation_version,\n c.start_h3_r5,\n c.end_h3_r5,\n c.corridor_h3_r7,\n c.traversal_count,\n c.progress_permille_sum,\n c.minimum_altitude_ft,\n c.maximum_altitude_ft,\n c.altitude_sum_ft,\n c.altitude_evidence_count,\n c.created_at,\n c.corridor_h3_r5\n FROM historical_bootstrap_route_corridor_aggregates c\n JOIN historical_bootstrap_datasets d ON d.dataset_id=c.dataset_id\n WHERE d.state='READY'\nUNION ALL\nSELECT p.dataset_id,\n d.derivation_version,\n p.start_h3_r5,\n p.end_h3_r5,\n p.corridor_h3_r7[s.i],\n p.traversal_count[s.i],\n p.progress_permille_sum[s.i],\n p.minimum_altitude_ft[s.i],\n p.maximum_altitude_ft[s.i],\n p.altitude_sum_ft[s.i],\n p.altitude_evidence_count[s.i],\n p.created_at,\n p.corridor_h3_r5\n FROM historical_bootstrap_route_corridor_packs p\n JOIN historical_bootstrap_datasets d ON d.dataset_id=p.dataset_id\n CROSS JOIN LATERAL generate_subscripts(p.corridor_h3_r7,1) AS s(i)\n WHERE d.state='READY';\n\nCOMMENT ON TABLE historical_bootstrap_route_corridor_packs IS\n 'Compact immutable route x R5-parent packs; each pack contains at most the 49 R7 children of one R5 parent.';\nCOMMENT ON VIEW historical_bootstrap_ready_route_corridor_aggregates IS\n 'Compatibility READY seam over legacy row-per-cell evidence and compact corridor packs; HOLDOUT truth remains excluded.';\n" }, { "path": "backend/truth-model/src/main/resources/db/migration/V103__historical_retention_lookup.sql", "encoding": "UTF-8", "sha256": "8bcc0f53bf51bf1e60d9e3f3db8bce69b9996dcac768743c0997b256e5deb089", "content": "-- Bounded lookup for regional historical evidence retention maintenance.\n-- Immutable authority/day-seal rows remain untouched; only heavy dataset evidence is purged.\nCREATE INDEX IF NOT EXISTS ix_hb_dataset_scoring_retention\n ON historical_bootstrap_datasets(scoring_expires_at, dataset_id)\n WHERE dataset_purpose='BOOTSTRAP_SCORING'\n AND state='READY'\n AND scoring_expires_at IS NOT NULL;\n\nCOMMENT ON INDEX ix_hb_dataset_scoring_retention IS\n 'Bounded READY scoring-dataset retention scan for heavy-evidence purge; metadata/seals remain immutable.';\n" }, { "path": "backend/truth-model/src/main/resources/db/migration/V69__historical_bootstrap_persistence.sql", "encoding": "UTF-8", "sha256": "fb9b410ae00847eecce660a57fe1deb82b63a84980b6d7f59b0da18047106c07", "content": "CREATE TABLE historical_bootstrap_datasets (\n dataset_id UUID PRIMARY KEY,\n dataset_purpose VARCHAR(32) NOT NULL CHECK (dataset_purpose IN ('QUALIFICATION_EXPERIMENT','BOOTSTRAP_SCORING')),\n provider VARCHAR(32) NOT NULL CHECK (provider = 'ADSB_LOL'),\n source_repository VARCHAR(160) NOT NULL,\n source_year SMALLINT NOT NULL CHECK (source_year BETWEEN 2000 AND 2100),\n license_identifier VARCHAR(32) NOT NULL CHECK (license_identifier = 'ODBL-1.0'),\n from_date DATE NOT NULL,\n to_date DATE NOT NULL,\n as_of_date DATE,\n lookback_days INTEGER,\n learning_end DATE,\n embargo_date DATE,\n holdout_start DATE,\n importer_version VARCHAR(80) NOT NULL,\n parser_version VARCHAR(80) NOT NULL,\n archive_safety_profile VARCHAR(320) NOT NULL,\n eligibility_version VARCHAR(80) NOT NULL,\n segmentation_version VARCHAR(80) NOT NULL,\n derivation_version VARCHAR(80) NOT NULL,\n semantic_fingerprint VARCHAR(64) NOT NULL CHECK (semantic_fingerprint ~ '^[0-9a-f]{64}$'),\n coverage_version VARCHAR(80) NOT NULL,\n coverage_hash VARCHAR(64) NOT NULL CHECK (coverage_hash ~ '^[0-9a-f]{64}$'),\n manifest_sha256 VARCHAR(64) NOT NULL CHECK (manifest_sha256 ~ '^[0-9a-f]{64}$'),\n state VARCHAR(20) NOT NULL CHECK (state IN ('PLANNED','MANIFEST_LOCKED','DERIVING','STAGED','READY','RETIRED','EXPIRED')),\n scoring_eligible_from TIMESTAMPTZ,\n scoring_expires_at TIMESTAMPTZ,\n created_at TIMESTAMPTZ NOT NULL,\n ready_at TIMESTAMPTZ,\n retired_at TIMESTAMPTZ,\n CONSTRAINT ck_hb_dataset_dates CHECK (to_date >= from_date),\n CONSTRAINT ck_hb_dataset_window CHECK (\n (dataset_purpose = 'QUALIFICATION_EXPERIMENT'\n AND as_of_date IS NULL AND lookback_days IS NULL\n AND learning_end BETWEEN from_date AND to_date\n AND embargo_date = learning_end + 1\n AND holdout_start = embargo_date + 1\n AND holdout_start <= to_date)\n OR\n (dataset_purpose = 'BOOTSTRAP_SCORING'\n AND learning_end IS NULL AND embargo_date IS NULL AND holdout_start IS NULL\n AND as_of_date = to_date\n AND lookback_days = (to_date - from_date + 1)\n AND lookback_days BETWEEN 1 AND 730)\n ),\n CONSTRAINT ck_hb_dataset_state_time CHECK (\n (state IN ('PLANNED','MANIFEST_LOCKED','DERIVING','STAGED') AND ready_at IS NULL AND retired_at IS NULL)\n OR (state = 'READY' AND ready_at IS NOT NULL AND retired_at IS NULL)\n OR (state IN ('RETIRED','EXPIRED') AND ready_at IS NOT NULL AND retired_at IS NOT NULL)\n ),\n CONSTRAINT ck_hb_dataset_scoring_dates CHECK (\n scoring_expires_at IS NULL OR scoring_eligible_from IS NULL OR scoring_expires_at > scoring_eligible_from\n )\n);\n\nCREATE TABLE historical_bootstrap_days (\n dataset_id UUID NOT NULL REFERENCES historical_bootstrap_datasets(dataset_id) ON DELETE CASCADE,\n source_date DATE NOT NULL,\n experiment_role VARCHAR(16) NOT NULL CHECK (experiment_role IN ('LEARNING','EMBARGO','HOLDOUT','SCORING')),\n selected_release_id BIGINT,\n selected_release_tag VARCHAR(160),\n selected_release_variant VARCHAR(24),\n terminal_quality VARCHAR(16) NOT NULL CHECK (terminal_quality IN ('QUALIFIED','DEGRADED','GAP')),\n reason_code VARCHAR(64),\n selected_archive_family VARCHAR(16),\n selected_bytes BIGINT NOT NULL CHECK (selected_bytes >= 0),\n parsed_trace_count INTEGER NOT NULL CHECK (parsed_trace_count >= 0),\n trace_parse_failure_count INTEGER NOT NULL CHECK (trace_parse_failure_count >= 0),\n malformed_point_count BIGINT NOT NULL CHECK (malformed_point_count >= 0),\n accepted_point_count BIGINT NOT NULL CHECK (accepted_point_count >= 0),\n rejected_point_count BIGINT NOT NULL CHECK (rejected_point_count >= 0),\n finalized_segment_count BIGINT NOT NULL CHECK (finalized_segment_count >= 0),\n day_manifest_sha256 VARCHAR(64) NOT NULL CHECK (day_manifest_sha256 ~ '^[0-9a-f]{64}$'),\n day_delta_checksum VARCHAR(64) NOT NULL CHECK (day_delta_checksum ~ '^[0-9a-f]{64}$'),\n qualified_at TIMESTAMPTZ NOT NULL,\n PRIMARY KEY (dataset_id, source_date),\n CONSTRAINT ck_hb_day_reason CHECK (\n (terminal_quality = 'QUALIFIED' AND reason_code IS NULL)\n OR (terminal_quality IN ('DEGRADED','GAP') AND reason_code ~ '^[A-Z0-9_]{1,64}$')\n ),\n CONSTRAINT ck_hb_day_source CHECK (\n (selected_release_id IS NULL AND selected_release_tag IS NULL AND selected_release_variant IS NULL\n AND selected_archive_family IS NULL AND selected_bytes = 0)\n OR\n (selected_release_id > 0 AND selected_release_tag IS NOT NULL AND selected_release_variant IS NOT NULL\n AND selected_archive_family IN ('UNSPLIT','SPLIT') AND selected_bytes > 0)\n )\n);\n\nCREATE TABLE historical_bootstrap_assets (\n dataset_id UUID NOT NULL,\n source_date DATE NOT NULL,\n asset_ordinal SMALLINT NOT NULL CHECK (asset_ordinal BETWEEN 0 AND 255),\n release_asset_id BIGINT NOT NULL CHECK (release_asset_id > 0),\n filename VARCHAR(240) NOT NULL,\n byte_size BIGINT NOT NULL CHECK (byte_size > 0),\n upstream_sha256 VARCHAR(64) NOT NULL CHECK (upstream_sha256 ~ '^[0-9a-f]{64}$'),\n verified_sha256 VARCHAR(64) NOT NULL CHECK (verified_sha256 ~ '^[0-9a-f]{64}$'),\n source_repository VARCHAR(160) NOT NULL,\n release_tag VARCHAR(160) NOT NULL,\n archive_family VARCHAR(16) NOT NULL CHECK (archive_family IN ('UNSPLIT','SPLIT')),\n PRIMARY KEY (dataset_id, source_date, asset_ordinal),\n UNIQUE (dataset_id, source_date, release_asset_id),\n FOREIGN KEY (dataset_id, source_date) REFERENCES historical_bootstrap_days(dataset_id, source_date) ON DELETE CASCADE,\n CHECK (verified_sha256 = upstream_sha256)\n);\n\nCREATE TABLE historical_bootstrap_pattern_aggregates (\n pattern_id UUID PRIMARY KEY,\n dataset_id UUID NOT NULL REFERENCES historical_bootstrap_datasets(dataset_id) ON DELETE CASCADE,\n derivation_version VARCHAR(80) NOT NULL,\n pattern_kind VARCHAR(24) NOT NULL CHECK (pattern_kind IN ('ROUTE','AIRCRAFT_ROUTE','CALLSIGN_ROUTE')),\n subject_key VARCHAR(64) NOT NULL,\n start_h3_r5 VARCHAR(15) NOT NULL CHECK (start_h3_r5 ~ '^85[0-9a-f]{13}$'),\n end_h3_r5 VARCHAR(15) NOT NULL CHECK (end_h3_r5 ~ '^85[0-9a-f]{13}$'),\n evidence_count BIGINT NOT NULL,\n qualified_evidence_count BIGINT NOT NULL,\n first_seen_at TIMESTAMPTZ NOT NULL,\n last_seen_at TIMESTAMPTZ NOT NULL,\n minimum_altitude_ft INTEGER NOT NULL,\n maximum_altitude_ft INTEGER NOT NULL,\n mean_altitude_sum_ft BIGINT NOT NULL,\n altitude_evidence_count BIGINT NOT NULL,\n duration_seconds_sum BIGINT NOT NULL,\n observation_count_sum BIGINT NOT NULL,\n adsb_icao_count BIGINT NOT NULL,\n adsr_icao_count BIGINT NOT NULL,\n tisb_icao_count BIGINT NOT NULL,\n mlat_count BIGINT NOT NULL,\n created_at TIMESTAMPTZ NOT NULL,\n UNIQUE (dataset_id, derivation_version, pattern_kind, subject_key, start_h3_r5, end_h3_r5),\n CONSTRAINT ck_hb_pattern_subject CHECK (\n (pattern_kind='ROUTE' AND subject_key='-')\n OR (pattern_kind='AIRCRAFT_ROUTE' AND subject_key ~ '^[0-9a-f]{6}$')\n OR (pattern_kind='CALLSIGN_ROUTE' AND subject_key ~ '^[A-Z0-9][A-Z0-9-]{0,31}$')\n ),\n CONSTRAINT ck_hb_pattern_evidence CHECK (evidence_count >= 1 AND qualified_evidence_count BETWEEN 0 AND evidence_count),\n CONSTRAINT ck_hb_pattern_time CHECK (last_seen_at >= first_seen_at),\n CONSTRAINT ck_hb_pattern_altitude CHECK (\n (\n altitude_evidence_count = 0\n AND minimum_altitude_ft = 0\n AND maximum_altitude_ft = 0\n AND mean_altitude_sum_ft = 0\n )\n OR\n (\n altitude_evidence_count BETWEEN 1 AND evidence_count\n AND minimum_altitude_ft BETWEEN -2000 AND 100000\n AND maximum_altitude_ft BETWEEN minimum_altitude_ft AND 100000\n AND mean_altitude_sum_ft BETWEEN altitude_evidence_count * (-2000::BIGINT)\n AND altitude_evidence_count * (100000::BIGINT)\n )\n ),\n CONSTRAINT ck_hb_pattern_accumulators CHECK (\n duration_seconds_sum BETWEEN 0 AND evidence_count * (64800::BIGINT)\n AND observation_count_sum BETWEEN evidence_count AND evidence_count * (1000000::BIGINT)\n ),\n CONSTRAINT ck_hb_pattern_sources CHECK (\n adsb_icao_count >= 0 AND adsr_icao_count >= 0 AND tisb_icao_count >= 0 AND mlat_count >= 0\n AND adsb_icao_count + adsr_icao_count + tisb_icao_count + mlat_count = observation_count_sum\n )\n);\n\nCREATE TABLE historical_bootstrap_hour_bins (\n pattern_id UUID NOT NULL REFERENCES historical_bootstrap_pattern_aggregates(pattern_id) ON DELETE CASCADE,\n hour_of_week SMALLINT NOT NULL CHECK (hour_of_week BETWEEN 0 AND 167),\n evidence_count BIGINT NOT NULL CHECK (evidence_count >= 1),\n PRIMARY KEY (pattern_id, hour_of_week)\n);\n\nCREATE TABLE historical_bootstrap_holdout_segments (\n dataset_id UUID NOT NULL,\n source_date DATE NOT NULL,\n segment_id VARCHAR(64) NOT NULL CHECK (segment_id ~ '^[0-9a-f]{64}$'),\n icao24 VARCHAR(6) NOT NULL CHECK (icao24 ~ '^[0-9a-f]{6}$'),\n callsign VARCHAR(64),\n first_observed_at TIMESTAMPTZ NOT NULL,\n last_observed_at TIMESTAMPTZ NOT NULL,\n start_h3_r5 VARCHAR(15) NOT NULL CHECK (start_h3_r5 ~ '^85[0-9a-f]{13}$'),\n end_h3_r5 VARCHAR(15) NOT NULL CHECK (end_h3_r5 ~ '^85[0-9a-f]{13}$'),\n minimum_altitude_ft INTEGER NOT NULL,\n maximum_altitude_ft INTEGER NOT NULL,\n mean_altitude_ft INTEGER NOT NULL,\n altitude_evidence_count BIGINT NOT NULL,\n duration_seconds BIGINT NOT NULL,\n observation_count BIGINT NOT NULL,\n adsb_icao_count BIGINT NOT NULL,\n adsr_icao_count BIGINT NOT NULL,\n tisb_icao_count BIGINT NOT NULL,\n mlat_count BIGINT NOT NULL,\n delete_after TIMESTAMPTZ NOT NULL,\n PRIMARY KEY (dataset_id, segment_id),\n FOREIGN KEY (dataset_id, source_date) REFERENCES historical_bootstrap_days(dataset_id, source_date) ON DELETE CASCADE,\n CHECK (callsign IS NULL OR callsign ~ '^[A-Z0-9][A-Z0-9-]{0,31}$'),\n CHECK (last_observed_at >= first_observed_at AND delete_after > last_observed_at),\n CONSTRAINT ck_hb_holdout_altitude CHECK (\n (\n altitude_evidence_count = 0\n AND minimum_altitude_ft = 0\n AND maximum_altitude_ft = 0\n AND mean_altitude_ft = 0\n )\n OR\n (\n altitude_evidence_count BETWEEN 1 AND observation_count\n AND minimum_altitude_ft BETWEEN -2000 AND 100000\n AND maximum_altitude_ft BETWEEN minimum_altitude_ft AND 100000\n AND mean_altitude_ft BETWEEN minimum_altitude_ft AND maximum_altitude_ft\n )\n ),\n CHECK (duration_seconds BETWEEN 0 AND 64800 AND observation_count BETWEEN 1 AND 1000000),\n CHECK (adsb_icao_count >= 0 AND adsr_icao_count >= 0 AND tisb_icao_count >= 0 AND mlat_count >= 0),\n CHECK (adsb_icao_count + adsr_icao_count + tisb_icao_count + mlat_count = observation_count)\n);\n\nCREATE INDEX ix_hb_dataset_state ON historical_bootstrap_datasets(state,dataset_purpose,ready_at);\nCREATE INDEX ix_hb_day_quality ON historical_bootstrap_days(dataset_id,terminal_quality,source_date);\nCREATE INDEX ix_hb_pattern_lookup ON historical_bootstrap_pattern_aggregates(dataset_id,derivation_version,pattern_kind,subject_key);\nCREATE INDEX ix_hb_pattern_route ON historical_bootstrap_pattern_aggregates(dataset_id,derivation_version,start_h3_r5,end_h3_r5);\nCREATE INDEX ix_hb_hour_lookup ON historical_bootstrap_hour_bins(hour_of_week,pattern_id);\nCREATE INDEX ix_hb_holdout_date ON historical_bootstrap_holdout_segments(dataset_id,source_date);\nCREATE INDEX ix_hb_holdout_delete ON historical_bootstrap_holdout_segments(delete_after,dataset_id);\n\nCREATE OR REPLACE FUNCTION historical_bootstrap_dataset_update_guard()\nRETURNS trigger LANGUAGE plpgsql AS $$\nBEGIN\n IF (to_jsonb(NEW) - ARRAY['state','ready_at','retired_at']) IS DISTINCT FROM\n (to_jsonb(OLD) - ARRAY['state','ready_at','retired_at']) THEN\n RAISE EXCEPTION 'historical bootstrap dataset identity is immutable';\n END IF;\n IF NEW.state = OLD.state THEN\n IF NEW.ready_at IS DISTINCT FROM OLD.ready_at OR NEW.retired_at IS DISTINCT FROM OLD.retired_at THEN\n RAISE EXCEPTION 'historical bootstrap lifecycle timestamps require a state transition';\n END IF;\n RETURN NEW;\n END IF;\n IF NOT ((OLD.state='PLANNED' AND NEW.state='MANIFEST_LOCKED')\n OR (OLD.state='MANIFEST_LOCKED' AND NEW.state='DERIVING')\n OR (OLD.state='DERIVING' AND NEW.state='STAGED')\n OR (OLD.state='STAGED' AND NEW.state='READY')\n OR (OLD.state='READY' AND NEW.state='RETIRED')\n OR (OLD.state='RETIRED' AND NEW.state='EXPIRED')) THEN\n RAISE EXCEPTION 'invalid historical bootstrap dataset state transition: % -> %', OLD.state, NEW.state;\n END IF;\n IF NEW.state='READY' AND (NEW.ready_at IS NULL OR NEW.retired_at IS NOT NULL) THEN\n RAISE EXCEPTION 'READY requires ready_at and no retired_at';\n ELSIF NEW.state='RETIRED' AND (NEW.ready_at IS DISTINCT FROM OLD.ready_at OR NEW.ready_at IS NULL OR NEW.retired_at IS NULL) THEN\n RAISE EXCEPTION 'RETIRED requires immutable ready_at and retired_at';\n ELSIF NEW.state='EXPIRED' AND (NEW.ready_at IS DISTINCT FROM OLD.ready_at OR NEW.retired_at IS DISTINCT FROM OLD.retired_at OR NEW.ready_at IS NULL OR NEW.retired_at IS NULL) THEN\n RAISE EXCEPTION 'EXPIRED preserves ready/retired timestamps';\n ELSIF NEW.state IN ('PLANNED','MANIFEST_LOCKED','DERIVING','STAGED') AND (NEW.ready_at IS NOT NULL OR NEW.retired_at IS NOT NULL) THEN\n RAISE EXCEPTION 'pre-READY state cannot have terminal timestamps';\n END IF;\n RETURN NEW;\nEND $$;\nCREATE TRIGGER trg_hb_dataset_update_guard BEFORE UPDATE ON historical_bootstrap_datasets\nFOR EACH ROW EXECUTE FUNCTION historical_bootstrap_dataset_update_guard();\n\nCREATE OR REPLACE FUNCTION historical_bootstrap_dataset_delete_guard()\nRETURNS trigger LANGUAGE plpgsql AS $$\nBEGIN\n IF OLD.state <> 'EXPIRED' THEN RAISE EXCEPTION 'historical bootstrap dataset can only be deleted after EXPIRED'; END IF;\n RETURN OLD;\nEND $$;\nCREATE TRIGGER trg_hb_dataset_delete_guard BEFORE DELETE ON historical_bootstrap_datasets\nFOR EACH ROW EXECUTE FUNCTION historical_bootstrap_dataset_delete_guard();\n\nCREATE OR REPLACE FUNCTION historical_bootstrap_child_mutation_guard()\nRETURNS trigger LANGUAGE plpgsql AS $$\nDECLARE target_dataset UUID; dataset_state VARCHAR(20);\nBEGIN\n IF TG_OP = 'UPDATE' THEN\n RAISE EXCEPTION 'historical bootstrap child rows are immutable after insert';\n END IF;\n\n target_dataset := CASE WHEN TG_OP='DELETE' THEN OLD.dataset_id ELSE NEW.dataset_id END;\n SELECT state INTO dataset_state FROM historical_bootstrap_datasets WHERE dataset_id=target_dataset;\n IF dataset_state IS NULL THEN\n IF TG_OP='DELETE' THEN RETURN OLD; END IF;\n RAISE EXCEPTION 'historical bootstrap child requires existing dataset';\n END IF;\n IF TG_OP='DELETE' THEN\n IF dataset_state='EXPIRED' THEN RETURN OLD; END IF;\n RAISE EXCEPTION 'historical bootstrap child rows can only be deleted with an EXPIRED dataset';\n END IF;\n IF dataset_state NOT IN ('MANIFEST_LOCKED','DERIVING','STAGED') THEN\n RAISE EXCEPTION 'historical bootstrap dataset % cannot accept staged evidence in state %', target_dataset, dataset_state;\n END IF;\n RETURN NEW;\nEND $$;\nCREATE TRIGGER trg_hb_day_mutation_guard BEFORE INSERT OR UPDATE OR DELETE ON historical_bootstrap_days\nFOR EACH ROW EXECUTE FUNCTION historical_bootstrap_child_mutation_guard();\nCREATE TRIGGER trg_hb_asset_mutation_guard BEFORE INSERT OR UPDATE OR DELETE ON historical_bootstrap_assets\nFOR EACH ROW EXECUTE FUNCTION historical_bootstrap_child_mutation_guard();\nCREATE TRIGGER trg_hb_pattern_mutation_guard BEFORE INSERT OR UPDATE OR DELETE ON historical_bootstrap_pattern_aggregates\nFOR EACH ROW EXECUTE FUNCTION historical_bootstrap_child_mutation_guard();\n\nCREATE OR REPLACE FUNCTION historical_bootstrap_holdout_mutation_guard()\nRETURNS trigger LANGUAGE plpgsql AS $$\nDECLARE target_dataset UUID; dataset_state VARCHAR(20);\nBEGIN\n IF TG_OP = 'UPDATE' THEN\n RAISE EXCEPTION 'historical bootstrap holdout rows are immutable after insert';\n END IF;\n\n target_dataset := CASE WHEN TG_OP='DELETE' THEN OLD.dataset_id ELSE NEW.dataset_id END;\n SELECT state INTO dataset_state FROM historical_bootstrap_datasets WHERE dataset_id=target_dataset;\n IF dataset_state IS NULL THEN\n IF TG_OP='DELETE' THEN RETURN OLD; END IF;\n RAISE EXCEPTION 'historical bootstrap holdout row requires existing dataset';\n END IF;\n IF TG_OP='DELETE' THEN\n IF dataset_state='EXPIRED' THEN RETURN OLD; END IF;\n IF dataset_state IN ('READY','RETIRED') AND OLD.delete_after <= now() THEN RETURN OLD; END IF;\n RAISE EXCEPTION 'historical bootstrap holdout row is not retention-eligible for deletion';\n END IF;\n IF dataset_state NOT IN ('MANIFEST_LOCKED','DERIVING','STAGED') THEN\n RAISE EXCEPTION 'historical bootstrap dataset % cannot accept holdout evidence in state %', target_dataset, dataset_state;\n END IF;\n RETURN NEW;\nEND $$;\nCREATE TRIGGER trg_hb_holdout_mutation_guard BEFORE INSERT OR UPDATE OR DELETE ON historical_bootstrap_holdout_segments\nFOR EACH ROW EXECUTE FUNCTION historical_bootstrap_holdout_mutation_guard();\n\nCREATE OR REPLACE FUNCTION historical_bootstrap_hour_mutation_guard()\nRETURNS trigger LANGUAGE plpgsql AS $$\nDECLARE target_pattern UUID; dataset_state VARCHAR(20);\nBEGIN\n IF TG_OP = 'UPDATE' THEN\n RAISE EXCEPTION 'historical bootstrap hour bins are immutable after insert';\n END IF;\n\n target_pattern := CASE WHEN TG_OP='DELETE' THEN OLD.pattern_id ELSE NEW.pattern_id END;\n SELECT d.state INTO dataset_state\n FROM historical_bootstrap_pattern_aggregates p\n JOIN historical_bootstrap_datasets d ON d.dataset_id=p.dataset_id\n WHERE p.pattern_id=target_pattern;\n IF dataset_state IS NULL THEN\n IF TG_OP='DELETE' THEN RETURN OLD; END IF;\n RAISE EXCEPTION 'historical bootstrap hour bin requires existing pattern';\n END IF;\n IF TG_OP='DELETE' THEN\n IF dataset_state='EXPIRED' THEN RETURN OLD; END IF;\n RAISE EXCEPTION 'historical bootstrap hour bins can only be deleted with an EXPIRED dataset';\n END IF;\n IF dataset_state NOT IN ('MANIFEST_LOCKED','DERIVING','STAGED') THEN\n RAISE EXCEPTION 'historical bootstrap hour bin cannot be staged in dataset state %', dataset_state;\n END IF;\n RETURN NEW;\nEND $$;\nCREATE TRIGGER trg_hb_hour_mutation_guard BEFORE INSERT OR UPDATE OR DELETE ON historical_bootstrap_hour_bins\nFOR EACH ROW EXECUTE FUNCTION historical_bootstrap_hour_mutation_guard();\n\nCREATE OR REPLACE FUNCTION historical_bootstrap_holdout_role_guard()\nRETURNS trigger LANGUAGE plpgsql AS $$\nDECLARE day_role VARCHAR(16);\nBEGIN\n SELECT experiment_role INTO day_role FROM historical_bootstrap_days\n WHERE dataset_id=NEW.dataset_id AND source_date=NEW.source_date;\n IF day_role IS DISTINCT FROM 'HOLDOUT' THEN\n RAISE EXCEPTION 'historical bootstrap holdout segment requires HOLDOUT source day';\n END IF;\n RETURN NEW;\nEND $$;\nCREATE TRIGGER trg_hb_holdout_role_guard BEFORE INSERT ON historical_bootstrap_holdout_segments\nFOR EACH ROW EXECUTE FUNCTION historical_bootstrap_holdout_role_guard();\n\nCREATE VIEW historical_bootstrap_ready_datasets AS\nSELECT * FROM historical_bootstrap_datasets WHERE state='READY';\nCREATE VIEW historical_bootstrap_ready_pattern_aggregates AS\nSELECT p.* FROM historical_bootstrap_pattern_aggregates p\nJOIN historical_bootstrap_datasets d ON d.dataset_id=p.dataset_id WHERE d.state='READY';\nCREATE VIEW historical_bootstrap_ready_hour_bins AS\nSELECT h.* FROM historical_bootstrap_hour_bins h\nJOIN historical_bootstrap_pattern_aggregates p ON p.pattern_id=h.pattern_id\nJOIN historical_bootstrap_datasets d ON d.dataset_id=p.dataset_id WHERE d.state='READY';\n" }, { "path": "backend/truth-model/src/main/resources/db/migration/V70__historical_bootstrap_asset_verification_provenance.sql", "encoding": "UTF-8", "sha256": "ced72591d89f0264019312f53b530600c766b1a6da41e8d1d212601eae4ee2a4", "content": "-- P7.5 staging hardening: selected source metadata is not equivalent to verified payload bytes.\n--\n-- V69 intentionally records the frozen source identity for every calendar day. Gate D, however,\n-- does not download EMBARGO payload and may retain metadata for other terminal DEGRADED/GAP days.\n-- Therefore verified_sha256 must be nullable and may be populated only for a QUALIFIED source day\n-- whose payload was actually materialized and digest-verified by the historical importer.\n\nALTER TABLE historical_bootstrap_assets\n ALTER COLUMN verified_sha256 DROP NOT NULL;\n\nCOMMENT ON COLUMN historical_bootstrap_assets.verified_sha256 IS\n 'SHA-256 verified from downloaded payload bytes; NULL means metadata selected but payload was not verified/consumed';\n\nCREATE OR REPLACE FUNCTION historical_bootstrap_asset_verification_provenance_guard()\nRETURNS trigger LANGUAGE plpgsql AS $$\nDECLARE\n day_quality VARCHAR(16);\nBEGIN\n SELECT terminal_quality\n INTO day_quality\n FROM historical_bootstrap_days\n WHERE dataset_id = NEW.dataset_id\n AND source_date = NEW.source_date;\n\n IF day_quality IS NULL THEN\n RAISE EXCEPTION 'historical bootstrap asset requires an existing source day';\n END IF;\n\n IF day_quality = 'QUALIFIED' THEN\n IF NEW.verified_sha256 IS NULL OR NEW.verified_sha256 IS DISTINCT FROM NEW.upstream_sha256 THEN\n RAISE EXCEPTION 'QUALIFIED historical bootstrap asset requires verified upstream SHA-256';\n END IF;\n ELSE\n -- A DEGRADED/GAP day can preserve immutable selected-source metadata, but claiming payload\n -- verification would be false provenance. Force the verification field to NULL even if an\n -- older loader supplies the upstream digest in both columns.\n NEW.verified_sha256 := NULL;\n END IF;\n\n RETURN NEW;\nEND $$;\n\nCREATE TRIGGER trg_hb_asset_verification_provenance\nBEFORE INSERT OR UPDATE ON historical_bootstrap_assets\nFOR EACH ROW EXECUTE FUNCTION historical_bootstrap_asset_verification_provenance_guard();\n" }, { "path": "backend/truth-model/src/main/resources/db/migration/V71__historical_bootstrap_route_corridor_evidence.sql", "encoding": "UTF-8", "sha256": "9186748a23048037afa3fb0f56ad948667f363597bebb7d7e0c63e1a1f97d237", "content": "-- P7 Hungary/v3 spatial hardening: bounded H3 R7 route-corridor evidence for later P8 relevance.\n-- Learning corridor aggregates remain separate from HOLDOUT spatial truth. No raw track/polyline is stored.\n\nCREATE TABLE historical_bootstrap_route_corridor_aggregates (\n dataset_id UUID NOT NULL REFERENCES historical_bootstrap_datasets(dataset_id) ON DELETE CASCADE,\n derivation_version VARCHAR(80) NOT NULL,\n start_h3_r5 VARCHAR(15) NOT NULL CHECK (start_h3_r5 ~ '^85[0-9a-f]{13}$'),\n end_h3_r5 VARCHAR(15) NOT NULL CHECK (end_h3_r5 ~ '^85[0-9a-f]{13}$'),\n corridor_h3_r7 VARCHAR(15) NOT NULL CHECK (corridor_h3_r7 ~ '^87[0-9a-f]{13}$'),\n traversal_count BIGINT NOT NULL CHECK (traversal_count >= 1),\n progress_permille_sum BIGINT NOT NULL CHECK (\n progress_permille_sum >= 0\n AND progress_permille_sum <= traversal_count * (1000::BIGINT)\n ),\n minimum_altitude_ft INTEGER NOT NULL CHECK (minimum_altitude_ft BETWEEN -2000 AND 100000),\n maximum_altitude_ft INTEGER NOT NULL CHECK (maximum_altitude_ft BETWEEN minimum_altitude_ft AND 100000),\n altitude_sum_ft BIGINT NOT NULL,\n altitude_evidence_count BIGINT NOT NULL CHECK (altitude_evidence_count >= traversal_count),\n created_at TIMESTAMPTZ NOT NULL,\n PRIMARY KEY (dataset_id, derivation_version, start_h3_r5, end_h3_r5, corridor_h3_r7),\n CONSTRAINT ck_hb_corridor_altitude_sum CHECK (\n altitude_sum_ft BETWEEN altitude_evidence_count * (-2000::BIGINT)\n AND altitude_evidence_count * (100000::BIGINT)\n )\n);\n\nCREATE INDEX ix_hb_corridor_cell_lookup\n ON historical_bootstrap_route_corridor_aggregates(dataset_id, derivation_version, corridor_h3_r7);\nCREATE INDEX ix_hb_corridor_route_lookup\n ON historical_bootstrap_route_corridor_aggregates(dataset_id, derivation_version, start_h3_r5, end_h3_r5);\n\n-- Existing v1/v2 holdout rows have no corridor by definition. Default TRUE preserves that legacy meaning.\nALTER TABLE historical_bootstrap_holdout_segments\n ADD COLUMN corridor_overflow BOOLEAN NOT NULL DEFAULT TRUE;\n\nCOMMENT ON COLUMN historical_bootstrap_holdout_segments.corridor_overflow IS\n 'TRUE means bounded R7 corridor unavailable (legacy generation or >512 unique cells); FALSE requires a complete ordered corridor child set';\n\nCREATE TABLE historical_bootstrap_holdout_corridor_cells (\n dataset_id UUID NOT NULL,\n segment_id VARCHAR(64) NOT NULL CHECK (segment_id ~ '^[0-9a-f]{64}$'),\n sequence_index SMALLINT NOT NULL CHECK (sequence_index BETWEEN 0 AND 511),\n corridor_h3_r7 VARCHAR(15) NOT NULL CHECK (corridor_h3_r7 ~ '^87[0-9a-f]{13}$'),\n progress_permille SMALLINT NOT NULL CHECK (progress_permille BETWEEN 0 AND 1000),\n minimum_altitude_ft INTEGER NOT NULL CHECK (minimum_altitude_ft BETWEEN -2000 AND 100000),\n maximum_altitude_ft INTEGER NOT NULL CHECK (maximum_altitude_ft BETWEEN minimum_altitude_ft AND 100000),\n altitude_sum_ft BIGINT NOT NULL,\n altitude_evidence_count BIGINT NOT NULL CHECK (altitude_evidence_count >= 1),\n PRIMARY KEY (dataset_id, segment_id, sequence_index),\n UNIQUE (dataset_id, segment_id, corridor_h3_r7),\n FOREIGN KEY (dataset_id, segment_id)\n REFERENCES historical_bootstrap_holdout_segments(dataset_id, segment_id) ON DELETE CASCADE,\n CONSTRAINT ck_hb_holdout_corridor_altitude_sum CHECK (\n altitude_sum_ft BETWEEN altitude_evidence_count * (-2000::BIGINT)\n AND altitude_evidence_count * (100000::BIGINT)\n )\n);\n\nCREATE INDEX ix_hb_holdout_corridor_cell\n ON historical_bootstrap_holdout_corridor_cells(corridor_h3_r7, dataset_id);\n\nCREATE OR REPLACE FUNCTION historical_bootstrap_corridor_mutation_guard()\nRETURNS trigger LANGUAGE plpgsql AS $$\nDECLARE\n dataset_state VARCHAR(20);\n route_evidence BIGINT;\nBEGIN\n IF TG_OP = 'UPDATE' THEN\n RAISE EXCEPTION 'historical bootstrap corridor aggregates are immutable after insert';\n END IF;\n\n SELECT state INTO dataset_state\n FROM historical_bootstrap_datasets\n WHERE dataset_id = CASE WHEN TG_OP='DELETE' THEN OLD.dataset_id ELSE NEW.dataset_id END;\n\n IF dataset_state IS NULL THEN\n IF TG_OP='DELETE' THEN RETURN OLD; END IF;\n RAISE EXCEPTION 'historical bootstrap corridor aggregate requires existing dataset';\n END IF;\n\n IF TG_OP='DELETE' THEN\n IF dataset_state='EXPIRED' THEN RETURN OLD; END IF;\n RAISE EXCEPTION 'historical bootstrap corridor aggregates can only be deleted with an EXPIRED dataset';\n END IF;\n\n IF dataset_state NOT IN ('MANIFEST_LOCKED','DERIVING','STAGED') THEN\n RAISE EXCEPTION 'historical bootstrap corridor cannot be staged in dataset state %', dataset_state;\n END IF;\n\n SELECT evidence_count INTO route_evidence\n FROM historical_bootstrap_pattern_aggregates\n WHERE dataset_id = NEW.dataset_id\n AND derivation_version = NEW.derivation_version\n AND pattern_kind = 'ROUTE'\n AND subject_key = '-'\n AND start_h3_r5 = NEW.start_h3_r5\n AND end_h3_r5 = NEW.end_h3_r5;\n\n IF route_evidence IS NULL THEN\n RAISE EXCEPTION 'historical bootstrap corridor requires corresponding coarse ROUTE aggregate';\n END IF;\n IF NEW.traversal_count > route_evidence THEN\n RAISE EXCEPTION 'historical bootstrap corridor traversal count % exceeds ROUTE evidence %',\n NEW.traversal_count, route_evidence;\n END IF;\n RETURN NEW;\nEND $$;\n\nCREATE TRIGGER trg_hb_corridor_mutation_guard\nBEFORE INSERT OR UPDATE OR DELETE ON historical_bootstrap_route_corridor_aggregates\nFOR EACH ROW EXECUTE FUNCTION historical_bootstrap_corridor_mutation_guard();\n\nCREATE OR REPLACE FUNCTION historical_bootstrap_holdout_corridor_mutation_guard()\nRETURNS trigger LANGUAGE plpgsql AS $$\nDECLARE\n target_dataset UUID;\n target_segment VARCHAR(64);\n dataset_state VARCHAR(20);\n parent_overflow BOOLEAN;\n parent_delete_after TIMESTAMPTZ;\nBEGIN\n IF TG_OP = 'UPDATE' THEN\n RAISE EXCEPTION 'historical bootstrap holdout corridor rows are immutable after insert';\n END IF;\n\n target_dataset := CASE WHEN TG_OP='DELETE' THEN OLD.dataset_id ELSE NEW.dataset_id END;\n target_segment := CASE WHEN TG_OP='DELETE' THEN OLD.segment_id ELSE NEW.segment_id END;\n\n SELECT d.state, h.corridor_overflow, h.delete_after\n INTO dataset_state, parent_overflow, parent_delete_after\n FROM historical_bootstrap_holdout_segments h\n JOIN historical_bootstrap_datasets d ON d.dataset_id=h.dataset_id\n WHERE h.dataset_id=target_dataset AND h.segment_id=target_segment;\n\n IF dataset_state IS NULL THEN\n IF TG_OP='DELETE' THEN RETURN OLD; END IF;\n RAISE EXCEPTION 'historical bootstrap holdout corridor requires existing holdout segment';\n END IF;\n\n IF TG_OP='DELETE' THEN\n IF dataset_state='EXPIRED' THEN RETURN OLD; END IF;\n IF dataset_state IN ('READY','RETIRED') AND parent_delete_after <= now() THEN RETURN OLD; END IF;\n RAISE EXCEPTION 'historical bootstrap holdout corridor is not retention-eligible for deletion';\n END IF;\n\n IF dataset_state NOT IN ('MANIFEST_LOCKED','DERIVING','STAGED') THEN\n RAISE EXCEPTION 'historical bootstrap holdout corridor cannot be staged in dataset state %', dataset_state;\n END IF;\n IF parent_overflow THEN\n RAISE EXCEPTION 'corridor-overflow HOLDOUT segment cannot accept a biased partial corridor';\n END IF;\n RETURN NEW;\nEND $$;\n\nCREATE TRIGGER trg_hb_holdout_corridor_mutation_guard\nBEFORE INSERT OR UPDATE OR DELETE ON historical_bootstrap_holdout_corridor_cells\nFOR EACH ROW EXECUTE FUNCTION historical_bootstrap_holdout_corridor_mutation_guard();\n\n-- Extend the lifecycle transition guard with v3 corridor completeness checks before STAGED/READY exposure.\nCREATE OR REPLACE FUNCTION historical_bootstrap_dataset_update_guard()\nRETURNS trigger LANGUAGE plpgsql AS $$\nBEGIN\n IF (to_jsonb(NEW) - ARRAY['state','ready_at','retired_at']) IS DISTINCT FROM\n (to_jsonb(OLD) - ARRAY['state','ready_at','retired_at']) THEN\n RAISE EXCEPTION 'historical bootstrap dataset identity is immutable';\n END IF;\n IF NEW.state = OLD.state THEN\n IF NEW.ready_at IS DISTINCT FROM OLD.ready_at OR NEW.retired_at IS DISTINCT FROM OLD.retired_at THEN\n RAISE EXCEPTION 'historical bootstrap lifecycle timestamps require a state transition';\n END IF;\n RETURN NEW;\n END IF;\n IF NOT ((OLD.state='PLANNED' AND NEW.state='MANIFEST_LOCKED')\n OR (OLD.state='MANIFEST_LOCKED' AND NEW.state='DERIVING')\n OR (OLD.state='DERIVING' AND NEW.state='STAGED')\n OR (OLD.state='STAGED' AND NEW.state='READY')\n OR (OLD.state='READY' AND NEW.state='RETIRED')\n OR (OLD.state='RETIRED' AND NEW.state='EXPIRED')) THEN\n RAISE EXCEPTION 'invalid historical bootstrap dataset state transition: % -> %', OLD.state, NEW.state;\n END IF;\n\n IF OLD.state='DERIVING' AND NEW.state='STAGED' THEN\n IF EXISTS (\n SELECT 1\n FROM historical_bootstrap_holdout_segments h\n WHERE h.dataset_id=NEW.dataset_id\n AND h.corridor_overflow=TRUE\n AND EXISTS (\n SELECT 1 FROM historical_bootstrap_holdout_corridor_cells c\n WHERE c.dataset_id=h.dataset_id AND c.segment_id=h.segment_id\n )\n ) THEN\n RAISE EXCEPTION 'overflow HOLDOUT segment contains forbidden partial corridor';\n END IF;\n\n IF EXISTS (\n SELECT 1\n FROM historical_bootstrap_holdout_segments h\n WHERE h.dataset_id=NEW.dataset_id\n AND h.corridor_overflow=FALSE\n AND NOT EXISTS (\n SELECT 1 FROM historical_bootstrap_holdout_corridor_cells c\n WHERE c.dataset_id=h.dataset_id AND c.segment_id=h.segment_id\n )\n ) THEN\n RAISE EXCEPTION 'complete HOLDOUT corridor flag requires at least one corridor cell';\n END IF;\n\n IF EXISTS (\n SELECT 1\n FROM historical_bootstrap_holdout_segments h\n JOIN (\n SELECT dataset_id,segment_id,count(*) AS cell_count,min(sequence_index) AS min_seq,max(sequence_index) AS max_seq\n FROM historical_bootstrap_holdout_corridor_cells\n WHERE dataset_id=NEW.dataset_id\n GROUP BY dataset_id,segment_id\n ) c ON c.dataset_id=h.dataset_id AND c.segment_id=h.segment_id\n WHERE h.dataset_id=NEW.dataset_id\n AND h.corridor_overflow=FALSE\n AND (c.cell_count > 512 OR c.min_seq <> 0 OR c.max_seq <> c.cell_count - 1)\n ) THEN\n RAISE EXCEPTION 'HOLDOUT corridor sequence must be contiguous, zero-based and bounded to 512 cells';\n END IF;\n END IF;\n\n IF NEW.state='READY' AND (NEW.ready_at IS NULL OR NEW.retired_at IS NOT NULL) THEN\n RAISE EXCEPTION 'READY requires ready_at and no retired_at';\n ELSIF NEW.state='RETIRED' AND (NEW.ready_at IS DISTINCT FROM OLD.ready_at OR NEW.ready_at IS NULL OR NEW.retired_at IS NULL) THEN\n RAISE EXCEPTION 'RETIRED requires immutable ready_at and retired_at';\n ELSIF NEW.state='EXPIRED' AND (NEW.ready_at IS DISTINCT FROM OLD.ready_at OR NEW.retired_at IS DISTINCT FROM OLD.retired_at OR NEW.ready_at IS NULL OR NEW.retired_at IS NULL) THEN\n RAISE EXCEPTION 'EXPIRED preserves ready/retired timestamps';\n ELSIF NEW.state IN ('PLANNED','MANIFEST_LOCKED','DERIVING','STAGED') AND (NEW.ready_at IS NOT NULL OR NEW.retired_at IS NOT NULL) THEN\n RAISE EXCEPTION 'pre-READY state cannot have terminal timestamps';\n END IF;\n RETURN NEW;\nEND $$;\n\nCREATE VIEW historical_bootstrap_ready_route_corridor_aggregates AS\nSELECT c.*\n FROM historical_bootstrap_route_corridor_aggregates c\n JOIN historical_bootstrap_datasets d ON d.dataset_id=c.dataset_id\n WHERE d.state='READY';\n\nCOMMENT ON VIEW historical_bootstrap_ready_route_corridor_aggregates IS\n 'Learning/scoring R7 corridor aggregates only; HOLDOUT corridor truth is deliberately excluded from this READY read seam';\n" }, { "path": "backend/truth-model/src/main/resources/db/migration/V72__t24_forecast_shadow_foundation.sql", "encoding": "UTF-8", "sha256": "d31a9089a16f612acb7b40f135205a2ac4b0ddd322a32addfc8cedef119f36d2", "content": "CREATE TABLE flight_forecast_models (\n model_id UUID PRIMARY KEY,\n model_fingerprint CHAR(64) NOT NULL UNIQUE CHECK (model_fingerprint ~ '^[0-9a-f]{64}$'),\n algorithm_version VARCHAR(80) NOT NULL,\n target_contract_version VARCHAR(80) NOT NULL,\n arithmetic_version VARCHAR(80) NOT NULL,\n schema_generation SMALLINT NOT NULL CHECK (schema_generation >= 1),\n p7_dataset_id UUID NOT NULL,\n p7_dataset_purpose VARCHAR(32) NOT NULL CHECK (p7_dataset_purpose IN ('QUALIFICATION_EXPERIMENT','BOOTSTRAP_SCORING')),\n p7_manifest_sha256 CHAR(64) NOT NULL CHECK (p7_manifest_sha256 ~ '^[0-9a-f]{64}$'),\n p7_semantic_fingerprint CHAR(64) NOT NULL CHECK (p7_semantic_fingerprint ~ '^[0-9a-f]{64}$'),\n p7_coverage_version VARCHAR(80) NOT NULL,\n p7_coverage_hash CHAR(64) NOT NULL CHECK (p7_coverage_hash ~ '^[0-9a-f]{64}$'),\n p7_importer_version VARCHAR(80) NOT NULL,\n p7_parser_version VARCHAR(80) NOT NULL,\n p7_eligibility_version VARCHAR(80) NOT NULL,\n p7_segmentation_version VARCHAR(80) NOT NULL,\n p7_derivation_version VARCHAR(80) NOT NULL,\n p6_model_version SMALLINT CHECK (p6_model_version >= 1),\n fit_from DATE NOT NULL,\n fit_to DATE NOT NULL,\n calibration_from DATE NOT NULL,\n calibration_to DATE NOT NULL,\n embargo_date DATE NOT NULL,\n canonical_config JSONB NOT NULL,\n canonical_config_sha256 CHAR(64) NOT NULL CHECK (canonical_config_sha256 ~ '^[0-9a-f]{64}$'),\n state VARCHAR(24) NOT NULL CHECK (state IN ('DRAFT','FROZEN','HOLDOUT_EVALUATED','QUALIFIED','REJECTED','RETIRED')),\n created_at TIMESTAMPTZ NOT NULL,\n frozen_at TIMESTAMPTZ,\n holdout_evaluated_at TIMESTAMPTZ,\n qualified_at TIMESTAMPTZ,\n rejected_at TIMESTAMPTZ,\n retired_at TIMESTAMPTZ,\n CONSTRAINT ck_forecast_model_windows CHECK (\n fit_from <= fit_to AND fit_to < calibration_from AND calibration_from <= calibration_to\n AND calibration_to < embargo_date\n ),\n CONSTRAINT ck_forecast_recurrence_v1_p6 CHECK (\n NOT (algorithm_version='recurrence-v1' AND schema_generation=1 AND p6_model_version IS NOT NULL)\n ),\n CONSTRAINT ck_forecast_model_state_times CHECK (\n (state='DRAFT' AND frozen_at IS NULL AND holdout_evaluated_at IS NULL AND qualified_at IS NULL AND rejected_at IS NULL AND retired_at IS NULL)\n OR (state='FROZEN' AND frozen_at IS NOT NULL AND holdout_evaluated_at IS NULL AND qualified_at IS NULL AND rejected_at IS NULL AND retired_at IS NULL)\n OR (state='HOLDOUT_EVALUATED' AND frozen_at IS NOT NULL AND holdout_evaluated_at IS NOT NULL AND qualified_at IS NULL AND rejected_at IS NULL AND retired_at IS NULL)\n OR (state='QUALIFIED' AND frozen_at IS NOT NULL AND holdout_evaluated_at IS NOT NULL AND qualified_at IS NOT NULL AND rejected_at IS NULL AND retired_at IS NULL)\n OR (state='REJECTED' AND frozen_at IS NOT NULL AND rejected_at IS NOT NULL AND qualified_at IS NULL AND retired_at IS NULL)\n OR (state='RETIRED' AND frozen_at IS NOT NULL AND retired_at IS NOT NULL)\n ),\n CONSTRAINT ck_forecast_model_timestamp_order CHECK (\n frozen_at IS NULL OR frozen_at >= created_at\n AND (holdout_evaluated_at IS NULL OR holdout_evaluated_at >= frozen_at)\n AND (qualified_at IS NULL OR qualified_at >= holdout_evaluated_at)\n AND (rejected_at IS NULL OR rejected_at >= frozen_at AND (holdout_evaluated_at IS NULL OR rejected_at >= holdout_evaluated_at))\n AND (retired_at IS NULL OR retired_at >= created_at AND (frozen_at IS NULL OR retired_at >= frozen_at)\n AND (holdout_evaluated_at IS NULL OR retired_at >= holdout_evaluated_at)\n AND (qualified_at IS NULL OR retired_at >= qualified_at) AND (rejected_at IS NULL OR retired_at >= rejected_at))\n )\n);\n\nCREATE TABLE flight_forecast_runs (\n run_id UUID PRIMARY KEY,\n run_key CHAR(64) NOT NULL UNIQUE CHECK (run_key ~ '^[0-9a-f]{64}$'),\n model_id UUID NOT NULL REFERENCES flight_forecast_models(model_id),\n purpose VARCHAR(24) NOT NULL CHECK (purpose IN ('LEARNING_CALIBRATION','HOLDOUT_BACKTEST','LIVE_SHADOW')),\n state VARCHAR(20) NOT NULL CHECK (state IN ('WRITING','COMPLETE','FAILED','ABSTAINED_LATE','EXPIRED')),\n scheduled_as_of TIMESTAMPTZ NOT NULL,\n target_bucket_start TIMESTAMPTZ NOT NULL,\n target_bucket_end TIMESTAMPTZ NOT NULL,\n generated_at TIMESTAMPTZ NOT NULL,\n actual_lead_time_seconds INTEGER,\n evidence_cutoff_at TIMESTAMPTZ NOT NULL,\n evidence_snapshot_fingerprint CHAR(64) NOT NULL CHECK (evidence_snapshot_fingerprint ~ '^[0-9a-f]{64}$'),\n source_pattern_count BIGINT NOT NULL CHECK (source_pattern_count >= 0),\n source_hour_bin_count BIGINT NOT NULL CHECK (source_hour_bin_count >= 0),\n source_corridor_count BIGINT NOT NULL CHECK (source_corridor_count >= 0),\n candidate_count INTEGER NOT NULL DEFAULT 0 CHECK (candidate_count BETWEEN 0 AND 40000),\n abstention_count INTEGER NOT NULL DEFAULT 0 CHECK (abstention_count >= 0),\n output_count INTEGER NOT NULL DEFAULT 0 CHECK (output_count BETWEEN 0 AND 10000),\n result_sha256 CHAR(64) CHECK (result_sha256 IS NULL OR result_sha256 ~ '^[0-9a-f]{64}$'),\n runtime_millis BIGINT CHECK (runtime_millis IS NULL OR runtime_millis >= 0),\n peak_candidate_heap_bytes BIGINT CHECK (peak_candidate_heap_bytes IS NULL OR peak_candidate_heap_bytes >= 0),\n completed_at TIMESTAMPTZ,\n delete_after TIMESTAMPTZ,\n cleanup_authorized_at TIMESTAMPTZ,\n CONSTRAINT ck_forecast_run_hour_contract CHECK (\n scheduled_as_of = date_trunc('hour', scheduled_as_of)\n AND target_bucket_start = date_trunc('hour', target_bucket_start)\n AND target_bucket_end = date_trunc('hour', target_bucket_end)\n AND target_bucket_start = scheduled_as_of + interval '24 hours'\n AND target_bucket_end = target_bucket_start + interval '1 hour'\n ),\n CONSTRAINT ck_forecast_run_output_within_candidates CHECK (output_count <= candidate_count),\n CONSTRAINT ck_forecast_run_completion_after_generation CHECK (completed_at IS NULL OR completed_at >= generated_at),\n CONSTRAINT ck_forecast_run_generated_seconds CHECK (generated_at = date_trunc('second', generated_at)),\n CONSTRAINT ck_forecast_run_purpose_lead CHECK (\n (purpose='LIVE_SHADOW' AND actual_lead_time_seconds IS NOT NULL\n AND actual_lead_time_seconds = extract(epoch FROM (target_bucket_start-generated_at))::INTEGER)\n OR (purpose IN ('LEARNING_CALIBRATION','HOLDOUT_BACKTEST') AND actual_lead_time_seconds IS NULL)\n ),\n CONSTRAINT ck_forecast_run_live_retention CHECK (\n (purpose='LIVE_SHADOW' AND delete_after = generated_at + interval '7 days')\n OR (purpose IN ('LEARNING_CALIBRATION','HOLDOUT_BACKTEST') AND delete_after IS NULL)\n ),\n CONSTRAINT ck_forecast_run_state CHECK (\n (state='WRITING' AND completed_at IS NULL AND result_sha256 IS NULL)\n OR (state='COMPLETE' AND completed_at IS NOT NULL AND result_sha256 IS NOT NULL\n AND (purpose<>'LIVE_SHADOW' OR actual_lead_time_seconds BETWEEN 84600 AND 88200))\n OR (state='FAILED' AND completed_at IS NOT NULL AND result_sha256 IS NULL)\n OR (state='ABSTAINED_LATE' AND purpose='LIVE_SHADOW' AND completed_at IS NOT NULL\n AND result_sha256 IS NULL AND NOT (actual_lead_time_seconds BETWEEN 84600 AND 88200)\n AND candidate_count=0 AND output_count=0)\n OR (state='EXPIRED' AND completed_at IS NOT NULL)\n )\n);\n\nCREATE TABLE flight_forecast_shadow (\n run_id UUID NOT NULL REFERENCES flight_forecast_runs(run_id) ON DELETE CASCADE,\n candidate_id UUID NOT NULL,\n target_bucket_start TIMESTAMPTZ NOT NULL,\n start_h3_r5 VARCHAR(15) NOT NULL CHECK (start_h3_r5 ~ '^85[0-9a-f]{13}$'),\n end_h3_r5 VARCHAR(15) NOT NULL CHECK (end_h3_r5 ~ '^85[0-9a-f]{13}$'),\n corridor_h3_r7 VARCHAR(15) NOT NULL CHECK (corridor_h3_r7 ~ '^87[0-9a-f]{13}$'),\n raw_score NUMERIC(12,10) NOT NULL CHECK (raw_score BETWEEN 0 AND 1),\n confidence NUMERIC(12,10) NOT NULL CHECK (confidence BETWEEN 0 AND 1),\n calibration_lower NUMERIC(12,10) NOT NULL CHECK (calibration_lower BETWEEN 0 AND 1),\n calibration_upper NUMERIC(12,10) NOT NULL CHECK (calibration_upper BETWEEN calibration_lower AND 1),\n route_support BIGINT NOT NULL CHECK (route_support >= 2),\n qualified_support BIGINT NOT NULL CHECK (qualified_support BETWEEN 0 AND route_support),\n hour_support BIGINT NOT NULL CHECK (hour_support BETWEEN 1 AND route_support),\n corridor_support BIGINT NOT NULL CHECK (corridor_support BETWEEN 1 AND route_support),\n uncertainty_class VARCHAR(24) NOT NULL CHECK (uncertainty_class IN ('SUPPORTED','LOW_SUPPORT')),\n rank_in_bucket INTEGER NOT NULL CHECK (rank_in_bucket BETWEEN 1 AND 10000),\n PRIMARY KEY (run_id, candidate_id),\n UNIQUE (run_id, target_bucket_start, start_h3_r5, end_h3_r5, corridor_h3_r7),\n UNIQUE (run_id, target_bucket_start, rank_in_bucket)\n);\n\nCREATE TABLE flight_forecast_candidate_events (\n candidate_id UUID NOT NULL,\n run_id UUID NOT NULL,\n event_sequence INTEGER NOT NULL CHECK (event_sequence BETWEEN 0 AND 1000),\n event_id UUID NOT NULL,\n event_type VARCHAR(20) NOT NULL CHECK (event_type IN ('GENERATED','STRENGTHENED','WEAKENED','INVALIDATED','OBSERVED','MISSED')),\n event_at TIMESTAMPTZ NOT NULL,\n reason_code VARCHAR(64),\n evaluator_fingerprint CHAR(64) CHECK (evaluator_fingerprint IS NULL OR evaluator_fingerprint ~ '^[0-9a-f]{64}$'),\n previous_event_sha256 CHAR(64) CHECK (previous_event_sha256 IS NULL OR previous_event_sha256 ~ '^[0-9a-f]{64}$'),\n event_sha256 CHAR(64) NOT NULL CHECK (event_sha256 ~ '^[0-9a-f]{64}$'),\n PRIMARY KEY (candidate_id, event_sequence),\n UNIQUE (candidate_id, event_id),\n FOREIGN KEY (run_id, candidate_id) REFERENCES flight_forecast_shadow(run_id, candidate_id) ON DELETE CASCADE\n);\n\nCREATE INDEX ix_forecast_model_qualified ON flight_forecast_models(state, model_id);\nCREATE INDEX ix_forecast_run_target ON flight_forecast_runs(target_bucket_start, state, model_id);\nCREATE INDEX ix_forecast_run_expiry ON flight_forecast_runs(delete_after, run_id) WHERE delete_after IS NOT NULL;\nCREATE INDEX ix_forecast_shadow_target_rank ON flight_forecast_shadow(target_bucket_start, rank_in_bucket);\nCREATE INDEX ix_forecast_candidate_event_run ON flight_forecast_candidate_events(run_id, candidate_id);\n\nCREATE OR REPLACE FUNCTION flight_forecast_model_insert_delete_guard()\nRETURNS trigger LANGUAGE plpgsql AS $$\nBEGIN\n IF TG_OP='INSERT' AND NEW.state<>'DRAFT' THEN\n RAISE EXCEPTION 'forecast model must start DRAFT';\n ELSIF TG_OP='DELETE' THEN\n RAISE EXCEPTION 'forecast model deletion is prohibited; use RETIRED';\n END IF;\n RETURN NEW;\nEND $$;\nCREATE TRIGGER trg_forecast_model_insert_delete_guard BEFORE INSERT OR DELETE ON flight_forecast_models\nFOR EACH ROW EXECUTE FUNCTION flight_forecast_model_insert_delete_guard();\n\nCREATE OR REPLACE FUNCTION flight_forecast_model_update_guard()\nRETURNS trigger LANGUAGE plpgsql AS $$\nDECLARE p7_ok BOOLEAN;\nBEGIN\n IF NEW.created_at IS DISTINCT FROM OLD.created_at\n OR (OLD.frozen_at IS NOT NULL AND NEW.frozen_at IS DISTINCT FROM OLD.frozen_at)\n OR (OLD.holdout_evaluated_at IS NOT NULL AND NEW.holdout_evaluated_at IS DISTINCT FROM OLD.holdout_evaluated_at)\n OR (OLD.qualified_at IS NOT NULL AND NEW.qualified_at IS DISTINCT FROM OLD.qualified_at)\n OR (OLD.rejected_at IS NOT NULL AND NEW.rejected_at IS DISTINCT FROM OLD.rejected_at)\n OR (OLD.retired_at IS NOT NULL AND NEW.retired_at IS DISTINCT FROM OLD.retired_at) THEN\n RAISE EXCEPTION 'established forecast model lifecycle timestamps are immutable';\n END IF;\n IF OLD.state <> 'DRAFT' AND\n (to_jsonb(NEW) - ARRAY['state','holdout_evaluated_at','qualified_at','rejected_at','retired_at']) IS DISTINCT FROM\n (to_jsonb(OLD) - ARRAY['state','holdout_evaluated_at','qualified_at','rejected_at','retired_at']) THEN\n RAISE EXCEPTION 'frozen forecast model identity/configuration is immutable';\n END IF;\n IF NEW.state IN ('FROZEN','QUALIFIED') AND NEW.state <> OLD.state THEN\n SELECT d.state='READY' AND d.dataset_purpose='QUALIFICATION_EXPERIMENT'\n AND NEW.p7_dataset_purpose=d.dataset_purpose\n AND d.manifest_sha256=NEW.p7_manifest_sha256 AND d.semantic_fingerprint=NEW.p7_semantic_fingerprint\n AND d.coverage_version=NEW.p7_coverage_version AND d.coverage_hash=NEW.p7_coverage_hash\n AND d.importer_version=NEW.p7_importer_version AND d.parser_version=NEW.p7_parser_version\n AND d.eligibility_version=NEW.p7_eligibility_version AND d.segmentation_version=NEW.p7_segmentation_version\n AND d.derivation_version=NEW.p7_derivation_version\n AND NEW.fit_from >= d.from_date AND NEW.calibration_to <= d.learning_end\n AND NEW.embargo_date=d.embargo_date\n INTO p7_ok FROM historical_bootstrap_datasets d WHERE d.dataset_id=NEW.p7_dataset_id;\n IF p7_ok IS DISTINCT FROM TRUE THEN RAISE EXCEPTION 'forecast model requires exact READY P7 learning provenance and chronology'; END IF;\n END IF;\n IF NEW.state = OLD.state THEN\n IF NEW.frozen_at IS DISTINCT FROM OLD.frozen_at\n OR NEW.holdout_evaluated_at IS DISTINCT FROM OLD.holdout_evaluated_at\n OR NEW.qualified_at IS DISTINCT FROM OLD.qualified_at\n OR NEW.rejected_at IS DISTINCT FROM OLD.rejected_at\n OR NEW.retired_at IS DISTINCT FROM OLD.retired_at THEN\n RAISE EXCEPTION 'forecast model lifecycle timestamps require a state transition';\n END IF;\n RETURN NEW;\n END IF;\n IF NOT ((OLD.state='DRAFT' AND NEW.state='FROZEN')\n OR (OLD.state='FROZEN' AND NEW.state IN ('HOLDOUT_EVALUATED','REJECTED','RETIRED'))\n OR (OLD.state='HOLDOUT_EVALUATED' AND NEW.state IN ('QUALIFIED','REJECTED','RETIRED'))\n OR (OLD.state IN ('QUALIFIED','REJECTED') AND NEW.state='RETIRED')) THEN\n RAISE EXCEPTION 'invalid forecast model state transition: % -> %', OLD.state, NEW.state;\n END IF;\n IF OLD.state='DRAFT' AND NEW.state='FROZEN' THEN\n IF NEW.holdout_evaluated_at IS DISTINCT FROM OLD.holdout_evaluated_at\n OR NEW.qualified_at IS DISTINCT FROM OLD.qualified_at\n OR NEW.rejected_at IS DISTINCT FROM OLD.rejected_at\n OR NEW.retired_at IS DISTINCT FROM OLD.retired_at THEN\n RAISE EXCEPTION 'DRAFT -> FROZEN may only introduce frozen_at';\n END IF;\n ELSIF OLD.state='FROZEN' AND NEW.state='HOLDOUT_EVALUATED' THEN\n IF NEW.qualified_at IS DISTINCT FROM OLD.qualified_at\n OR NEW.rejected_at IS DISTINCT FROM OLD.rejected_at\n OR NEW.retired_at IS DISTINCT FROM OLD.retired_at THEN\n RAISE EXCEPTION 'FROZEN -> HOLDOUT_EVALUATED may only introduce holdout_evaluated_at';\n END IF;\n ELSIF OLD.state='FROZEN' AND NEW.state='REJECTED' THEN\n IF NEW.holdout_evaluated_at IS DISTINCT FROM OLD.holdout_evaluated_at\n OR NEW.qualified_at IS DISTINCT FROM OLD.qualified_at\n OR NEW.retired_at IS DISTINCT FROM OLD.retired_at THEN\n RAISE EXCEPTION 'FROZEN -> REJECTED may only introduce rejected_at';\n END IF;\n ELSIF OLD.state='FROZEN' AND NEW.state='RETIRED' THEN\n IF NEW.holdout_evaluated_at IS DISTINCT FROM OLD.holdout_evaluated_at\n OR NEW.qualified_at IS DISTINCT FROM OLD.qualified_at\n OR NEW.rejected_at IS DISTINCT FROM OLD.rejected_at THEN\n RAISE EXCEPTION 'FROZEN -> RETIRED may only introduce retired_at';\n END IF;\n ELSIF OLD.state='HOLDOUT_EVALUATED' AND NEW.state='QUALIFIED' THEN\n IF NEW.rejected_at IS DISTINCT FROM OLD.rejected_at\n OR NEW.retired_at IS DISTINCT FROM OLD.retired_at THEN\n RAISE EXCEPTION 'HOLDOUT_EVALUATED -> QUALIFIED may only introduce qualified_at';\n END IF;\n ELSIF OLD.state='HOLDOUT_EVALUATED' AND NEW.state='REJECTED' THEN\n IF NEW.qualified_at IS DISTINCT FROM OLD.qualified_at\n OR NEW.retired_at IS DISTINCT FROM OLD.retired_at THEN\n RAISE EXCEPTION 'HOLDOUT_EVALUATED -> REJECTED may only introduce rejected_at';\n END IF;\n ELSIF OLD.state='HOLDOUT_EVALUATED' AND NEW.state='RETIRED' THEN\n IF NEW.qualified_at IS DISTINCT FROM OLD.qualified_at\n OR NEW.rejected_at IS DISTINCT FROM OLD.rejected_at THEN\n RAISE EXCEPTION 'HOLDOUT_EVALUATED -> RETIRED may only introduce retired_at';\n END IF;\n ELSIF OLD.state IN ('QUALIFIED','REJECTED') AND NEW.state='RETIRED' THEN\n IF NEW.holdout_evaluated_at IS DISTINCT FROM OLD.holdout_evaluated_at\n OR NEW.qualified_at IS DISTINCT FROM OLD.qualified_at\n OR NEW.rejected_at IS DISTINCT FROM OLD.rejected_at THEN\n RAISE EXCEPTION 'terminal model -> RETIRED may only introduce retired_at';\n END IF;\n END IF;\n RETURN NEW;\nEND $$;\nCREATE TRIGGER trg_forecast_model_update_guard BEFORE UPDATE ON flight_forecast_models\nFOR EACH ROW EXECUTE FUNCTION flight_forecast_model_update_guard();\n\nCREATE OR REPLACE FUNCTION flight_forecast_run_insert_delete_guard()\nRETURNS trigger LANGUAGE plpgsql AS $$\nBEGIN\n IF TG_OP='INSERT' AND NEW.state<>'WRITING' THEN\n RAISE EXCEPTION 'forecast run must start WRITING';\n ELSIF TG_OP='DELETE' THEN\n IF OLD.state='EXPIRED' THEN RETURN OLD; END IF;\n RAISE EXCEPTION 'forecast run deletion requires EXPIRED state';\n END IF;\n RETURN COALESCE(NEW,OLD);\nEND $$;\nCREATE TRIGGER trg_forecast_run_insert_delete_guard BEFORE INSERT OR DELETE ON flight_forecast_runs\nFOR EACH ROW EXECUTE FUNCTION flight_forecast_run_insert_delete_guard();\n\nCREATE OR REPLACE FUNCTION flight_forecast_run_update_guard()\nRETURNS trigger LANGUAGE plpgsql AS $$\nDECLARE snapshot_count BIGINT; generated_count BIGINT;\nBEGIN\n IF (to_jsonb(NEW) - ARRAY['state','candidate_count','abstention_count','output_count','result_sha256','runtime_millis','peak_candidate_heap_bytes','completed_at','cleanup_authorized_at']) IS DISTINCT FROM\n (to_jsonb(OLD) - ARRAY['state','candidate_count','abstention_count','output_count','result_sha256','runtime_millis','peak_candidate_heap_bytes','completed_at','cleanup_authorized_at']) THEN\n RAISE EXCEPTION 'forecast run identity and time contract are immutable';\n END IF;\n IF OLD.state <> 'WRITING' AND NEW IS DISTINCT FROM OLD THEN\n IF NEW.state=OLD.state AND OLD.purpose IN ('LEARNING_CALIBRATION','HOLDOUT_BACKTEST')\n AND OLD.cleanup_authorized_at IS NULL AND NEW.cleanup_authorized_at IS NOT NULL\n AND (to_jsonb(NEW)-'cleanup_authorized_at') IS NOT DISTINCT FROM\n (to_jsonb(OLD)-'cleanup_authorized_at') THEN\n RETURN NEW;\n ELSIF NEW.state='EXPIRED' AND OLD.state IN ('COMPLETE','FAILED','ABSTAINED_LATE')\n AND (to_jsonb(NEW)-'state') IS NOT DISTINCT FROM (to_jsonb(OLD)-'state') THEN\n NULL;\n ELSE\n RAISE EXCEPTION 'terminal forecast run is immutable except expiry';\n END IF;\n END IF;\n IF NEW.state <> OLD.state AND NOT (\n (OLD.state='WRITING' AND NEW.state IN ('COMPLETE','FAILED','ABSTAINED_LATE'))\n OR (OLD.state IN ('COMPLETE','FAILED','ABSTAINED_LATE') AND NEW.state='EXPIRED')) THEN\n RAISE EXCEPTION 'invalid forecast run state transition: % -> %', OLD.state, NEW.state;\n END IF;\n IF OLD.state='WRITING' AND NEW.state IN ('COMPLETE','ABSTAINED_LATE') THEN\n SELECT count(*) INTO snapshot_count FROM flight_forecast_shadow WHERE run_id=NEW.run_id;\n SELECT count(*) INTO generated_count FROM flight_forecast_candidate_events\n WHERE run_id=NEW.run_id AND event_type='GENERATED';\n IF NEW.state='COMPLETE' AND (snapshot_count <> NEW.output_count OR generated_count <> snapshot_count) THEN\n RAISE EXCEPTION 'COMPLETE run counts/generated events do not match snapshots';\n END IF;\n IF NEW.state='ABSTAINED_LATE' AND (snapshot_count <> 0 OR generated_count <> 0) THEN\n RAISE EXCEPTION 'ABSTAINED_LATE run cannot retain predictions or GENERATED events';\n END IF;\n END IF;\n IF NEW.state='EXPIRED' AND OLD.state<>'EXPIRED' THEN\n IF NEW.purpose='LIVE_SHADOW' AND (NEW.delete_after IS NULL OR NEW.delete_after > now()) THEN\n RAISE EXCEPTION 'LIVE_SHADOW run is not retention-eligible';\n ELSIF NEW.purpose IN ('LEARNING_CALIBRATION','HOLDOUT_BACKTEST')\n AND (NEW.cleanup_authorized_at IS NULL OR NEW.cleanup_authorized_at > now()) THEN\n RAISE EXCEPTION 'historical qualification evidence requires explicit cleanup authorization';\n END IF;\n END IF;\n RETURN NEW;\nEND $$;\nCREATE TRIGGER trg_forecast_run_update_guard BEFORE UPDATE ON flight_forecast_runs\nFOR EACH ROW EXECUTE FUNCTION flight_forecast_run_update_guard();\n\nCREATE OR REPLACE FUNCTION flight_forecast_snapshot_mutation_guard()\nRETURNS trigger LANGUAGE plpgsql AS $$\nDECLARE run_state VARCHAR(20); run_target TIMESTAMPTZ;\nBEGIN\n IF TG_OP='UPDATE' THEN RAISE EXCEPTION 'forecast prediction snapshots are immutable'; END IF;\n SELECT state,target_bucket_start INTO run_state,run_target FROM flight_forecast_runs\n WHERE run_id=CASE WHEN TG_OP='DELETE' THEN OLD.run_id ELSE NEW.run_id END;\n IF TG_OP='DELETE' THEN\n IF run_state='EXPIRED' OR run_state IS NULL THEN RETURN OLD; END IF;\n RAISE EXCEPTION 'forecast snapshot deletion requires EXPIRED run';\n END IF;\n IF run_state <> 'WRITING' THEN RAISE EXCEPTION 'forecast snapshots can only be staged for WRITING runs'; END IF;\n IF NEW.target_bucket_start IS DISTINCT FROM run_target THEN RAISE EXCEPTION 'snapshot target differs from run target'; END IF;\n RETURN NEW;\nEND $$;\nCREATE TRIGGER trg_forecast_snapshot_mutation_guard BEFORE INSERT OR UPDATE OR DELETE ON flight_forecast_shadow\nFOR EACH ROW EXECUTE FUNCTION flight_forecast_snapshot_mutation_guard();\n\nCREATE OR REPLACE FUNCTION flight_forecast_event_mutation_guard()\nRETURNS trigger LANGUAGE plpgsql AS $$\nDECLARE run_state VARCHAR(20); run_purpose VARCHAR(24); previous_type VARCHAR(20); previous_hash CHAR(64); expected_sequence INTEGER;\nBEGIN\n IF TG_OP='UPDATE' THEN RAISE EXCEPTION 'forecast candidate events are append-only'; END IF;\n SELECT state,purpose INTO run_state,run_purpose FROM flight_forecast_runs\n WHERE run_id=CASE WHEN TG_OP='DELETE' THEN OLD.run_id ELSE NEW.run_id END;\n IF TG_OP='DELETE' THEN\n IF run_state='EXPIRED' OR run_state IS NULL THEN RETURN OLD; END IF;\n RAISE EXCEPTION 'forecast candidate event deletion requires EXPIRED run';\n END IF;\n SELECT event_type,event_sha256,event_sequence+1 INTO previous_type,previous_hash,expected_sequence\n FROM flight_forecast_candidate_events WHERE candidate_id=NEW.candidate_id\n ORDER BY event_sequence DESC LIMIT 1;\n IF previous_type IS NULL THEN\n IF NEW.event_sequence<>0 OR NEW.event_type<>'GENERATED' OR run_state<>'WRITING'\n OR NEW.previous_event_sha256 IS NOT NULL THEN\n RAISE EXCEPTION 'first candidate event must be GENERATED while run is WRITING';\n END IF;\n ELSE\n IF NEW.event_sequence<>expected_sequence OR NEW.previous_event_sha256 IS DISTINCT FROM previous_hash THEN\n RAISE EXCEPTION 'candidate event sequence/previous-hash linkage is invalid';\n END IF;\n IF run_purpose<>'LIVE_SHADOW' THEN\n RAISE EXCEPTION 'historical candidate outcomes must remain aggregate-only';\n END IF;\n IF run_state<>'COMPLETE' OR previous_type<>'GENERATED'\n OR NEW.event_type NOT IN ('OBSERVED','MISSED','INVALIDATED') THEN\n RAISE EXCEPTION 'candidate transition is not active in recurrence-only v1';\n END IF;\n END IF;\n RETURN NEW;\nEND $$;\nCREATE TRIGGER trg_forecast_event_mutation_guard BEFORE INSERT OR UPDATE OR DELETE ON flight_forecast_candidate_events\nFOR EACH ROW EXECUTE FUNCTION flight_forecast_event_mutation_guard();\n\nCREATE VIEW flight_forecast_complete_shadow AS\nSELECT s.*,r.model_id,r.purpose,r.scheduled_as_of,r.target_bucket_end,r.generated_at,r.actual_lead_time_seconds\n FROM flight_forecast_shadow s JOIN flight_forecast_runs r ON r.run_id=s.run_id\n WHERE r.state='COMPLETE';\n\nCREATE VIEW flight_forecast_qualified_shadow AS\nSELECT s.* FROM flight_forecast_complete_shadow s\n JOIN flight_forecast_models m ON m.model_id=s.model_id\n JOIN historical_bootstrap_datasets d ON d.dataset_id=m.p7_dataset_id\n WHERE s.purpose='LIVE_SHADOW' AND m.state='QUALIFIED' AND s.generated_at + interval '7 days' > now()\n AND d.state='READY' AND d.dataset_purpose='QUALIFICATION_EXPERIMENT' AND m.p7_dataset_purpose=d.dataset_purpose\n AND d.manifest_sha256=m.p7_manifest_sha256 AND d.semantic_fingerprint=m.p7_semantic_fingerprint\n AND d.coverage_version=m.p7_coverage_version AND d.coverage_hash=m.p7_coverage_hash\n AND d.importer_version=m.p7_importer_version AND d.parser_version=m.p7_parser_version\n AND d.eligibility_version=m.p7_eligibility_version AND d.segmentation_version=m.p7_segmentation_version\n AND d.derivation_version=m.p7_derivation_version;\n" }, { "path": "backend/truth-model/src/main/resources/db/migration/V73__qualified_learning_days_forecast_seam.sql", "encoding": "UTF-8", "sha256": "fb959a7a976ec5f8ebbc35a6b0c82874f743a167576f3be49a9817a968c50c9c", "content": "-- Narrow P8 recurrence denominator seam. It deliberately exposes no payload or outcome metadata.\nCREATE VIEW historical_bootstrap_ready_qualified_learning_days AS\nSELECT day.dataset_id, day.source_date\n FROM historical_bootstrap_days day\n JOIN historical_bootstrap_datasets dataset ON dataset.dataset_id=day.dataset_id\n WHERE dataset.state='READY'\n AND day.experiment_role='LEARNING'\n AND day.terminal_quality='QUALIFIED';\n\nCOMMENT ON VIEW historical_bootstrap_ready_qualified_learning_days IS\n 'READY QUALIFIED LEARNING dates only, for deterministic P8 recurrence denominators';\n" }, { "path": "backend/truth-model/src/main/resources/db/migration/V76__p862_live_shadow_prospective_evidence.sql", "encoding": "UTF-8", "sha256": "395846e0c325682d4a4cd13bc46a3538a0c5ee71a3262993475848c6b262859a", "content": "-- P8.6.2 prospective evidence only. No outbox, opportunity, notification or entitlement relationship exists.\nALTER TABLE observed_flight_summaries DROP CONSTRAINT ck_observed_flight_summary_schema;\nALTER TABLE observed_flight_summaries ADD CONSTRAINT ck_observed_flight_summary_schema\n CHECK (observer_schema_version IN (1,2));\nALTER TABLE observed_flight_summaries DROP CONSTRAINT ck_observed_flight_summary_quality;\nALTER TABLE observed_flight_summaries ADD CONSTRAINT ck_observed_flight_summary_quality CHECK (\n quality_flags <@ ARRAY['CORRIDOR_TRUNCATED','FORECAST_CORRIDOR_TRUNCATED','PROVIDER_TRUNCATED']::VARCHAR[]);\n\nCREATE TABLE observed_flight_summary_r7_cells (\n summary_id UUID NOT NULL REFERENCES observed_flight_summaries(summary_id) ON DELETE CASCADE,\n corridor_h3_r7 VARCHAR(15) NOT NULL CHECK (corridor_h3_r7 ~ '^87[0-9a-f]{13}$'),\n first_observed_at TIMESTAMPTZ NOT NULL,\n PRIMARY KEY(summary_id,corridor_h3_r7)\n);\nCREATE INDEX ix_observed_summary_r7_entry ON observed_flight_summary_r7_cells\n (first_observed_at,corridor_h3_r7,summary_id);\n\nCREATE TABLE p862_prospective_windows (\n window_id VARCHAR(96) PRIMARY KEY,\n experiment_id VARCHAR(96) NOT NULL CHECK (experiment_id='p862-live-shadow-prospective-v2'),\n started_at TIMESTAMPTZ NOT NULL CHECK (\n started_at AT TIME ZONE 'UTC'=date_trunc('day',started_at AT TIME ZONE 'UTC')),\n planned_end_at TIMESTAMPTZ NOT NULL CHECK (\n planned_end_at=(started_at AT TIME ZONE 'UTC'+interval '28 days') AT TIME ZONE 'UTC'),\n pipeline_fingerprint CHAR(64) NOT NULL CHECK (pipeline_fingerprint ~ '^[0-9a-f]{64}$'),\n truth_authority_fingerprint CHAR(64) NOT NULL CHECK (truth_authority_fingerprint ~ '^[0-9a-f]{64}$'),\n thinned_fingerprint CHAR(64) NOT NULL CHECK (thinned_fingerprint ~ '^[0-9a-f]{64}$'),\n candidate_generator_fingerprint CHAR(64) NOT NULL CHECK (candidate_generator_fingerprint ~ '^[0-9a-f]{64}$'),\n hourly_selector_fingerprint CHAR(64) NOT NULL CHECK (hourly_selector_fingerprint ~ '^[0-9a-f]{64}$'),\n daily_selector_fingerprint CHAR(64) NOT NULL CHECK (daily_selector_fingerprint ~ '^[0-9a-f]{64}$'),\n daily_decision_fingerprint CHAR(64) NOT NULL CHECK (daily_decision_fingerprint ~ '^[0-9a-f]{64}$'),\n feature_source_fingerprint CHAR(64) NOT NULL CHECK (feature_source_fingerprint ~ '^[0-9a-f]{64}$'),\n outcome_maturity_fingerprint CHAR(64) NOT NULL CHECK (outcome_maturity_fingerprint ~ '^[0-9a-f]{64}$'),\n metrics_fingerprint CHAR(64) NOT NULL CHECK (metrics_fingerprint ~ '^[0-9a-f]{64}$'),\n selection_contract VARCHAR(256) NOT NULL,\n metrics_contract VARCHAR(512) NOT NULL,\n status VARCHAR(16) NOT NULL CHECK (status IN ('PLANNED','ACTIVE','INVALIDATED','COMPLETE')),\n invalidation_reason VARCHAR(256),\n created_at TIMESTAMPTZ NOT NULL DEFAULT now(),\n CHECK (created_at<=started_at-interval '24 hours'),\n CHECK ((status='INVALIDATED')=(invalidation_reason IS NOT NULL))\n);\n\nCREATE TABLE p862_shadow_invocations (\n invocation_id UUID PRIMARY KEY,\n window_id VARCHAR(96) NOT NULL REFERENCES p862_prospective_windows(window_id),\n target_hour TIMESTAMPTZ NOT NULL CHECK (target_hour=date_trunc('hour',target_hour)),\n as_of TIMESTAMPTZ NOT NULL,\n generated_at TIMESTAMPTZ NOT NULL,\n status VARCHAR(40) NOT NULL CHECK (status IN ('FORECASTED','MISSED_CAUSAL_INVOCATION','CANDIDATE_GENERATION_FAILURE')),\n candidate_count INTEGER NOT NULL CHECK (candidate_count BETWEEN 0 AND 2),\n failure_reason VARCHAR(256),\n pipeline_fingerprint CHAR(64) NOT NULL CHECK (pipeline_fingerprint ~ '^[0-9a-f]{64}$'),\n truth_authority_fingerprint CHAR(64) NOT NULL CHECK (truth_authority_fingerprint ~ '^[0-9a-f]{64}$'),\n created_at TIMESTAMPTZ NOT NULL DEFAULT now(),\n UNIQUE(window_id,target_hour),\n CHECK (as_of=target_hour-interval '24 hours'),\n CHECK (generated_at>=as_of),\n CHECK (status<>'FORECASTED' OR generated_at<=as_of+interval '15 minutes'),\n CHECK ((status='FORECASTED')=(failure_reason IS NULL))\n);\n\nCREATE TABLE p862_hourly_candidates (\n candidate_id UUID PRIMARY KEY,\n window_id VARCHAR(96) NOT NULL REFERENCES p862_prospective_windows(window_id),\n target_hour TIMESTAMPTZ NOT NULL,\n as_of TIMESTAMPTZ NOT NULL,\n generated_at TIMESTAMPTZ NOT NULL,\n start_h3_r5 VARCHAR(15) NOT NULL CHECK (start_h3_r5 ~ '^85[0-9a-f]{13}$'),\n end_h3_r5 VARCHAR(15) NOT NULL CHECK (end_h3_r5 ~ '^85[0-9a-f]{13}$'),\n corridor_h3_r7 VARCHAR(15) NOT NULL CHECK (corridor_h3_r7 ~ '^87[0-9a-f]{13}$'),\n route_key VARCHAR(31) NOT NULL,\n candidate_key VARCHAR(48) NOT NULL,\n hourly_rank SMALLINT NOT NULL CHECK (hourly_rank BETWEEN 1 AND 2),\n thinned_score NUMERIC(12,10) NOT NULL CHECK (thinned_score BETWEEN 0 AND 1),\n route_support BIGINT NOT NULL CHECK (route_support >= 2),\n qualified_support BIGINT NOT NULL CHECK (qualified_support = route_support),\n hour_support BIGINT NOT NULL CHECK (hour_support >= 1),\n corridor_support BIGINT NOT NULL CHECK (corridor_support >= 1),\n qualified_weekday_exposures INTEGER NOT NULL CHECK (qualified_weekday_exposures >= 1),\n scorer_version VARCHAR(96) NOT NULL,\n scorer_fingerprint CHAR(64) NOT NULL CHECK (scorer_fingerprint ~ '^[0-9a-f]{64}$'),\n candidate_generator_version VARCHAR(96) NOT NULL,\n candidate_generator_fingerprint CHAR(64) NOT NULL CHECK (candidate_generator_fingerprint ~ '^[0-9a-f]{64}$'),\n hourly_selector_version VARCHAR(96) NOT NULL,\n hourly_selector_fingerprint CHAR(64) NOT NULL CHECK (hourly_selector_fingerprint ~ '^[0-9a-f]{64}$'),\n feature_source_version VARCHAR(96) NOT NULL,\n feature_source_fingerprint CHAR(64) NOT NULL CHECK (feature_source_fingerprint ~ '^[0-9a-f]{64}$'),\n snapshot_sha256 CHAR(64) NOT NULL UNIQUE CHECK (snapshot_sha256 ~ '^[0-9a-f]{64}$'),\n created_at TIMESTAMPTZ NOT NULL DEFAULT now(),\n UNIQUE(window_id,target_hour,candidate_key),\n UNIQUE(window_id,target_hour,hourly_rank),\n FOREIGN KEY(window_id,target_hour) REFERENCES p862_shadow_invocations(window_id,target_hour),\n CHECK (as_of=target_hour-interval '24 hours'),\n CHECK (route_key=start_h3_r5||'>'||end_h3_r5),\n CHECK (candidate_key=route_key||'>'||corridor_h3_r7)\n);\n\nCREATE TABLE p862_hourly_diagnostics (\n diagnostic_id UUID PRIMARY KEY,\n window_id VARCHAR(96) NOT NULL REFERENCES p862_prospective_windows(window_id),\n target_hour TIMESTAMPTZ NOT NULL,\n as_of TIMESTAMPTZ NOT NULL,\n generated_at TIMESTAMPTZ NOT NULL,\n start_h3_r5 VARCHAR(15) NOT NULL CHECK (start_h3_r5 ~ '^85[0-9a-f]{13}$'),\n end_h3_r5 VARCHAR(15) NOT NULL CHECK (end_h3_r5 ~ '^85[0-9a-f]{13}$'),\n corridor_h3_r7 VARCHAR(15) NOT NULL CHECK (corridor_h3_r7 ~ '^87[0-9a-f]{13}$'),\n route_key VARCHAR(31) NOT NULL,\n candidate_key VARCHAR(48) NOT NULL,\n hourly_rank SMALLINT NOT NULL CHECK (hourly_rank BETWEEN 1 AND 3),\n thinned_score NUMERIC(12,10) NOT NULL CHECK (thinned_score BETWEEN 0 AND 1),\n route_support BIGINT NOT NULL CHECK (route_support>=2),\n qualified_support BIGINT NOT NULL CHECK (qualified_support = route_support),\n hour_support BIGINT NOT NULL CHECK (hour_support>=1),\n corridor_support BIGINT NOT NULL CHECK (corridor_support>=1),\n snapshot_sha256 CHAR(64) NOT NULL UNIQUE CHECK (snapshot_sha256 ~ '^[0-9a-f]{64}$'),\n UNIQUE(window_id,target_hour,hourly_rank),\n UNIQUE(window_id,target_hour,candidate_key),\n FOREIGN KEY(window_id,target_hour) REFERENCES p862_shadow_invocations(window_id,target_hour),\n CHECK(as_of=target_hour-interval '24 hours'),\n CHECK(route_key=start_h3_r5||'>'||end_h3_r5),\n CHECK(candidate_key=route_key||'>'||corridor_h3_r7)\n);\n\nCREATE TABLE p862_shadow_predictions (\n prediction_id UUID PRIMARY KEY,\n candidate_id UUID NOT NULL UNIQUE REFERENCES p862_hourly_candidates(candidate_id),\n window_id VARCHAR(96) NOT NULL REFERENCES p862_prospective_windows(window_id),\n target_hour TIMESTAMPTZ NOT NULL,\n as_of TIMESTAMPTZ NOT NULL,\n generated_at TIMESTAMPTZ NOT NULL,\n daily_rank SMALLINT NOT NULL CHECK (daily_rank BETWEEN 1 AND 5),\n daily_selector_version VARCHAR(96) NOT NULL,\n daily_selector_fingerprint CHAR(64) NOT NULL CHECK (daily_selector_fingerprint ~ '^[0-9a-f]{64}$'),\n experiment_id VARCHAR(96) NOT NULL,\n prediction_sha256 CHAR(64) NOT NULL UNIQUE CHECK (prediction_sha256 ~ '^[0-9a-f]{64}$'),\n created_at TIMESTAMPTZ NOT NULL DEFAULT now(),\n CHECK (as_of=target_hour-interval '24 hours')\n);\n\nCREATE TABLE p862_daily_selections (\n window_id VARCHAR(96) NOT NULL REFERENCES p862_prospective_windows(window_id),\n target_day DATE NOT NULL,\n generated_at TIMESTAMPTZ NOT NULL,\n hourly_candidate_count INTEGER NOT NULL CHECK (hourly_candidate_count BETWEEN 0 AND 48),\n selected_count INTEGER NOT NULL CHECK (selected_count BETWEEN 0 AND 5 AND selected_count<=hourly_candidate_count),\n daily_selector_fingerprint CHAR(64) NOT NULL CHECK (daily_selector_fingerprint ~ '^[0-9a-f]{64}$'),\n PRIMARY KEY(window_id,target_day)\n);\n\nCREATE TABLE p862_shadow_outcomes (\n outcome_id UUID PRIMARY KEY,\n prediction_id UUID NOT NULL UNIQUE REFERENCES p862_shadow_predictions(prediction_id),\n prediction_sha256 CHAR(64) NOT NULL,\n outcome VARCHAR(32) NOT NULL CHECK (outcome IN ('POSITIVE','NEGATIVE','INELIGIBLE_OVERFLOW','UNKNOWN_INCOMPLETE_TRUTH')),\n matched_at TIMESTAMPTZ NOT NULL,\n reason VARCHAR(256) NOT NULL,\n outcome_sha256 CHAR(64) NOT NULL UNIQUE CHECK (outcome_sha256 ~ '^[0-9a-f]{64}$'),\n created_at TIMESTAMPTZ NOT NULL DEFAULT now()\n);\n\nCREATE TABLE p862_hourly_diagnostic_outcomes (\n outcome_id UUID PRIMARY KEY,\n diagnostic_id UUID NOT NULL UNIQUE REFERENCES p862_hourly_diagnostics(diagnostic_id),\n diagnostic_sha256 CHAR(64) NOT NULL,\n outcome VARCHAR(32) NOT NULL CHECK (outcome IN ('POSITIVE','NEGATIVE','INELIGIBLE_OVERFLOW','UNKNOWN_INCOMPLETE_TRUTH')),\n matched_at TIMESTAMPTZ NOT NULL,\n reason VARCHAR(256) NOT NULL,\n outcome_sha256 CHAR(64) NOT NULL UNIQUE CHECK (outcome_sha256 ~ '^[0-9a-f]{64}$')\n);\n\nCREATE INDEX ix_p862_invocation_window_status ON p862_shadow_invocations(window_id,status,target_hour);\nCREATE INDEX ix_p862_candidate_day ON p862_hourly_candidates(window_id,target_hour,route_key);\nCREATE INDEX ix_p862_prediction_day ON p862_shadow_predictions(window_id,target_hour,daily_rank);\nCREATE UNIQUE INDEX uq_p862_prediction_daily_rank ON p862_shadow_predictions\n (window_id,((target_hour AT TIME ZONE 'UTC')::date),daily_rank);\nCREATE INDEX ix_p862_outcome_prediction ON p862_shadow_outcomes(prediction_id,outcome);\nCREATE INDEX ix_p862_hourly_diagnostic_day ON p862_hourly_diagnostics(window_id,target_hour,hourly_rank);\n\nCREATE OR REPLACE FUNCTION p862_immutable_guard() RETURNS trigger LANGUAGE plpgsql AS $$\nBEGIN RAISE EXCEPTION 'P8.6.2 evidence is immutable'; END $$;\nCREATE TRIGGER trg_p862_invocation_immutable BEFORE UPDATE OR DELETE ON p862_shadow_invocations\nFOR EACH ROW EXECUTE FUNCTION p862_immutable_guard();\nCREATE TRIGGER trg_p862_candidate_immutable BEFORE UPDATE OR DELETE ON p862_hourly_candidates\nFOR EACH ROW EXECUTE FUNCTION p862_immutable_guard();\nCREATE TRIGGER trg_p862_hourly_diagnostic_immutable BEFORE UPDATE OR DELETE ON p862_hourly_diagnostics\nFOR EACH ROW EXECUTE FUNCTION p862_immutable_guard();\nCREATE TRIGGER trg_p862_prediction_immutable BEFORE UPDATE OR DELETE ON p862_shadow_predictions\nFOR EACH ROW EXECUTE FUNCTION p862_immutable_guard();\nCREATE TRIGGER trg_p862_daily_selection_immutable BEFORE UPDATE OR DELETE ON p862_daily_selections\nFOR EACH ROW EXECUTE FUNCTION p862_immutable_guard();\nCREATE TRIGGER trg_p862_outcome_immutable BEFORE UPDATE OR DELETE ON p862_shadow_outcomes\nFOR EACH ROW EXECUTE FUNCTION p862_immutable_guard();\nCREATE TRIGGER trg_p862_hourly_diagnostic_outcome_immutable BEFORE UPDATE OR DELETE ON p862_hourly_diagnostic_outcomes\nFOR EACH ROW EXECUTE FUNCTION p862_immutable_guard();\n\nCREATE OR REPLACE FUNCTION p862_window_guard() RETURNS trigger LANGUAGE plpgsql AS $$\nBEGIN\n IF OLD.started_at IS DISTINCT FROM NEW.started_at OR OLD.planned_end_at IS DISTINCT FROM NEW.planned_end_at\n OR OLD.pipeline_fingerprint IS DISTINCT FROM NEW.pipeline_fingerprint\n OR OLD.truth_authority_fingerprint IS DISTINCT FROM NEW.truth_authority_fingerprint\n OR OLD.thinned_fingerprint IS DISTINCT FROM NEW.thinned_fingerprint\n OR OLD.candidate_generator_fingerprint IS DISTINCT FROM NEW.candidate_generator_fingerprint\n OR OLD.hourly_selector_fingerprint IS DISTINCT FROM NEW.hourly_selector_fingerprint\n OR OLD.daily_selector_fingerprint IS DISTINCT FROM NEW.daily_selector_fingerprint\n OR OLD.daily_decision_fingerprint IS DISTINCT FROM NEW.daily_decision_fingerprint\n OR OLD.feature_source_fingerprint IS DISTINCT FROM NEW.feature_source_fingerprint\n OR OLD.outcome_maturity_fingerprint IS DISTINCT FROM NEW.outcome_maturity_fingerprint\n OR OLD.metrics_fingerprint IS DISTINCT FROM NEW.metrics_fingerprint\n OR OLD.selection_contract IS DISTINCT FROM NEW.selection_contract\n OR OLD.metrics_contract IS DISTINCT FROM NEW.metrics_contract THEN\n RAISE EXCEPTION 'P8.6.2 frozen window contract is immutable';\n END IF;\n IF OLD.status IN ('INVALIDATED','COMPLETE') THEN RAISE EXCEPTION 'P8.6.2 terminal window is immutable'; END IF;\n IF NOT ((OLD.status='PLANNED' AND NEW.status IN ('ACTIVE','INVALIDATED'))\n OR (OLD.status='ACTIVE' AND NEW.status IN ('INVALIDATED','COMPLETE'))) THEN\n RAISE EXCEPTION 'Invalid P8.6.2 window transition';\n END IF;\n RETURN NEW;\nEND $$;\nCREATE TRIGGER trg_p862_window_guard BEFORE UPDATE ON p862_prospective_windows\nFOR EACH ROW EXECUTE FUNCTION p862_window_guard();\n\nCREATE OR REPLACE FUNCTION p862_target_window_guard() RETURNS trigger LANGUAGE plpgsql AS $$\nDECLARE window_start TIMESTAMPTZ; window_end TIMESTAMPTZ; target TIMESTAMPTZ;\nBEGIN\n SELECT started_at,planned_end_at INTO window_start,window_end\n FROM p862_prospective_windows WHERE window_id=NEW.window_id;\n IF window_start IS NULL THEN RAISE EXCEPTION 'P8.6.2 target has no window'; END IF;\n IF TG_TABLE_NAME='p862_daily_selections' THEN\n target := NEW.target_day::timestamp AT TIME ZONE 'UTC';\n ELSE\n target := NEW.target_hour;\n END IF;\n IF target < window_start OR target >= window_end THEN\n RAISE EXCEPTION 'P8.6.2 target is outside frozen window';\n END IF;\n RETURN NEW;\nEND $$;\nCREATE TRIGGER trg_p862_invocation_window BEFORE INSERT ON p862_shadow_invocations\nFOR EACH ROW EXECUTE FUNCTION p862_target_window_guard();\nCREATE TRIGGER trg_p862_daily_selection_window BEFORE INSERT ON p862_daily_selections\nFOR EACH ROW EXECUTE FUNCTION p862_target_window_guard();\n\nCREATE OR REPLACE FUNCTION p862_prediction_guard() RETURNS trigger LANGUAGE plpgsql AS $$\nDECLARE c p862_hourly_candidates%ROWTYPE;\nBEGIN\n SELECT * INTO c FROM p862_hourly_candidates WHERE candidate_id=NEW.candidate_id;\n IF c.candidate_id IS NULL OR c.window_id<>NEW.window_id OR c.target_hour<>NEW.target_hour OR c.as_of<>NEW.as_of\n OR NEW.generated_at < c.generated_at THEN RAISE EXCEPTION 'P8.6.2 prediction/candidate provenance mismatch'; END IF;\n RETURN NEW;\nEND $$;\nCREATE TRIGGER trg_p862_prediction_link BEFORE INSERT ON p862_shadow_predictions\nFOR EACH ROW EXECUTE FUNCTION p862_prediction_guard();\n\nCOMMENT ON TABLE p862_hourly_candidates IS 'Bounded at 48 rows/window UTC day; retain through evidence review then archive/delete only in a future reviewed migration.';\nCOMMENT ON TABLE p862_shadow_predictions IS 'At most 140 immutable selected predictions per 28-day window.';\n" }, { "path": "backend/truth-model/src/main/resources/db/migration/V78__forecast_product_continuous_materialization.sql", "encoding": "UTF-8", "sha256": "dd8c6355edeefa997172686d7dd7ef6ffa88291ed453c4bafe95428d7707a1c7", "content": "-- P10 product forecast persistence is deliberately separate from P862 prospective evidence.\n-- It may be populated continuously only from exact T-24 frozen pipeline output and never mutates evidence rows.\n\ncreate table forecast_product_invocations (\n target_hour timestamptz primary key,\n as_of timestamptz not null,\n generated_at timestamptz not null,\n status varchar(32) not null,\n candidate_count integer not null default 0,\n pipeline_fingerprint char(64) not null,\n truth_authority_fingerprint char(64) not null,\n failure_code varchar(160),\n created_at timestamptz not null default now(),\n constraint ck_forecast_product_invocation_hour check (target_hour = date_trunc('hour', target_hour)),\n constraint ck_forecast_product_invocation_asof check (as_of = target_hour - interval '24 hours'),\n constraint ck_forecast_product_invocation_status check (\n status in ('FORECASTED','MISSED_CAUSAL_INVOCATION','CANDIDATE_GENERATION_FAILURE')),\n constraint ck_forecast_product_invocation_count check (candidate_count between 0 and 2),\n constraint ck_forecast_product_pipeline_fingerprint check (pipeline_fingerprint ~ '^[0-9a-f]{64}$'),\n constraint ck_forecast_product_truth_fingerprint check (truth_authority_fingerprint ~ '^[0-9a-f]{64}$')\n);\n\ncreate table forecast_product_hourly_candidates (\n product_candidate_id uuid primary key,\n target_hour timestamptz not null references forecast_product_invocations(target_hour) on delete cascade,\n as_of timestamptz not null,\n generated_at timestamptz not null,\n hourly_rank smallint not null,\n route_key varchar(160) not null,\n corridor_h3_r7 varchar(32) not null,\n candidate_key varchar(256) not null,\n thinned_score numeric(24,12) not null,\n route_support bigint not null,\n qualified_support bigint not null,\n hour_support bigint not null,\n corridor_support bigint not null,\n qualified_weekday_exposures integer not null,\n pipeline_fingerprint char(64) not null,\n truth_authority_fingerprint char(64) not null,\n snapshot_sha256 char(64) not null,\n created_at timestamptz not null default now(),\n constraint uq_forecast_product_target_rank unique(target_hour, hourly_rank),\n constraint ck_forecast_product_hourly_rank check (hourly_rank between 1 and 2),\n constraint ck_forecast_product_candidate_asof check (as_of = target_hour - interval '24 hours'),\n constraint ck_forecast_product_support_nonnegative check (\n route_support >= 0 and qualified_support >= 0 and hour_support >= 0 and corridor_support >= 0\n and qualified_weekday_exposures >= 0),\n constraint ck_forecast_product_candidate_pipeline_fingerprint check (pipeline_fingerprint ~ '^[0-9a-f]{64}$'),\n constraint ck_forecast_product_candidate_truth_fingerprint check (truth_authority_fingerprint ~ '^[0-9a-f]{64}$'),\n constraint ck_forecast_product_candidate_sha check (snapshot_sha256 ~ '^[0-9a-f]{64}$')\n);\n\ncreate index idx_forecast_product_candidates_target\n on forecast_product_hourly_candidates(target_hour, hourly_rank);\ncreate index idx_forecast_product_invocations_created\n on forecast_product_invocations(created_at);\n" }, { "path": "backend/truth-model/src/main/resources/db/migration/V82__forecast_product_materialization_immutability.sql", "encoding": "UTF-8", "sha256": "8160e7910096408690dcfbfc97db551beddad0387088bd6863a40dc6e3fccf77", "content": "-- T-24 product materialization is a causal immutable copy of frozen P862 output.\n-- Rows may age out through the bounded parent cleanup path, but neither invocation\n-- nor candidate facts may be rewritten after insertion.\n\ncreate function forecast_product_reject_materialization_update()\nreturns trigger language plpgsql as $$\nbegin\n raise exception 'Forecast product materialization rows are immutable';\nend\n$$;\n\ncreate trigger trg_forecast_product_invocation_immutable\nbefore update on forecast_product_invocations\nfor each row execute function forecast_product_reject_materialization_update();\n\ncreate trigger trg_forecast_product_candidate_immutable\nbefore update on forecast_product_hourly_candidates\nfor each row execute function forecast_product_reject_materialization_update();\n" }, { "path": "backend/truth-model/src/main/resources/db/migration/V83__regional_historical_truth_authority.sql", "encoding": "UTF-8", "sha256": "c711a282038e30b779e810093c239eeefbc4c16f4b7c9c918c58b5f7e0c3aad4", "content": "-- Region-aware historical truth authority. This is the only supported P862 truth generation.\n\nALTER TABLE p862_prospective_windows\n ADD COLUMN authority_generation VARCHAR(64) NOT NULL DEFAULT 'REGIONAL_HISTORICAL_TRUTH_AUTHORITY_V1';\n\nALTER TABLE p862_prospective_windows\n DROP CONSTRAINT IF EXISTS p862_prospective_windows_experiment_id_check;\n\nALTER TABLE p862_prospective_windows\n ADD CONSTRAINT ck_p862_experiment_generation CHECK (\n authority_generation='REGIONAL_HISTORICAL_TRUTH_AUTHORITY_V1'\n AND experiment_id='p862-regional-historical-prospective-v1'\n );\n\nALTER TABLE p862_prospective_windows\n ALTER COLUMN authority_generation DROP DEFAULT;\n\nCREATE TABLE regional_historical_truth_authorities (\n authority_fingerprint CHAR(64) PRIMARY KEY CHECK (authority_fingerprint ~ '^[0-9a-f]{64}$'),\n authority_generation VARCHAR(64) NOT NULL CHECK (\n authority_generation='REGIONAL_HISTORICAL_TRUTH_AUTHORITY_V1'),\n region_id VARCHAR(64) NOT NULL CHECK (region_id ~ '^[A-Z][A-Z0-9_]{1,63}$'),\n region_canonical_sha256 CHAR(64) NOT NULL CHECK (region_canonical_sha256 ~ '^[0-9a-f]{64}$'),\n coverage_version VARCHAR(120) NOT NULL,\n evidence_coverage_hash CHAR(64) NOT NULL CHECK (evidence_coverage_hash ~ '^[0-9a-f]{64}$'),\n provider VARCHAR(32) NOT NULL,\n source_repository_policy VARCHAR(160) NOT NULL,\n license_identifier VARCHAR(32) NOT NULL,\n source_manifest_policy_version VARCHAR(96) NOT NULL,\n importer_version VARCHAR(80) NOT NULL,\n parser_version VARCHAR(80) NOT NULL,\n archive_safety_profile VARCHAR(512) NOT NULL,\n eligibility_version VARCHAR(80) NOT NULL,\n segmentation_version VARCHAR(80) NOT NULL,\n derivation_version VARCHAR(80) NOT NULL,\n completeness_semantics_version VARCHAR(96) NOT NULL,\n maturity_seconds BIGINT NOT NULL CHECK (maturity_seconds BETWEEN 0 AND 1209600),\n seal_semantics_version VARCHAR(96) NOT NULL,\n created_at TIMESTAMPTZ NOT NULL DEFAULT now()\n);\n\nCREATE TABLE regional_historical_truth_day_seals (\n authority_fingerprint CHAR(64) NOT NULL\n REFERENCES regional_historical_truth_authorities(authority_fingerprint),\n target_day DATE NOT NULL,\n dataset_id UUID REFERENCES historical_bootstrap_datasets(dataset_id),\n status VARCHAR(32) NOT NULL CHECK (status IN (\n 'AUTHORITATIVE_COMPLETE','UNKNOWN_GAP','UNKNOWN_DEGRADED')),\n authoritative_ready_at TIMESTAMPTZ NOT NULL,\n source_manifest_sha256 CHAR(64) NOT NULL CHECK (source_manifest_sha256 ~ '^[0-9a-f]{64}$'),\n day_manifest_sha256 CHAR(64) NOT NULL CHECK (day_manifest_sha256 ~ '^[0-9a-f]{64}$'),\n day_delta_checksum CHAR(64) CHECK (day_delta_checksum ~ '^[0-9a-f]{64}$'),\n dataset_semantic_fingerprint CHAR(64) CHECK (dataset_semantic_fingerprint ~ '^[0-9a-f]{64}$'),\n terminal_quality VARCHAR(16) NOT NULL CHECK (terminal_quality IN ('QUALIFIED','DEGRADED','GAP')),\n reason_code VARCHAR(64),\n seal_sha256 CHAR(64) NOT NULL UNIQUE CHECK (seal_sha256 ~ '^[0-9a-f]{64}$'),\n created_at TIMESTAMPTZ NOT NULL DEFAULT now(),\n PRIMARY KEY(authority_fingerprint,target_day),\n UNIQUE(authority_fingerprint,dataset_id,target_day),\n CHECK (\n (status='AUTHORITATIVE_COMPLETE' AND terminal_quality='QUALIFIED'\n AND dataset_id IS NOT NULL AND day_delta_checksum IS NOT NULL\n AND dataset_semantic_fingerprint IS NOT NULL AND reason_code IS NULL)\n OR\n (status='UNKNOWN_GAP' AND terminal_quality='GAP'\n AND dataset_id IS NULL AND day_delta_checksum IS NULL\n AND dataset_semantic_fingerprint IS NULL AND reason_code IS NOT NULL)\n OR\n (status='UNKNOWN_DEGRADED' AND terminal_quality='DEGRADED'\n AND dataset_id IS NULL AND day_delta_checksum IS NULL\n AND dataset_semantic_fingerprint IS NULL AND reason_code IS NOT NULL)\n )\n);\n\nCREATE INDEX ix_regional_historical_truth_day_status\n ON regional_historical_truth_day_seals(authority_fingerprint,status,target_day);\n\nCREATE OR REPLACE FUNCTION regional_historical_truth_day_guard()\nRETURNS trigger LANGUAGE plpgsql AS $$\nDECLARE authority regional_historical_truth_authorities%ROWTYPE;\n dataset historical_bootstrap_datasets%ROWTYPE;\n source_day historical_bootstrap_days%ROWTYPE;\n minimum_ready_at TIMESTAMPTZ;\nBEGIN\n SELECT * INTO authority\n FROM regional_historical_truth_authorities\n WHERE authority_fingerprint=NEW.authority_fingerprint;\n IF authority.authority_fingerprint IS NULL THEN\n RAISE EXCEPTION 'regional truth day requires registered authority';\n END IF;\n\n minimum_ready_at := ((NEW.target_day + 1)::timestamp AT TIME ZONE 'UTC')\n + make_interval(secs => authority.maturity_seconds);\n IF NEW.authoritative_ready_at < minimum_ready_at THEN\n RAISE EXCEPTION 'regional truth day is not mature under authority policy';\n END IF;\n\n IF NEW.status='AUTHORITATIVE_COMPLETE' THEN\n SELECT * INTO dataset FROM historical_bootstrap_datasets WHERE dataset_id=NEW.dataset_id;\n SELECT * INTO source_day\n FROM historical_bootstrap_days\n WHERE dataset_id=NEW.dataset_id AND source_date=NEW.target_day;\n\n IF dataset.dataset_id IS NULL OR source_day.dataset_id IS NULL\n OR dataset.state<>'READY'\n OR dataset.dataset_purpose<>'BOOTSTRAP_SCORING'\n OR dataset.from_date<>NEW.target_day OR dataset.to_date<>NEW.target_day\n OR source_day.experiment_role<>'SCORING'\n OR source_day.terminal_quality<>'QUALIFIED' THEN\n RAISE EXCEPTION 'regional COMPLETE requires one-day READY QUALIFIED SCORING evidence';\n END IF;\n\n IF dataset.provider<>authority.provider\n OR NOT (\n authority.source_repository_policy='adsblol/globe_history_{yyyy}'\n AND dataset.source_repository ~ '^adsblol/globe_history_[0-9]{4}$'\n )\n OR dataset.license_identifier<>authority.license_identifier\n OR dataset.coverage_version<>authority.coverage_version\n OR dataset.coverage_hash<>authority.evidence_coverage_hash\n OR dataset.importer_version<>authority.importer_version\n OR dataset.parser_version<>authority.parser_version\n OR dataset.archive_safety_profile<>authority.archive_safety_profile\n OR dataset.eligibility_version<>authority.eligibility_version\n OR dataset.segmentation_version<>authority.segmentation_version\n OR dataset.derivation_version<>authority.derivation_version\n OR dataset.manifest_sha256<>NEW.source_manifest_sha256\n OR dataset.semantic_fingerprint<>NEW.dataset_semantic_fingerprint\n OR source_day.day_manifest_sha256<>NEW.day_manifest_sha256\n OR source_day.day_delta_checksum<>NEW.day_delta_checksum THEN\n RAISE EXCEPTION 'regional COMPLETE authority/dataset provenance mismatch';\n END IF;\n\n IF NEW.authoritative_ready_at < dataset.ready_at THEN\n RAISE EXCEPTION 'regional COMPLETE seal predates READY dataset';\n END IF;\n END IF;\n RETURN NEW;\nEND $$;\n\nCREATE TRIGGER trg_regional_historical_truth_day_guard\nBEFORE INSERT ON regional_historical_truth_day_seals\nFOR EACH ROW EXECUTE FUNCTION regional_historical_truth_day_guard();\n\nCREATE OR REPLACE FUNCTION regional_historical_truth_immutable_guard()\nRETURNS trigger LANGUAGE plpgsql AS $$\nBEGIN\n RAISE EXCEPTION 'regional historical truth authority evidence is immutable';\nEND $$;\n\nCREATE TRIGGER trg_regional_historical_authority_immutable\nBEFORE UPDATE OR DELETE ON regional_historical_truth_authorities\nFOR EACH ROW EXECUTE FUNCTION regional_historical_truth_immutable_guard();\n\nCREATE TRIGGER trg_regional_historical_day_seal_immutable\nBEFORE UPDATE OR DELETE ON regional_historical_truth_day_seals\nFOR EACH ROW EXECUTE FUNCTION regional_historical_truth_immutable_guard();\n\nCREATE VIEW p862_effective_truth_day_seals AS\nSELECT target_day,\n CASE status\n WHEN 'AUTHORITATIVE_COMPLETE' THEN 'AUTHORITATIVE_COMPLETE'\n ELSE 'UNKNOWN_INCOMPLETE_TRUTH'\n END AS status,\n authoritative_ready_at,\n authority_fingerprint AS source_fingerprint,\n 'REGIONAL_HISTORICAL_TRUTH_AUTHORITY_V1'::varchar(64) AS authority_generation,\n dataset_id AS historical_dataset_id,\n created_at\n FROM regional_historical_truth_day_seals;\n\nCOMMENT ON VIEW p862_effective_truth_day_seals IS\n 'Single regional P862 truth seam. GAP/DEGRADED remain UNKNOWN and can never create a negative label.';\n\nCREATE OR REPLACE FUNCTION p862_prediction_guard() RETURNS trigger LANGUAGE plpgsql AS $$\nDECLARE c p862_hourly_candidates%ROWTYPE;\n expected_experiment VARCHAR(96);\nBEGIN\n SELECT * INTO c FROM p862_hourly_candidates WHERE candidate_id=NEW.candidate_id;\n SELECT experiment_id INTO expected_experiment\n FROM p862_prospective_windows WHERE window_id=NEW.window_id;\n IF c.candidate_id IS NULL\n OR c.window_id<>NEW.window_id\n OR c.target_hour<>NEW.target_hour\n OR c.as_of<>NEW.as_of\n OR NEW.generated_atexpected_experiment THEN\n RAISE EXCEPTION 'P8.6.2 prediction/candidate/window provenance mismatch';\n END IF;\n RETURN NEW;\nEND $$;\n\nCREATE OR REPLACE FUNCTION p862_outcome_guard() RETURNS trigger LANGUAGE plpgsql AS $$\nDECLARE expected CHAR(64); target TIMESTAMPTZ; window_key VARCHAR(96);\n expected_authority CHAR(64); expected_generation VARCHAR(64);\n seal_authority CHAR(64); seal_status VARCHAR(32); ready TIMESTAMPTZ;\nBEGIN\n SELECT prediction_sha256,target_hour,window_id INTO expected,target,window_key\n FROM p862_shadow_predictions WHERE prediction_id=NEW.prediction_id;\n SELECT truth_authority_fingerprint,authority_generation INTO expected_authority,expected_generation\n FROM p862_prospective_windows WHERE window_id=window_key;\n SELECT status,authoritative_ready_at,source_fingerprint INTO seal_status,ready,seal_authority\n FROM p862_effective_truth_day_seals\n WHERE target_day=(target AT TIME ZONE 'UTC')::date\n AND source_fingerprint=expected_authority\n AND authority_generation=expected_generation;\n IF expected IS NULL OR expected IS DISTINCT FROM NEW.prediction_sha256 OR ready IS NULL OR NEW.matched_at'UNKNOWN_INCOMPLETE_TRUTH' THEN\n RAISE EXCEPTION 'P8.6.2 incomplete truth cannot create a label'; END IF;\n IF seal_status='AUTHORITATIVE_COMPLETE' AND NEW.outcome='UNKNOWN_INCOMPLETE_TRUTH' THEN\n RAISE EXCEPTION 'P8.6.2 complete truth cannot create an incomplete label'; END IF;\n RETURN NEW;\nEND $$;\n\nCREATE OR REPLACE FUNCTION p862_diagnostic_outcome_guard() RETURNS trigger LANGUAGE plpgsql AS $$\nDECLARE expected CHAR(64); target TIMESTAMPTZ; window_key VARCHAR(96);\n expected_authority CHAR(64); expected_generation VARCHAR(64);\n seal_authority CHAR(64); seal_status VARCHAR(32); ready TIMESTAMPTZ;\nBEGIN\n SELECT snapshot_sha256,target_hour,window_id INTO expected,target,window_key\n FROM p862_hourly_diagnostics WHERE diagnostic_id=NEW.diagnostic_id;\n SELECT truth_authority_fingerprint,authority_generation INTO expected_authority,expected_generation\n FROM p862_prospective_windows WHERE window_id=window_key;\n SELECT status,authoritative_ready_at,source_fingerprint INTO seal_status,ready,seal_authority\n FROM p862_effective_truth_day_seals\n WHERE target_day=(target AT TIME ZONE 'UTC')::date\n AND source_fingerprint=expected_authority\n AND authority_generation=expected_generation;\n IF expected IS NULL OR expected IS DISTINCT FROM NEW.diagnostic_sha256 OR ready IS NULL OR NEW.matched_at'UNKNOWN_INCOMPLETE_TRUTH' THEN\n RAISE EXCEPTION 'P8.6.2 incomplete truth cannot create a diagnostic label'; END IF;\n IF seal_status='AUTHORITATIVE_COMPLETE' AND NEW.outcome='UNKNOWN_INCOMPLETE_TRUTH' THEN\n RAISE EXCEPTION 'P8.6.2 complete truth cannot create an incomplete diagnostic label'; END IF;\n RETURN NEW;\nEND $$;\n\nDROP TRIGGER IF EXISTS trg_p862_outcome_link ON p862_shadow_outcomes;\nCREATE TRIGGER trg_p862_outcome_link\nBEFORE INSERT ON p862_shadow_outcomes\nFOR EACH ROW EXECUTE FUNCTION p862_outcome_guard();\n\nDROP TRIGGER IF EXISTS trg_p862_diagnostic_outcome_link ON p862_hourly_diagnostic_outcomes;\nCREATE TRIGGER trg_p862_diagnostic_outcome_link\nBEFORE INSERT ON p862_hourly_diagnostic_outcomes\nFOR EACH ROW EXECUTE FUNCTION p862_diagnostic_outcome_guard();\n\nCREATE OR REPLACE FUNCTION p862_window_guard() RETURNS trigger LANGUAGE plpgsql AS $$\nBEGIN\n IF OLD.started_at IS DISTINCT FROM NEW.started_at OR OLD.planned_end_at IS DISTINCT FROM NEW.planned_end_at\n OR OLD.experiment_id IS DISTINCT FROM NEW.experiment_id\n OR OLD.authority_generation IS DISTINCT FROM NEW.authority_generation\n OR OLD.pipeline_fingerprint IS DISTINCT FROM NEW.pipeline_fingerprint\n OR OLD.truth_authority_fingerprint IS DISTINCT FROM NEW.truth_authority_fingerprint\n OR OLD.thinned_fingerprint IS DISTINCT FROM NEW.thinned_fingerprint\n OR OLD.candidate_generator_fingerprint IS DISTINCT FROM NEW.candidate_generator_fingerprint\n OR OLD.hourly_selector_fingerprint IS DISTINCT FROM NEW.hourly_selector_fingerprint\n OR OLD.daily_selector_fingerprint IS DISTINCT FROM NEW.daily_selector_fingerprint\n OR OLD.daily_decision_fingerprint IS DISTINCT FROM NEW.daily_decision_fingerprint\n OR OLD.feature_source_fingerprint IS DISTINCT FROM NEW.feature_source_fingerprint\n OR OLD.outcome_maturity_fingerprint IS DISTINCT FROM NEW.outcome_maturity_fingerprint\n OR OLD.metrics_fingerprint IS DISTINCT FROM NEW.metrics_fingerprint\n OR OLD.selection_contract IS DISTINCT FROM NEW.selection_contract\n OR OLD.metrics_contract IS DISTINCT FROM NEW.metrics_contract THEN\n RAISE EXCEPTION 'P8.6.2 frozen window contract is immutable';\n END IF;\n IF OLD.status IN ('INVALIDATED','COMPLETE') THEN\n RAISE EXCEPTION 'P8.6.2 terminal window is immutable';\n END IF;\n IF NOT ((OLD.status='PLANNED' AND NEW.status IN ('ACTIVE','INVALIDATED'))\n OR (OLD.status='ACTIVE' AND NEW.status IN ('INVALIDATED','COMPLETE'))) THEN\n RAISE EXCEPTION 'Invalid P8.6.2 window transition';\n END IF;\n RETURN NEW;\nEND $$;\n" }, { "path": "backend/truth-model/src/main/resources/db/migration/V84__forecast_product_zone_relevance.sql", "encoding": "UTF-8", "sha256": "080ec50abcf5bb2b15698b61a58e9f7733f2db17f1afd9134527fcfdbeaacf40", "content": "-- Product T-24 materialization is zone-demand relevant, while the frozen P862 scientific\n-- selector remains unchanged. Persist the product selection contract separately from the\n-- scientific pipeline fingerprint so rows from different relevance semantics can never mix.\n--\n-- FlightTrove has not shipped this schema to production yet, so the new product rows use\n-- strict NOT NULL authority columns instead of carrying a compatibility path.\n\nalter table forecast_product_invocations\n drop constraint ck_forecast_product_invocation_count;\n\nalter table forecast_product_invocations\n add constraint ck_forecast_product_invocation_count\n check (candidate_count between 0 and 4096);\n\nalter table forecast_product_hourly_candidates\n drop constraint ck_forecast_product_hourly_rank;\n\nalter table forecast_product_hourly_candidates\n add constraint ck_forecast_product_hourly_rank\n check (hourly_rank between 1 and 4096);\n\nalter table forecast_product_invocations\n add column selection_fingerprint char(64) not null;\n\nalter table forecast_product_invocations\n add constraint ck_forecast_product_selection_fingerprint\n check (selection_fingerprint ~ '^[0-9a-f]{64}$');\n\nalter table forecast_product_hourly_candidates\n add column demand_h3_r5 varchar(15) not null;\n\nalter table forecast_product_hourly_candidates\n add constraint ck_forecast_product_candidate_demand_h3_r5\n check (demand_h3_r5 ~ '^[0-9a-f]{15}$');\n\nalter table forecast_product_hourly_candidates\n add column selection_fingerprint char(64) not null;\n\nalter table forecast_product_hourly_candidates\n add constraint ck_forecast_product_candidate_selection_fingerprint\n check (selection_fingerprint ~ '^[0-9a-f]{64}$');\n\ncreate index idx_forecast_product_candidates_demand_target\n on forecast_product_hourly_candidates(demand_h3_r5, target_hour, hourly_rank);\n" }, { "path": "backend/truth-model/src/main/resources/db/migration/V85__forecast_product_active_zone_demand_index.sql", "encoding": "UTF-8", "sha256": "e855f1e5e4a12befc3982d4d79e6a0dae2268e1b406e033ca4b5f815dd12dcf6", "content": "-- Materialize the coarse R5 parent once at historical import time so exact T-24\n-- product generation can prefilter regional corridor evidence in PostgreSQL before\n-- transferring/scoring rows in Java. Both historical import seams persist this value.\nalter table historical_bootstrap_route_corridor_aggregates\n add column corridor_h3_r5 varchar(15) not null;\n\nalter table historical_bootstrap_route_corridor_aggregates\n add constraint ck_hb_corridor_parent_r5\n check (corridor_h3_r5 ~ '^85[0-9a-f]{13}$');\n\ncreate index ix_hb_corridor_parent_lookup\n on historical_bootstrap_route_corridor_aggregates(\n dataset_id, corridor_h3_r5, start_h3_r5, end_h3_r5, derivation_version);\n\n-- PostgreSQL expands SELECT * when a view is created. V71 therefore cannot expose a column\n-- added later by ALTER TABLE unless the READY seam is explicitly recreated. Keep all existing\n-- columns in their original order and append the new parent identity at the end.\ncreate or replace view historical_bootstrap_ready_route_corridor_aggregates as\nselect c.dataset_id,\n c.derivation_version,\n c.start_h3_r5,\n c.end_h3_r5,\n c.corridor_h3_r7,\n c.traversal_count,\n c.progress_permille_sum,\n c.minimum_altitude_ft,\n c.maximum_altitude_ft,\n c.altitude_sum_ft,\n c.altitude_evidence_count,\n c.created_at,\n c.corridor_h3_r5\n from historical_bootstrap_route_corridor_aggregates c\n join historical_bootstrap_datasets d on d.dataset_id=c.dataset_id\n where d.state='READY';\n\ncomment on view historical_bootstrap_ready_route_corridor_aggregates is\n 'Learning/scoring R7 corridor aggregates with materialized R5 parent; HOLDOUT corridor truth is excluded';\n\n-- T-24 product demand scans only entitlement-active spotting zones. Keep that bounded\n-- global scan index-backed and intentionally independent of notification opt-in.\ncreate index ix_saved_alert_locations_t24_active\n on saved_alert_locations(id)\n where entitlement_active = true;\n" }, { "path": "backend/truth-model/src/main/resources/db/migration/V86__forecast_product_shared_anchor_snapshots.sql", "encoding": "UTF-8", "sha256": "eed1ee7f0c925049a5f116bbc7bd90c18b755836c1e28dfa0a387df6e17f5ae6", "content": "-- Product T+24 v3: shared, user-independent anchor-cell snapshots.\n--\n-- The scientific P862 evidence tables remain immutable and unchanged. This migration replaces only the\n-- product projection introduced by V78/V84. FlightTrove has not shipped the old projection to production;\n-- rebuilding it here removes user-union selection semantics instead of carrying a compatibility path.\n--\n-- A snapshot is authoritative for one target hour + one R5 anchor + one selection fingerprint. The anchor\n-- is the R5 cell containing a saved spotting zone. Candidate discovery covers a conservative ring around that\n-- anchor, while exact user/tier distance is always revalidated by TomorrowForecastReadService.\n\ndrop table if exists forecast_product_hourly_candidates cascade;\ndrop table if exists forecast_product_invocations cascade;\n\ncreate table forecast_product_invocations (\n target_hour timestamptz not null,\n demand_anchor_h3_r5 varchar(15) not null,\n selection_fingerprint char(64) not null,\n as_of timestamptz not null,\n generated_at timestamptz not null,\n generation_mode varchar(32) not null,\n status varchar(32) not null,\n candidate_count integer not null default 0,\n pipeline_fingerprint char(64) not null,\n truth_authority_fingerprint char(64) not null,\n failure_code varchar(160),\n created_at timestamptz not null default now(),\n primary key (target_hour, demand_anchor_h3_r5, selection_fingerprint),\n constraint ck_forecast_product_anchor_hour check (target_hour = date_trunc('hour', target_hour)),\n constraint ck_forecast_product_anchor_h3 check (demand_anchor_h3_r5 ~ '^85[0-9a-f]{13}$'),\n constraint ck_forecast_product_anchor_asof check (as_of = target_hour - interval '24 hours'),\n constraint ck_forecast_product_anchor_generation_mode check (\n generation_mode in ('ON_TIME','CAUSAL_BACKFILL')),\n constraint ck_forecast_product_anchor_status check (\n status in ('FORECASTED','CAPACITY_EXCEEDED')),\n constraint ck_forecast_product_anchor_count check (candidate_count between 0 and 4096),\n constraint ck_forecast_product_anchor_pipeline check (pipeline_fingerprint ~ '^[0-9a-f]{64}$'),\n constraint ck_forecast_product_anchor_truth check (truth_authority_fingerprint ~ '^[0-9a-f]{64}$'),\n constraint ck_forecast_product_anchor_selection check (selection_fingerprint ~ '^[0-9a-f]{64}$')\n);\n\ncreate table forecast_product_hourly_candidates (\n product_candidate_id uuid primary key,\n target_hour timestamptz not null,\n demand_anchor_h3_r5 varchar(15) not null,\n selection_fingerprint char(64) not null,\n as_of timestamptz not null,\n generated_at timestamptz not null,\n hourly_rank integer not null,\n route_key varchar(160) not null,\n corridor_h3_r7 varchar(32) not null,\n candidate_key varchar(256) not null,\n thinned_score numeric(24,12) not null,\n route_support bigint not null,\n qualified_support bigint not null,\n hour_support bigint not null,\n corridor_support bigint not null,\n qualified_weekday_exposures integer not null,\n pipeline_fingerprint char(64) not null,\n truth_authority_fingerprint char(64) not null,\n snapshot_sha256 char(64) not null,\n created_at timestamptz not null default now(),\n constraint fk_forecast_product_anchor_invocation\n foreign key (target_hour, demand_anchor_h3_r5, selection_fingerprint)\n references forecast_product_invocations(target_hour, demand_anchor_h3_r5, selection_fingerprint)\n on delete cascade,\n constraint uq_forecast_product_anchor_rank\n unique(target_hour, demand_anchor_h3_r5, selection_fingerprint, hourly_rank),\n constraint ck_forecast_product_anchor_candidate_h3\n check (demand_anchor_h3_r5 ~ '^85[0-9a-f]{13}$'),\n constraint ck_forecast_product_anchor_candidate_rank\n check (hourly_rank between 1 and 4096),\n constraint ck_forecast_product_anchor_candidate_asof\n check (as_of = target_hour - interval '24 hours'),\n constraint ck_forecast_product_anchor_candidate_support check (\n route_support >= 0 and qualified_support >= 0 and hour_support >= 0 and corridor_support >= 0\n and qualified_weekday_exposures >= 0),\n constraint ck_forecast_product_anchor_candidate_pipeline\n check (pipeline_fingerprint ~ '^[0-9a-f]{64}$'),\n constraint ck_forecast_product_anchor_candidate_truth\n check (truth_authority_fingerprint ~ '^[0-9a-f]{64}$'),\n constraint ck_forecast_product_anchor_candidate_selection\n check (selection_fingerprint ~ '^[0-9a-f]{64}$'),\n constraint ck_forecast_product_anchor_candidate_sha\n check (snapshot_sha256 ~ '^[0-9a-f]{64}$')\n);\n\ncreate index idx_forecast_product_anchor_candidates_read\n on forecast_product_hourly_candidates(\n demand_anchor_h3_r5, target_hour, selection_fingerprint, hourly_rank);\n\ncreate index idx_forecast_product_anchor_invocations_created\n on forecast_product_invocations(created_at);\n\n-- V82 created the immutable trigger function. Rebind it to the replacement product tables.\ncreate trigger trg_forecast_product_invocation_immutable\nbefore update on forecast_product_invocations\nfor each row execute function forecast_product_reject_materialization_update();\n\ncreate trigger trg_forecast_product_candidate_immutable\nbefore update on forecast_product_hourly_candidates\nfor each row execute function forecast_product_reject_materialization_update();\n" } ] }