feat: add create mapsheet coverage command TDE-1130 (#1048)

#### Motivation It it useful to know what areas of the basemaps config overlaps #### Modification adds create mapsheet coverage command, it takes a basemaps configuration tileset and uses the capturea-area.geojson to determine the coverage area for the configuration it outputs: - `layers-source.geojson` FeatureCollection of all the source layers - `layers-combined.geojson` Feature of all the layers combined together (full coverage) - `layers-required.geojson` Arease of each layer that are required for full coverage - `remove-${layerName}.geojson` any layers that are fully covered and should be removed. #### Checklist _If not applicable, provide explanation of why._ - [ ] Tests updated - [ ] Docs updated - [ ] Issue linked in Title --------- Co-authored-by: paulfouquet <[email protected]> Co-authored-by: Victor Engmark <[email protected]> Co-authored-by: Paul Fouquet <[email protected]>
linz · Sep 25, 2024 · 1447e85 · 1447e85
1 parent fe0aa11
commit 1447e85
Show file tree

Hide file tree

Showing 13 changed files with 788 additions and 180 deletions.
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -37,12 +37,12 @@
     "@aws-sdk/client-s3": "^3.440.0",
     "@aws-sdk/credential-providers": "^3.438.0",
     "@aws-sdk/lib-storage": "^3.440.0",
-    "@basemaps/config": "^7.1.0",
-    "@basemaps/geo": "^7.1.0",
+    "@basemaps/config": "^7.7.0",
+    "@basemaps/geo": "^7.5.0",
     "@chunkd/fs": "^10.0.9",
     "@chunkd/source-aws-v3": "^10.1.3",
     "@cogeotiff/core": "^9.0.3",
-    "@linzjs/geojson": "^6.43.0",
+    "@linzjs/geojson": "^7.10.0",
     "@octokit/core": "^5.0.0",
     "@octokit/plugin-rest-endpoint-methods": "^10.1.1",
     "@wtrpc/core": "^1.0.2",

diff --git a/src/commands/common.ts b/src/commands/common.ts
@@ -1,6 +1,6 @@
 import { fsa } from '@chunkd/fs';
 import { Tiff } from '@cogeotiff/core';
-import { boolean, flag, option, optional, string } from 'cmd-ts';
+import { boolean, flag, option, optional, string, Type } from 'cmd-ts';
 import pLimit from 'p-limit';
 import { fileURLToPath, pathToFileURL } from 'url';
 
@@ -109,7 +109,7 @@ export function createTiff(loc: string): Promise<Tiff> {
  *
  * Relative paths will be converted into file urls.
  */
-function tryParseUrl(loc: string): URL {
+export function tryParseUrl(loc: string): URL {
   try {
     return new URL(loc);
   } catch (e) {
@@ -127,3 +127,35 @@ export function urlToString(u: URL): string {
   if (u.protocol === 'file:') return fileURLToPath(u);
   return u.href;
 }
+
+/**
+ * Parse a input parameter as a URL.
+ *
+ * If it looks like a file path, it will be converted using `pathToFileURL`.
+ **/
+export const Url: Type<string, URL> = {
+  from(str) {
+    try {
+      return Promise.resolve(new URL(str));
+    } catch (e) {
+      return Promise.resolve(pathToFileURL(str));
+    }
+  },
+};
+
+/**
+ * Parse a input parameter as a URL which represents a folder.
+ *
+ * If it looks like a file path, it will be converted using `pathToFileURL`.
+ * Any search parameters or hash will be removed, and a trailing slash added
+ * to the path section if it's not present.
+ **/
+export const UrlFolder: Type<string, URL> = {
+  async from(str) {
+    const url = await Url.from(str);
+    url.search = '';
+    url.hash = '';
+    if (!url.pathname.endsWith('/')) url.pathname += '/';
+    return url;
+  },
+};
diff --git a/src/commands/index.ts b/src/commands/index.ts
@@ -9,6 +9,7 @@ import { commandPrettyPrint } from './format/pretty.print.js';
 import { commandGroup } from './group/group.js';
 import { commandLdsFetch } from './lds-cache/lds.cache.js';
 import { commandList } from './list/list.js';
+import { commandMapSheetCoverage } from './mapsheet-coverage/mapsheet.coverage.js';
 import { commandGeneratePath } from './path/path.generate.js';
 import { commandStacCatalog } from './stac-catalog/stac.catalog.js';
 import { commandStacGithubImport } from './stac-github-import/stac.github.import.js';
@@ -24,6 +25,7 @@ export const AllCommands = {
   'lds-fetch-layer': commandLdsFetch,
   list: commandList,
   ls: commandList,
+  'mapsheet-coverage': commandMapSheetCoverage,
   'stac-catalog': commandStacCatalog,
   'stac-github-import': commandStacGithubImport,
   'stac-sync': commandStacSync,

diff --git a/src/commands/mapsheet-coverage/README.md b/src/commands/mapsheet-coverage/README.md
@@ -0,0 +1,26 @@
+# mapsheet-coverage
+
+Create a list of mapsheets needing to be created from a basemaps configuration
+
+## Usage
+
+mapsheet-coverage <options>
+
+### Options
+
+| Usage                | Description                                                  | Options                                                                                              |
+| -------------------- | ------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------- |
+| --config <str>       | Location of role configuration file                          | optional                                                                                             |
+| --epsg-code <number> | Basemaps configuration layer ESPG code to use                | default: 2193                                                                                        |
+| --location <value>   | Location of the basemaps configuration file                  | default: https://raw.githubusercontent.com/linz/basemaps-config/master/config/tileset/elevation.json |
+| --mapsheet <str>     | Limit the output to a specific mapsheet eg "BX01"            | optional                                                                                             |
+| --compare <str>      | Compare the output with an existing combined collection.json | optional                                                                                             |
+| --output <value>     | Where to store output files                                  | default: file:///tmp/mapsheet-coverage/                                                              |
+
+### Flags
+
+| Usage     | Description     | Options |
+| --------- | --------------- | ------- |
+| --verbose | Verbose logging |         |
+
+<!-- This file has been autogenerated by src/readme.generate.ts -->
diff --git a/src/commands/mapsheet-coverage/__test__/mapsheet.coverage.test.ts b/src/commands/mapsheet-coverage/__test__/mapsheet.coverage.test.ts
@@ -0,0 +1,188 @@
+import assert from 'node:assert';
+import { afterEach, before, describe, it } from 'node:test';
+
+import { Nztm2000QuadTms, Projection } from '@basemaps/geo';
+import { fsa } from '@chunkd/fs';
+import { FsMemory } from '@chunkd/source-memory';
+import { featuresToMultiPolygon } from '@linzjs/geojson';
+import { StacCollection } from 'stac-ts';
+
+import { MapSheet } from '../../../utils/mapsheet.js';
+import { commandMapSheetCoverage } from '../mapsheet.coverage.js';
+
+// convert a collection of map sheets into a multipolygon
+function mapSheetToGeoJson(...sheetCodes: string[]): GeoJSON.Feature {
+  const features = sheetCodes.map((sc) => {
+    const ms = MapSheet.getMapTileIndex(sc);
+    if (ms == null) throw new Error('Invalid mapsheet');
+    return Projection.get(Nztm2000QuadTms).boundsToGeoJsonFeature(ms, { sheetCode: sc });
+  });
+  return { geometry: featuresToMultiPolygon(features), type: 'Feature', properties: {} };
+}
+
+/** Convert a collection's links to their approx map sheet area */
+function collectionToCaptureArea(c: StacCollection): GeoJSON.Feature {
+  return mapSheetToGeoJson(
+    ...c.links.filter((f) => f.rel === 'item').map((m) => m.href.replace('./', '').replace('.json', '')),
+  );
+}
+
+function fakeCollection(id: string): StacCollection {
+  return {
+    stac_version: '1.0.0',
+    type: 'Collection',
+    license: 'CC-BY-4.0',
+    id: `layer-${id}-id`,
+    title: `Layer ${id.toUpperCase()} Title`,
+    description: `Layer ${id.toUpperCase()} Description`,
+    assets: { capture_area: { href: './capture-area.json' } },
+    links: [],
+  } as unknown as StacCollection;
+}
+
+describe('mapsheet-coverage', () => {
+  const mem = new FsMemory();
+  const baseArgs = {
+    epsgCode: 2193,
+    location: new URL('ms://config.json'),
+    output: new URL('ms://output/'),
+    compare: undefined,
+    verbose: false,
+    mapSheet: undefined,
+    config: undefined,
+  } as const;
+
+  before(() => {
+    fsa.register('ms://', mem);
+  });
+
+  afterEach(() => {
+    mem.files.clear();
+  });
+
+  it('should run with a empty config', async () => {
+    await fsa.write('ms://config.json', JSON.stringify({ layers: [] }));
+    await commandMapSheetCoverage.handler(baseArgs);
+
+    const files = await fsa.toArray(fsa.list('ms://output/'));
+    files.sort();
+    assert.deepEqual(files, [
+      'ms://output/capture-dates.geojson',
+      'ms://output/file-list.json',
+      'ms://output/layers-source.geojson.gz',
+    ]);
+
+    const fileList = await fsa.readJson('ms://output/file-list.json');
+    assert.deepEqual(fileList, []);
+  });
+
+  it('should error if collection is missing', async () => {
+    await fsa.write('ms://config.json', JSON.stringify({ layers: [{ 2193: 'ms://layers/a/', name: 'layer-a' }] }));
+    const out = await commandMapSheetCoverage.handler(baseArgs).catch((e) => e as Error);
+    assert.equal(String(out), 'CompositeError: Not found');
+  });
+
+  it('should error if collection has no capture-area', async () => {
+    await fsa.write('ms://config.json', JSON.stringify({ layers: [{ 2193: 'ms://layers/a/', name: 'layer-a' }] }));
+    await fsa.write('ms://layers/a/collection.json', JSON.stringify({}));
+    const out = await commandMapSheetCoverage.handler(baseArgs).catch((e) => e as Error);
+    assert.equal(String(out), 'Error: Missing capture area asset in collection "ms://layers/a/collection.json"');
+  });
+
+  it('should cover has no capture-area', async () => {
+    const colA = fakeCollection('a');
+    colA.links.push({ rel: 'item', href: './BP27.json' });
+
+    await fsa.write('ms://config.json', JSON.stringify({ layers: [{ 2193: 'ms://layers/a/', name: 'layer-a' }] }));
+    await fsa.write('ms://layers/a/collection.json', JSON.stringify(colA));
+    await fsa.write('ms://layers/a/capture-area.json', JSON.stringify(mapSheetToGeoJson('BP27')));
+
+    const out = await commandMapSheetCoverage.handler(baseArgs).catch((e) => e as Error);
+    assert.equal(out, undefined);
+
+    const fileList = await fsa.readJson('ms://output/file-list.json');
+    assert.deepEqual(fileList, [{ output: 'BP27', input: ['ms://layers/a/BP27.tiff'], includeDerived: true }]);
+
+    const captureDates = await fsa.readJson<GeoJSON.FeatureCollection>('ms://output/capture-dates.geojson');
+
+    assert.deepEqual(captureDates.features[0]?.properties, {
+      title: 'Layer A Title',
+      description: 'Layer A Description',
+      id: 'layer-a-id',
+      license: 'CC-BY-4.0',
+      source: 'ms://layers/a/collection.json',
+    });
+    assert.equal(captureDates.features.length, 1);
+  });
+
+  it('should include files with overlap', async () => {
+    const colA = fakeCollection('a');
+    colA.links.push({ rel: 'item', href: './BP27.json' });
+    colA.links.push({ rel: 'item', href: './BP28.json' });
+
+    const colB = fakeCollection('b');
+    colB.links.push({ rel: 'item', href: './BP27.json' });
+
+    await fsa.write(
+      'ms://config.json',
+      JSON.stringify({
+        layers: [
+          { 2193: 'ms://layers/a/', name: 'layer-a' },
+          { 2193: 'ms://layers/b/', name: 'layer-b' },
+        ],
+      }),
+    );
+    await fsa.write('ms://layers/a/collection.json', JSON.stringify(colA));
+    await fsa.write('ms://layers/a/capture-area.json', JSON.stringify(collectionToCaptureArea(colA)));
+    await fsa.write('ms://layers/b/collection.json', JSON.stringify(colB));
+    await fsa.write('ms://layers/b/capture-area.json', JSON.stringify(collectionToCaptureArea(colB)));
+
+    const out = await commandMapSheetCoverage.handler(baseArgs).catch((e) => e as Error);
+    assert.equal(out, undefined);
+
+    const fileList = await fsa.readJson('ms://output/file-list.json');
+    assert.deepEqual(fileList, [
+      { output: 'BP27', input: ['ms://layers/a/BP27.tiff', 'ms://layers/b/BP27.tiff'], includeDerived: true },
+      { output: 'BP28', input: ['ms://layers/a/BP28.tiff'], includeDerived: true },
+    ]);
+
+    const captureDates = await fsa.readJson<GeoJSON.FeatureCollection>('ms://output/capture-dates.geojson');
+
+    assert.deepEqual(captureDates.features[0]?.properties?.['source'], 'ms://layers/b/collection.json');
+    assert.deepEqual(captureDates.features[1]?.properties?.['source'], 'ms://layers/a/collection.json');
+    assert.equal(captureDates.features.length, 2);
+  });
+
+  it('should exclude fully overlapping files', async () => {
+    const colA = fakeCollection('a');
+    colA.links.push({ rel: 'item', href: './BP27.json' });
+
+    const colB = fakeCollection('b');
+    colB.links.push({ rel: 'item', href: './BP27.json' });
+
+    await fsa.write(
+      'ms://config.json',
+      JSON.stringify({
+        layers: [
+          { 2193: 'ms://layers/a/', name: 'layer-a' },
+          { 2193: 'ms://layers/b/', name: 'layer-b' },
+        ],
+      }),
+    );
+    await fsa.write('ms://layers/a/collection.json', JSON.stringify(colA));
+    await fsa.write('ms://layers/a/capture-area.json', JSON.stringify(collectionToCaptureArea(colA)));
+    await fsa.write('ms://layers/b/collection.json', JSON.stringify(colB));
+    await fsa.write('ms://layers/b/capture-area.json', JSON.stringify(collectionToCaptureArea(colB)));
+
+    const out = await commandMapSheetCoverage.handler(baseArgs).catch((e) => e as Error);
+    assert.equal(out, undefined);
+
+    const fileList = await fsa.readJson('ms://output/file-list.json');
+    assert.deepEqual(fileList, [{ output: 'BP27', input: ['ms://layers/b/BP27.tiff'], includeDerived: true }]);
+
+    const captureDates = await fsa.readJson<GeoJSON.FeatureCollection>('ms://output/capture-dates.geojson');
+
+    assert.deepEqual(captureDates.features[0]?.properties?.['source'], 'ms://layers/b/collection.json');
+    assert.equal(captureDates.features.length, 1);
+  });
+});