HARMONY-1876: Add shapefile simplification to try to speed up l2ss

services/harmony/app/frontends/free-text-query.ts

@@ -166,6 +166,7 @@ interface HarmonyJobStatus {
 async function submitHarmonyRequest(collection, variable, queryParams, geoJson: string, token): Promise<HarmonyJobStatus> {
   queryParams.forceAsync = true;
   queryParams.maxResults = 1;
+  queryParams.simplifyShapefile = true;
   const encodedVariable = encodeURIComponent(variable);
   const baseUrl = `http://localhost:3000/${collection}/ogc-api-coverages/1.0.0/collections/${encodedVariable}/coverage/rangeset`;
   const querystr = querystring.stringify(queryParams);
@@ -186,6 +187,7 @@ async function submitHarmonyRequest(collection, variable, queryParams, geoJson:
     filename: 'data.geojson',
     contentType: 'application/geo+json',
   });
+  // formData.append('simplifyShapefile', 'true');
 
   console.log(`Making request to ${baseUrl}?${querystr}`);
 

services/harmony/app/middleware/shapefile-converter.ts

@@ -3,6 +3,7 @@ import { get, isEqual, cloneDeep } from 'lodash';
 import rewind from '@mapbox/geojson-rewind';
 import * as togeojson from '@tmcw/togeojson';
 import splitGeoJson from 'geojson-antimeridian-cut';
+import * as turf from '@turf/turf';
 
 import { DOMParser } from '@xmldom/xmldom';
 import * as shpjs from 'shpjs';
@@ -15,6 +16,9 @@ import { RequestValidationError, HttpError, ServerError } from '../util/errors';
 import { defaultObjectStore } from '../util/object-store';
 import { listToText } from '@harmony/util/string';
 import { cookieOptions } from '../util/cookies';
+import HarmonyRequest from '../models/harmony-request';
+import { keysToLowerCase } from '../util/object';
+import { point } from 'leaflet';
 
 /**
  * Converts the given ESRI Shapefile to GeoJSON and returns the resulting file.   Note,
@@ -143,9 +147,10 @@ export function normalizeGeoJsonCoords(geojson: any): any {
  * Change longitudes of a geojson file to be in the [-180, 180] range and split at antimeridian
  * if needed. Will also change coordinate order to counter-clockwise if needed.
  * @param geoJson - An object representing the json for a geojson file
+ * @parm simplifyShapefile - if true, reduce the point count of the shapefile - defaults to false
  * @returns An object with the normalized geojson
  */
-export function normalizeGeoJson(geoJson: object): object {
+export function normalizeGeoJson(geoJson: object, simplifyShapefile = false): object {
   let newGeoJson = normalizeGeoJsonCoords(geoJson);
 
   // eslint-disable-next-line @typescript-eslint/dot-notation
@@ -159,16 +164,39 @@ export function normalizeGeoJson(geoJson: object): object {
 
   // force ccw winding
   newGeoJson = rewind(newGeoJson, false);
+
+  if (simplifyShapefile) {
+    let tolerance = 0.1;
+    const highQuality = true;
+    let pointCount = geoJsonPointCount(newGeoJson);
+    console.log(`INITIAL POINT COUNT: ${pointCount}`);
+    let loopCount = 0;
+    // loop up to five times, or until the point count of the geojson is <= 5000
+    // TODO put these constants (5, 5000) in env vars or some such
+    while (loopCount < 5 && pointCount > 5000) {
+      newGeoJson = turf.simplify(newGeoJson, { tolerance, highQuality });
+      pointCount = geoJsonPointCount(newGeoJson);
+      console.log(`POINT COUNT: ${pointCount}`);
+      tolerance = tolerance * 10.0;
+      loopCount++;
+    }
+    console.log(`FINAL POINT COUNT: ${pointCount}`);
+  } else {
+    console.log("NOT SIMPLIFYING SHAPEFILE");
+  }
+  console.log(`SHAPEFILE: ${JSON.stringify(newGeoJson, null, 2)}`);
+
   return newGeoJson;
 }
 
 /**
  * Handle any weird cases like splitting geometry that crosses the antimeridian
  * @param url - the url of the geojson file
  * @param isLocal - whether the url is a downloaded file (true) or needs to be downloaded (false)
+ * @parm simplifyShapefile - if true, reduce the point count of the shapefile - defaults to false
  * @returns the link to the geojson file
  */
-async function normalizeGeoJsonFile(url: string, isLocal: boolean): Promise<string> {
+async function normalizeGeoJsonFile(url: string, isLocal: boolean, simplifyShapefle = false): Promise<string> {
   const store = defaultObjectStore();
   let originalGeoJson: object;
   const localFile = url;
@@ -177,7 +205,7 @@ async function normalizeGeoJsonFile(url: string, isLocal: boolean): Promise<stri
   } else {
     originalGeoJson = (await fs.readFile(localFile)).toJSON();
   }
-  const normalizedGeoJson = normalizeGeoJson(originalGeoJson);
+  const normalizedGeoJson = normalizeGeoJson(originalGeoJson, simplifyShapefle);
 
   let resultUrl = url;
 
@@ -189,17 +217,37 @@ async function normalizeGeoJsonFile(url: string, isLocal: boolean): Promise<stri
   return resultUrl;
 }
 
+/**
+ * Get the count of the points in a geojson file
+ * @param geoJson A geojson object
+ * @returns the number of points in the geojson file
+ */
+function geoJsonPointCount(geoJson: turf.AllGeoJSON): number {
+  let pointCount = 0;
+  turf.coordEach(geoJson, () => {
+    pointCount += 1;
+  });
+  return pointCount;
+}
+
 /**
  * Express.js middleware which extracts shapefiles from the incoming request and
- * ensures that they are in GeoJSON in the data operation
+ * ensures that they are in GeoJSON in the data operation, optionally reducing the number
+ * of points in the shapefile to fit the CMR limit (5000)
  *
  * @param req - The client request, containing an operation
  * @param res - The client response
  * @param next - The next function in the middleware chain
  */
-export default async function shapefileConverter(req, res, next: NextFunction): Promise<void> {
+export default async function shapefileConverter(req: HarmonyRequest, res, next: NextFunction): Promise<void> {
   const { operation } = req;
 
+  // TODO add proper handling/documentation of the simplifyShapefile parameter
+  const lowerCaseQuery = keysToLowerCase(req.query);
+  const lowerCaseBody = keysToLowerCase(req.body);
+  const simplifyShapefile = lowerCaseQuery.simplifyshapefile || lowerCaseBody.simplifyshapefile;
+  // const simplifyShapefile = true;
+
   try {
     const shapefile = get(req, 'files.shapefile[0]') || get(req, 'file') || req.signedCookies.shapefile;
     res.clearCookie('shapefile', cookieOptions);
@@ -224,14 +272,19 @@ export default async function shapefileConverter(req, res, next: NextFunction):
       let convertedFile;
       try {
         convertedFile = await converter.geoJsonConverter(originalFile, req.context.logger);
+        // TODO handle simplification for converted file
         operation.geojson = await store.uploadFile(convertedFile, `${url}.geojson`);
       } finally {
         if (convertedFile) {
           await fs.unlink(convertedFile);
         }
       }
     } else {
-      operation.geojson = await normalizeGeoJsonFile(url, false);
+      console.log('NO NEED TO CONVERT GEOJSON');
+      let geoJson = await normalizeGeoJsonFile(url, false, simplifyShapefile);
+
+
+      operation.geojson = geoJson;
     }
   } catch (e) {
     if (e instanceof HttpError) {

services/harmony/app/schemas/ogc-api-coverages/1.0.0/ogc-api-coverages-v1.0.0.yml

@@ -345,6 +345,7 @@ paths:
         - $ref: "#/components/parameters/variable"
         - $ref: "#/components/parameters/average"
         - $ref: "#/components/parameters/label"
+        - $ref: "#/components/parameters/simplifyShapefile"
       responses:
         "200":
           description: A coverage's range set.
@@ -388,6 +389,7 @@ paths:
         - $ref: "#/components/parameters/variable"
         - $ref: "#/components/parameters/average"
         - $ref: "#/components/parameters/label"
+        - $ref: "#/components/parameters/simplifyShapefile"
       requestBody:
         content:
           multipart/form-data:
@@ -1008,4 +1010,11 @@ components:
         type: array
         items:
           type: string
-          minLength: 1
+          minLength: 1
+    simplifyShapefile:
+      name: simplifyShapefile
+      in: query
+      description: if "true", reduce the point count in uploaded shapefile
+      required: false
+      schema:
+        type: boolean

services/harmony/app/server.ts

@@ -142,6 +142,7 @@ function buildBackendServer(port: number, hostBinding: string, useHttps: string)
 function buildFrontendServer(port: number, hostBinding: string, config: RouterConfig): http.Server | https.Server {
   const appLogger = logger.child({ application: 'frontend' });
   const app = express();
+  app.use(express.json({ limit: '10mb' }));
   app.set('query parser', (str) =>
     qs.parse(str, { comma: true }),
   );