Use streams instead of in memory stores

Significant performance improvements from my local testing

README.md

@@ -11,10 +11,9 @@ Usage: geotiff2geojson [options]
 Options:
   -i, --input <file>         GeoTIFF file
   -f, --filter <expression>  Filter out features, value is assigned to "d"
-                             (default: "d !== -999000000")
-  -o, --output <file>        GeoJSON file to write, standard out used if
-                             missing
+  -o, --output <file>        GeoJSON file to write, standard out used if missing
   -p, --proj <projection>    Projection to use (default: "WGS84")
+  -v, --verbose              Provide more detailed output, use with -o only
   --pretty-print             Pretty-print GeoJSON
   -h, --help                 display help for command
 ```

index.mjs

@@ -1,31 +1,43 @@
 #!/usr/bin/env node
 
+import fs from 'fs'
+import path from 'path'
 import vm from 'vm'
-import { promises as fs } from 'fs'
 
 import { Command } from 'commander/esm.mjs';
 import { fromFile } from 'geotiff'
 import geokeysToProj4 from 'geotiff-geokeys-to-proj4'
 import proj4 from 'proj4'
+import Progress from 'progress'
+import transform from 'stream-transform'
 
+const cwd = process.cwd()
 const program = new Command()
 
 program
     .option('-i, --input <file>', 'GeoTIFF file ')
     .option('-f, --filter <expression>', 'Filter out features, value is assigned to "d"')
     .option('-o, --output <file>', 'GeoJSON file to write, standard out used if missing')
     .option('-p, --proj <projection>', 'Projection to use', 'WGS84')
+    .option('-v, --verbose', 'Provide more detailed output, use with -o only')
     .option('--pretty-print', 'Pretty-print GeoJSON')
 
 program.parse(process.argv)
 
 const options = program.opts()
+const { input, prettyPrint, output, verbose } = options
 
-if (!options.input) {
+if (!input) {
     throw new Error('Input file required, use -i flag or --help for more information')
 }
 
-const tiff = await fromFile(options.input)
+if (!output && verbose) {
+    throw new Error('Specify an output file if you use -v otherwise stdout will be polluted')
+}
+
+const writeStream = output ? fs.createWriteStream(path.resolve(cwd, output)) : process.stdout
+
+const tiff = await fromFile(input)
 const image = await tiff.getImage()
 const geoKeys = image.getGeoKeys()
 const projObj = geokeysToProj4.toProj4(geoKeys)
@@ -38,30 +50,42 @@ const resolution = image.getResolution()
 const xSize = resolution[0]
 const ySize = resolution[1]
 
-const features = []
-
 let raster = await image.readRasters({ window: [0, 0, maxX, maxY] });
-let color0 = raster[0]; // Raster is a TypedArray where elements are colors and their elements are pixel values of that color
-
-color0.forEach((d, i) => {
-    const context = { d }
-    vm.createContext(context)
-    if (options.filter && !vm.runInContext(options.filter, context)) return
+let color0 = raster[0]
+let isFirst = true
+
+const bar = verbose ? new Progress('Processing [:bar] :percent :etas', {
+    total: color0.length,
+    width: 40,
+}) : null
+
+
+const transformer = transform(args => {
+    const [d, i, total] = args
+    if (options.filter) {
+        const context = { d }
+        vm.createContext(context)
+        if (!vm.runInContext(options.filter, context)) {
+            return null
+        }
+    }
 
     const y = Math.floor(i / maxX)
     const x = i % maxX
-        // Convert current pixel's coordinates to CRS by:
-        // 1. Multiplying current coordinates by pixel size which will result in distance from top-left corner in CRS units.
-        // 2. Adding this value to top-left corner coordinates which will result in "global" coordinates in CRS units.
+
+    // Convert current pixel's coordinates to CRS by:
+    // 1. Multiplying current coordinates by pixel size which will result in distance from top-left corner in CRS units.
+    // 2. Adding this value to top-left corner coordinates which will result in "global" coordinates in CRS units.
     let crsX = origin[0] + x * xSize
     let crsY = origin[1] + y * ySize
 
     // Check if coordinates are already in meters (or other "standard" units). If not, convert them.
     let point;
-    if (projObj.shouldConvertCoordinates)
+    if (projObj.shouldConvertCoordinates) {
         point = geokeysToProj4.convertCoordinates(crsX, crsY, projObj.coordinatesConversionParameters);
-    else
+    } else {
         point = { x: crsX, y: crsY };
+    }
     // Or just multiply manually to speed up execution by removing function calls:
     point = {
         x: crsX * projObj.coordinatesConversionParameters.x,
@@ -71,7 +95,7 @@ color0.forEach((d, i) => {
     let projectedPoint = projection.forward(point); // Project these coordinates
 
     // Work with projected coordinates
-    features.push({
+    const feature = {
         "type": "Feature",
         "properties": {
             value: d,
@@ -80,13 +104,18 @@ color0.forEach((d, i) => {
             "type": "Point",
             "coordinates": [projectedPoint.x, projectedPoint.y]
         }
-    })
+    }
+    const geojson = prettyPrint ? JSON.stringify(feature, null, 2) : JSON.stringify(feature)
+    const row = `${isFirst ? '' : ','}${geojson}\n${i === total - 1 ? ']}' : ''}`
+    isFirst = false
+    if (bar) bar.tick()
+    return row
 })
 
-const geojson = options.prettyPrint ? JSON.stringify({ type: 'FeatureCollection', features }, null, 2) : JSON.stringify({ type: 'FeatureCollection', features })
+writeStream.write('{ "type": "FeatureCollection", "features": [\n')
+
+transformer.pipe(writeStream)
 
-if (options.output) {
-    await fs.writeFile(options.output, geojson)
-} else {
-    process.stdout.write(geojson)
-}
+color0.forEach((d, i) => {
+    transformer.write([d, i, color0.length])
+})

package.json

@@ -17,9 +17,11 @@
         "commander": "^8.2.0",
         "geotiff": "^1.0.6",
         "geotiff-geokeys-to-proj4": "^2021.7.19-beta",
-        "proj4": "^2.7.5"
+        "progress": "^2.0.3",
+        "proj4": "^2.7.5",
+        "stream-transform": "^2.1.3"
     },
     "bin": {
         "geotiff2geojson": "./index.mjs"
     }
-}
+}