Open-EO/openeo-geotrellis-extensions

Band name with "/" for netCDF output throws NPE

Open

#323 opened on Sep 20, 2024

 (1 comment) (0 reactions) (0 assignees)Scala (5 forks)auto 404
bugenhancementgood first issue

Repository metrics

Stars
 (7 stars)
PR merge metrics
 (PR metrics pending)

Description

This process graph will throw a NPE and it's not immediately apparent what causes it:

{
  "process_graph": {
    "applydimension1": {
      "arguments": {
        "data": {
          "from_node": "loadcollection1"
        },
        "dimension": "bands",
        "process": {
          "process_graph": {
            "arrayappend1": {
              "arguments": {
                "data": {
                  "from_parameter": "data"
                },
                "value": {
                  "from_node": "divide1"
                }
              },
              "process_id": "array_append",
              "result": true
            },
            "arrayelement1": {
              "arguments": {
                "data": {
                  "from_parameter": "data"
                },
                "index": 1
              },
              "process_id": "array_element"
            },
            "arrayelement2": {
              "arguments": {
                "data": {
                  "from_parameter": "data"
                },
                "index": 0
              },
              "process_id": "array_element"
            },
            "divide1": {
              "arguments": {
                "x": {
                  "from_node": "arrayelement1"
                },
                "y": {
                  "from_node": "arrayelement2"
                }
              },
              "process_id": "divide"
            }
          }
        }
      },
      "process_id": "apply_dimension"
    },
    "loadcollection1": {
      "arguments": {
        "bands": [
          "VV",
          "VH"
        ],
        "id": "SENTINEL1_GLOBAL_MOSAICS",
        "spatial_extent": {
          "crs": "EPSG:32755",
          "east": 576780,
          "north": 5246260,
          "south": 5244980,
          "west": 575500
        },
        "temporal_extent": [
          "2019-12-31T23:59:59Z",
          "2021-01-01T00:00:00Z"
        ]
      },
      "process_id": "load_collection"
    },
    "renamelabels1": {
      "arguments": {
        "data": {
          "from_node": "applydimension1"
        },
        "dimension": "bands",
        "target": [
          "VV",
          "VH",
          "VH/VV"
        ]
      },
      "process_id": "rename_labels"
    },
    "saveresult1": {
      "arguments": {
        "data": {
          "from_node": "renamelabels1"
        },
        "format": "NetCDF",
        "options": {
          "filename_prefix": "lcfm-s1-55GEN_062_19"
        }
      },
      "process_id": "save_result",
      "result": true
    }
  }
}
Traceback (most recent call last):
  File "/opt/openeo/lib/python3.8/site-packages/openeogeotrellis/deploy/batch_job.py", line 518, in start_main
    main(sys.argv)
  File "/opt/openeo/lib/python3.8/site-packages/openeogeotrellis/deploy/batch_job.py", line 194, in main
    run_driver()
  File "/opt/openeo/lib/python3.8/site-packages/openeogeotrellis/deploy/batch_job.py", line 165, in run_driver
    run_job(
  File "/opt/openeo/lib/python3.8/site-packages/openeogeotrellis/utils.py", line 56, in memory_logging_wrapper
    return function(*args, **kwargs)
  File "/opt/openeo/lib/python3.8/site-packages/openeogeotrellis/deploy/batch_job.py", line 315, in run_job
    the_assets_metadata = result.write_assets(str(output_file))
  File "/opt/openeo/lib/python3.8/site-packages/openeo_driver/save_result.py", line 150, in write_assets
    return self.cube.write_assets(filename=directory, format=self.format, format_options=self.options)
  File "/opt/openeo/lib/python3.8/site-packages/openeogeotrellis/geopysparkdatacube.py", line 1966, in write_assets
    asset_paths = get_jvm().org.openeo.geotrellis.netcdf.NetCDFRDDWriter.writeRasters(
  File "/usr/local/spark/python/lib/py4j-0.10.9.7-src.zip/py4j/java_gateway.py", line 1322, in __call__
    return_value = get_return_value(
  File "/usr/local/spark/python/lib/py4j-0.10.9.7-src.zip/py4j/protocol.py", line 326, in get_return_value
    raise Py4JJavaError(
py4j.protocol.Py4JJavaError: An error occurred while calling z:org.openeo.geotrellis.netcdf.NetCDFRDDWriter.writeRasters.
: java.lang.NullPointerException
	at ucar.nc2.jni.netcdf.Nc4Iosp.writeData(Nc4Iosp.java:2986)
	at ucar.nc2.NetcdfFileWriter.write(NetcdfFileWriter.java:1066)
	at ucar.nc2.NetcdfFileWriter.write(NetcdfFileWriter.java:1050)
	at org.openeo.geotrellis.netcdf.NetCDFRDDWriter$.writeTile(NetCDFRDDWriter.scala:308)
	at org.openeo.geotrellis.netcdf.NetCDFRDDWriter$.$anonfun$saveSingleNetCDFGeneric$5(NetCDFRDDWriter.scala:217)
	at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:158)
	at org.openeo.geotrellis.netcdf.NetCDFRDDWriter$.$anonfun$saveSingleNetCDFGeneric$3(NetCDFRDDWriter.scala:195)
	at org.openeo.geotrellis.netcdf.NetCDFRDDWriter$.$anonfun$saveSingleNetCDFGeneric$3$adapted(NetCDFRDDWriter.scala:158)
	at scala.collection.Iterator.foreach(Iterator.scala:943)
	at scala.collection.Iterator.foreach$(Iterator.scala:943)
	at scala.collection.AbstractIterator.foreach(Iterator.scala:1431)
	at org.openeo.geotrellis.netcdf.NetCDFRDDWriter$.saveSingleNetCDFGeneric(NetCDFRDDWriter.scala:158)
	at org.openeo.geotrellis.netcdf.NetCDFRDDWriter$.saveSingleNetCDFGeneric(NetCDFRDDWriter.scala:108)
	at org.openeo.geotrellis.netcdf.NetCDFRDDWriter$.writeRasters(NetCDFRDDWriter.scala:80)
	at org.openeo.geotrellis.netcdf.NetCDFRDDWriter.writeRasters(NetCDFRDDWriter.scala)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
	at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)
	at py4j.Gateway.invoke(Gateway.java:282)
	at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
	at py4j.commands.CallCommand.execute(CallCommand.java:79)
	at py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)
	at py4j.ClientServerConnection.run(ClientServerConnection.java:106)
	at java.base/java.lang.Thread.run(Thread.java:829)

The same process graph works fine for e.g. GeoTiff.

Maybe this one-way replace has something to do with it?

https://github.com/Open-EO/openeo-geotrellis-extensions/blob/9782c10f65510f0ff7214e15bb573c701f6afb6f/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/netcdf/NetCDFRDDWriter.scala#L692

Possible courses of action:

  • silently replace illegal characters but make sure to avoid this error further on;
  • throw a clearer error.

Contributor guide