Merge pull request #53 from terraref/canopycover-updates

Canopycover logic updates

bin2tif/terra_bin2tif.py

@@ -46,17 +46,13 @@ def check_message(self, connector, host, secret_key, resource, parameters):
         # Check metadata to verify we have what we need
         md = download_metadata(connector, host, secret_key, resource['id'])
         if get_terraref_metadata(md):
-            if get_extractor_metadata(md, self.extractor_info['name'], self.extractor_info['version']):
+            if not self.overwrite and get_extractor_metadata(md, self.extractor_info['name'], self.extractor_info['version']):
                 # Make sure outputs properly exist
                 timestamp = resource['dataset_info']['name'].split(" - ")[1]
                 left_tiff = self.sensors.create_sensor_path(timestamp, opts=['left'])
                 right_tiff = self.sensors.create_sensor_path(timestamp, opts=['right'])
                 if file_exists(left_tiff) and file_exists(right_tiff):
-                    if contains_required_files(resource, [os.path.basename(left_tiff), os.path.basename(right_tiff)]):
-                        self.log_skip(resource, "metadata v%s and outputs already exist" % self.extractor_info['version'])
-                        return CheckMessage.ignore
-                    else:
-                        self.log_info(resource, "output files exist but not yet uploaded")
+                    self.log_skip(resource, "metadata v%s and outputs already exist" % self.extractor_info['version'])
             # Have TERRA-REF metadata, but not any from this extractor
             return CheckMessage.download
         else:
@@ -98,15 +94,17 @@ def process_message(self, connector, host, secret_key, resource, parameters):
         right_tiff = self.sensors.create_sensor_path(timestamp, opts=['right'])
         uploaded_file_ids = []
 
-        # Attach LemnaTec source metadata to Level_1 product
-        self.log_info(resource, "uploading LemnaTec metadata to ds [%s]" % target_dsid)
-        remove_metadata(connector, host, secret_key, target_dsid, self.extractor_info['name'])
-        terra_md_trim = get_terraref_metadata(all_dsmd)
-        if updated_experiment is not None:
-            terra_md_trim['experiment_metadata'] = updated_experiment
-        terra_md_trim['raw_data_source'] = host + ("" if host.endswith("/") else "/") + "datasets/" + resource['id']
-        level1_md = build_metadata(host, self.extractor_info, target_dsid, terra_md_trim, 'dataset')
-        upload_metadata(connector, host, secret_key, target_dsid, level1_md)
+        # Attach LemnaTec source metadata to Level_1 product if necessary
+        target_md = download_metadata(connector, host, secret_key, target_dsid)
+        if not get_extractor_metadata(target_md, self.extractor_info['name']):
+            self.log_info(resource, "uploading LemnaTec metadata to ds [%s]" % target_dsid)
+            remove_metadata(connector, host, secret_key, target_dsid, self.extractor_info['name'])
+            terra_md_trim = get_terraref_metadata(all_dsmd)
+            if updated_experiment is not None:
+                terra_md_trim['experiment_metadata'] = updated_experiment
+            terra_md_trim['raw_data_source'] = host + ("" if host.endswith("/") else "/") + "datasets/" + resource['id']
+            level1_md = build_metadata(host, self.extractor_info, target_dsid, terra_md_trim, 'dataset')
+            upload_metadata(connector, host, secret_key, target_dsid, level1_md)
 
         try:
             left_shape = terraref.stereo_rgb.get_image_shape(terra_md_full, 'left')
@@ -127,8 +125,8 @@ def process_message(self, connector, host, secret_key, resource, parameters):
             self.created += 1
             self.bytes += os.path.getsize(left_tiff)
         # Check if the file should be uploaded, even if it was already created
-        found_in_dest = check_file_in_dataset(connector, host, secret_key, target_dsid, left_tiff, remove=self.overwrite)
-        if not found_in_dest or self.overwrite:
+        found_in_dest = check_file_in_dataset(connector, host, secret_key, target_dsid, left_tiff)
+        if not found_in_dest:
             self.log_info(resource, "uploading %s" % left_tiff)
             fileid = upload_to_dataset(connector, host, self.clowder_user, self.clowder_pass, target_dsid, left_tiff)
             uploaded_file_ids.append(host + ("" if host.endswith("/") else "/") + "files/" + fileid)
@@ -143,8 +141,8 @@ def process_message(self, connector, host, secret_key, resource, parameters):
             self.created += 1
             self.bytes += os.path.getsize(right_tiff)
         # Check if the file should be uploaded, even if it was already created
-        found_in_dest = check_file_in_dataset(connector, host, secret_key, target_dsid, right_tiff, remove=self.overwrite)
-        if not found_in_dest or self.overwrite:
+        found_in_dest = check_file_in_dataset(connector, host, secret_key, target_dsid, right_tiff)
+        if not found_in_dest:
             self.log_info(resource, "uploading %s" % right_tiff)
             fileid = upload_to_dataset(connector, host, self.clowder_user, self.clowder_pass, target_dsid, right_tiff)
             uploaded_file_ids.append(host + ("" if host.endswith("/") else "/") + "files/" + fileid)
@@ -162,7 +160,10 @@ def process_message(self, connector, host, secret_key, resource, parameters):
             }, 'dataset')
             self.log_info(resource, "uploading extractor metadata to raw dataset")
             remove_metadata(connector, host, secret_key, resource['id'], self.extractor_info['name'])
-            upload_metadata(connector, host, secret_key, resource['id'], extractor_md)
+            try:
+                upload_metadata(connector, host, secret_key, resource['id'], extractor_md)
+            except:
+                self.log_info(resource, "problem uploading extractor metadata...")
 
         self.end_message(resource)
 

canopycover/extractor_info.json

@@ -1,7 +1,7 @@
 {
   "@context": "http://clowder.ncsa.illinois.edu/contexts/extractors.jsonld",
   "name": "terra.stereo-rgb.canopycover",
-  "version": "2.3",
+  "version": "2.6",
   "description": "Canopy Cover by Plot (Percentage of Green Pixels)",
   "author": "Zongyang Li",
   "contributors": [],

canopycover/terra_canopycover.py

@@ -3,7 +3,7 @@
 import json
 import os
 import yaml
-from numpy import asarray, rollaxis, count_nonzero
+from numpy import asarray, rollaxis, count_nonzero, sum
 
 from pyclowder.utils import CheckMessage
 from pyclowder.datasets import get_info
@@ -15,8 +15,6 @@
 from terrautils.metadata import get_extractor_metadata, get_terraref_metadata
 from terrautils.spatial import geojson_to_tuples_betydb, clip_raster, centroid_from_geojson
 
-import terraref.stereo_rgb
-
 
 # TODO: Keep these in terrautils.bety instead
 def get_traits_table():
@@ -55,15 +53,23 @@ def calculate_canopycover_masked(pxarray):
     """Return greenness percentage of given numpy array of pixels.
 
     Arguments:
-      pxarray (numpy array): rgb image
+      pxarray (numpy array): rgba image where alpha 255=data and alpha 0=NoData
 
     Returns:
       (float): greenness percentage
     """
 
-    # For masked images, all nonzero pixels are considered canopy
-    nz = count_nonzero(pxarray)
-    ratio = nz/float(pxarray.size)
+    # If > 75% is NoData, return a -1 ccvalue for omission later
+    total_size = pxarray.shape[0] * pxarray.shape[1]
+    nodata = count_nonzero(pxarray[:, :, 3]==0)
+    nodata_ratio = nodata/float(total_size)
+    if nodata_ratio > 0.75:
+        return -1
+
+    # For masked images, all pixels with rgb>0,0,0 are considered canopy
+    data = pxarray[pxarray[:, :, 3]==255]
+    canopy = len(data[sum(data[:, 0:3], 1)>0])
+    ratio = canopy/float(total_size - nodata)
     # Scale ratio from 0-1 to 0-100
     ratio *= 100.0
 
@@ -92,7 +98,7 @@ def check_message(self, connector, host, secret_key, resource, parameters):
         if resource['name'].startswith('rgb_fullfield') > -1 and resource['name'].endswith('_mask.tif'):
             # Check metadata to verify we have what we need
             md = download_metadata(connector, host, secret_key, resource['id'])
-            if get_extractor_metadata(md, self.extractor_info['name']) and not self.overwrite:
+            if get_extractor_metadata(md, self.extractor_info['name'], self.extractor_info['version']) and not self.overwrite:
                 self.log_skip(resource,"metadata indicates it was already processed")
                 return CheckMessage.ignore
             return CheckMessage.download
@@ -147,15 +153,16 @@ def process_message(self, connector, host, secret_key, resource, parameters):
 
                     ccVal = calculate_canopycover_masked(rollaxis(pxarray,0,3))
 
-                    # Prepare and submit datapoint
-                    geo_file.write(','.join([plotname,
-                                             'Canopy Cover',
-                                             str(centroid_lonlat[1]),
-                                             str(centroid_lonlat[0]),
-                                             time_fmt,
-                                             host + ("" if host.endswith("/") else "/") + "files/" + resource['id'],
-                                             str(ccVal),
-                                             timestamp]) + '\n')
+                    if (ccVal > -1):
+                        # Prepare and submit datapoint
+                        geo_file.write(','.join([plotname,
+                                                 'Canopy Cover',
+                                                 str(centroid_lonlat[1]),
+                                                 str(centroid_lonlat[0]),
+                                                 time_fmt,
+                                                 host + ("" if host.endswith("/") else "/") + "files/" + resource['id'],
+                                                 str(ccVal),
+                                                 timestamp]) + '\n')
 
                     successful_plots += 1
                     if successful_plots % 10 == 0:
@@ -166,11 +173,12 @@ def process_message(self, connector, host, secret_key, resource, parameters):
                 self.log_error(resource, "error generating cc for %s" % plotname)
                 continue
 
-            traits['canopy_cover'] = str(ccVal)
-            traits['site'] = plotname
-            traits['local_datetime'] = timestamp+"T12:00:00"
-            trait_list = generate_traits_list(traits)
-            csv_file.write(','.join(map(str, trait_list)) + '\n')
+            if (ccVal > -1):
+                traits['canopy_cover'] = str(ccVal)
+                traits['site'] = plotname
+                traits['local_datetime'] = timestamp+"T12:00:00"
+                trait_list = generate_traits_list(traits)
+                csv_file.write(','.join(map(str, trait_list)) + '\n')
 
         csv_file.close()
         geo_file.close()

fieldmosaic/full_day_to_tiles.py

@@ -179,13 +179,16 @@ def createVrt(base_dir,tif_file_list):
     except Exception as ex:
         fail("\tFailed to create virtual tif: " + str(ex))
 
-def createVrtPermanent(base_dir, tif_file_list, out_vrt='virtualTif.vrt'):
+def createVrtPermanent(base_dir, tif_file_list, out_vrt='virtualTif.vrt', alpha=False):
     # Create virtual tif for the files in this folder
     # Build a virtual TIF that combines all of the tifs that we just created
     print "\tCreating virtual TIF..."
     try:
         vrtPath = path.join(base_dir, out_vrt)
-        cmd = 'gdalbuildvrt -srcnodata "-99 -99 -99" -overwrite -input_file_list ' + tif_file_list +' ' + vrtPath
+        if alpha:
+            cmd = 'gdalbuildvrt -addalpha -srcnodata "-99 -99 -99" -overwrite -input_file_list ' + tif_file_list +' ' + vrtPath
+        else:
+            cmd = 'gdalbuildvrt -srcnodata "-99 -99 -99" -overwrite -input_file_list ' + tif_file_list +' ' + vrtPath
         print(cmd)
         system(cmd)
     except Exception as ex:

fieldmosaic/terra_fieldmosaic.py

@@ -92,7 +92,7 @@ def process_message(self, connector, host, secret_key, resource, parameters):
         # If outputs already exist, we don't need to do anything else
         found_all = True
         if self.thumb:
-            output_files = [out_tif_thumb]
+            output_files = [out_vrt, out_tif_thumb]
         else:
             output_files = [out_tif_full, out_tif_medium, out_png]
         for output_file in output_files:
@@ -155,9 +155,9 @@ def process_message(self, connector, host, secret_key, resource, parameters):
             generated_files = [out_tif_medium, out_tif_full, out_png]
         for checked_file in generated_files:
             if os.path.isfile(checked_file):
-                found_in_dest = check_file_in_dataset(connector, host, secret_key, target_dsid, checked_file, remove=self.overwrite,
-                                                      replacements=[("ir_fullfield", "fullfield"), ("L2", "L1")])
-                if not found_in_dest or self.overwrite:
+                found_in_dest = check_file_in_dataset(connector, host, secret_key, target_dsid, checked_file)
+                                                      #, replacements=[("ir_fullfield", "fullfield"), ("L2", "L1")])
+                if not found_in_dest:
                     id = upload_to_dataset(connector, host, self.clowder_user, self.clowder_pass, target_dsid, checked_file)
                     meta = build_metadata(host, self.extractor_info, id, content, 'file')
                     upload_metadata(connector, host, secret_key, id, meta)
@@ -198,11 +198,13 @@ def generateSingleMosaic(self, connector, host, secret_key, out_dir,
                 filepath = self.remapMountPath(connector, tpath)
                 tifftxt.write("%s\n" % filepath)
 
-        if (self.thumb and ((not file_exists(out_vrt)) or self.overwrite)) or (
-                    not self.thumb and (not file_exists(out_vrt))):
+        if (self.thumb and ((not file_exists(out_vrt)) or self.overwrite)) or (not self.thumb and (not file_exists(out_vrt))):
             # Create VRT from every GeoTIFF
             self.log_info(resource, "Creating VRT %s..." % out_vrt)
-            full_day_to_tiles.createVrtPermanent(out_dir, tiflist, out_vrt)
+            if out_vrt.endswith("_mask.vrt"):
+                full_day_to_tiles.createVrtPermanent(out_dir, tiflist, out_vrt, alpha=True)
+            else:
+                full_day_to_tiles.createVrtPermanent(out_dir, tiflist, out_vrt)
             os.remove(tiflist)
             created += 1
             bytes += os.path.getsize(out_vrt)

rgbmask/Dockerfile

@@ -7,7 +7,9 @@ RUN useradd -u 49044 extractor
 RUN pip install terraref-stereo-rgb \
                 opencv-python \
                 Pillow \
-                scikit-image
+                dask \
+                'networkx==2.2' \
+                'scikit-image<0.15'
 RUN apt update && apt install -y libsm6 \
                                 libxext6 \
                                 libxrender-dev \

rgbmask/extractor_info.json

@@ -1,7 +1,7 @@
 {
   "@context": "http://clowder.ncsa.illinois.edu/contexts/extractors.jsonld",
   "name": "terra.stereo-rgb.rgbmask",
-  "version": "1.0",
+  "version": "1.1",
   "description": "Stereo RGB Image Enhancement & Masking",
   "author": "Max Burnette <[email protected]>",
   "contributors": ["Zongyang Li", "Todd Nicholson"],