Changeset 713

Timestamp:

01/20/11 18:11:24 (2 years ago)

Author:

jjr8

Message:

Incremented build number and fixed:
* #452: Batched versions of Convert ArcGIS Raster to Lines tool are missing the Background Value and Minimum Dangle Length parameters
* #453: AVHRR Pathfinder V5 SST tools fail with: the Python pydap.client.open_url function failed with: Exception: Unable to parse token
* #454: AVHRR Pathfinder V5 SST tools have yearly nighttime data for 2005 but not for 2006

Location:

MGET/Branches/Jason/PythonPackage/src/GeoEco

Files:

• AggregatedModules/pydap/parsers/das.py (modified) (1 diff)
• DataManagement/ArcGISRasters.py (modified) (1 diff)
• DataProducts/NOAA/NODC.py (modified) (4 diffs)
• Datasets/Virtual.py (modified) (1 diff)
• __init__.py (modified) (1 diff)

MGET/Branches/Jason/PythonPackage/src/GeoEco/AggregatedModules/pydap/parsers/das.py

@@ -104 +104 @@
                 value = self.consume(r'".*?[^\\]"|[^;,]+')
             else:
-                try:
-                    value = self.consume('"')
-                except:
-                    value = self.consume('[^; "]+')
+                if self.buffer[0] == '"':
+                    value = self.buffer[0]                          # It is a strong enclosed by double quotes. First consume the leading double-quote.
+                    self.buffer = self.buffer[1:]
+                    
+                    while not self.buffer.startswith('";\n'):       # Consume all characters leading up to a sequence of double-quote, semicolon, newline.
+                        value += self.buffer[0]
+                        self.buffer = self.buffer[1:]
+                        
+                    value += self.buffer[0]                         # Consume trailing double-quote.
+                    self.buffer = self.buffer[1:]
                 else:
-                    value += self.consume('[^"]*')
-                    value += self.consume('"')
+                    value = self.consume('[^;"]+')         # It is a strong not enclosed by double quotes. Consume it all up to a semicolon.
             
             if type_.lower() in ['string', 'url']:

MGET/Branches/Jason/PythonPackage/src/GeoEco/DataManagement/ArcGISRasters.py

@@ -3462 +3462 @@
     outputParamExpressionDescriptions=[_OutputFeatureClassParamExpressionDescription],
     outputParamDefaultExpressions=[u'os.path.join(outputWorkspace, inputRaster[len(workspaceToSearch)+1:])'],
-    constantParamNames=[u'simplify', u'field', u'projectedCoordinateSystem', u'geographicTransformation', u'resamplingTechnique', u'projectedCellSize', u'registrationPoint', u'clippingDataset', u'clippingRectangle', u'mapAlgebraExpression'],
+    constantParamNames=[u'backgroundValue', u'minDangleLength', u'simplify', u'field', u'projectedCoordinateSystem', u'geographicTransformation', u'resamplingTechnique', u'projectedCellSize', u'registrationPoint', u'clippingDataset', u'clippingRectangle', u'mapAlgebraExpression'],
     processListMethodName=u'ToLinesList',
     processListMethodShortDescription=_(u'Converts a list of ArcGIS rasters to lines that outline groups of adjacent raster cells having the same value.'),

MGET/Branches/Jason/PythonPackage/src/GeoEco/DataProducts/NOAA/NODC.py

@@ -152 +152 @@
         # Otherwise, get the oldest dataset from each catalog.
 
-        if expression is None:
-            version50InterimExpression = 'Year IN (' + ', '.join(self._Version50InterimYears) + ')'
-        else:
-            version50InterimExpression = '(' + expression + ') AND Year IN (' + ', '.join(self._Version50InterimYears) + ')'
+        version50InterimExpression = '(' + expression + ') AND Year IN (' + ', '.join(self._Version50InterimYears) + ') AND (TemporalResolution <> \'yearly\' OR Year <> 2005)'     # NODC says there is no yearly data available for 2005, but they nonethless have an interim NOAA-17 file available on their server. We want to ignore it.
 
         datasets = []
@@ -256 +253 @@
         # Otherwise, get the newest dataset from each catalog.
 
-        if expression is None:
-            version50InterimExpression = 'Year IN (' + ', '.join(self._Version50InterimYears) + ')'
-        else:
-            version50InterimExpression = '(' + expression + ') AND Year IN (' + ', '.join(self._Version50InterimYears) + ')'
+        version50InterimExpression = '(' + expression + ') AND Year IN (' + ', '.join(self._Version50InterimYears) + ') AND (TemporalResolution <> \'yearly\' OR Year <> 2005)'      # NODC says there is no yearly data available for 2005, but they nonethless have an interim NOAA-17 file available on their server. We want to ignore it.
 
         datasets = []
@@ -460 +454 @@
                        self._TemporalResolution == u'5day' and (version == u'5.0' and d.month == 1 and d.day <= 6 or version == u'5.1' and (d.month > 1 or d.month == 1 and d.day > 6)) or \
                        self._TemporalResolution == u'8day' and (version == u'5.0' and d.month == 1 and d.day == 1 or version == u'5.1' and (d.month > 1 or d.month == 1 and d.day > 1)) or\
-                       self._TemporalResolution == u'monthly' and version == u'5.1':
+                       self._TemporalResolution == u'monthly' and version == u'5.1':        # TODO: Check this logic for nighttime
                         
                         dataset.Close()
@@ -473 +467 @@
                 # NOAA-17 is in an opposite orbit unlike all the other
                 # data in the Pathfinder V5 time series."
-
-                elif d.year == 2005 and self._TemporalResolution == u'yearly':
-                    dataset.Close()
-                    continue
+                #
+                # Note that, according our definition, NOAA's yearly
+                # nighttime 2005 file actually begins on December 31,
+                # 2004 at 12:00:00. Therefore we have to check that
+                # d.year == 2004 if observationTime == u'nighttime'.
+
+                elif self._TemporalResolution == u'yearly':
+                    observationTime = dataset.GetQueryableAttributeValue(u'ObservationTime')
+                    if observationTime == u'daytime' and d.year == 2005 or observationTime == u'nighttime' and d.year == 2004:
+                        dataset.Close()
+                        continue
 
                 # If we got to here, the dataset is ok. Add it to the

MGET/Branches/Jason/PythonPackage/src/GeoEco/Datasets/Virtual.py

@@ -318 +318 @@
                 data[t] = self._CachedDatasets[i].UnscaledData.__getitem__(tuple(sliceList[1:]))
             elif self._CachedOldestGrid.UnscaledNoDataValue is not None:
-                self._LogWarning(_(u'There is no data in %(dn)s for the time slice %(ts)s.') % {u'dn': self._Collection.DisplayName, u'ts': tCoords[t].strftime('%Y-%m-%d %H:%M:%S')})
+                if tCornerCoordType == 'min':
+                    self._LogWarning(_(u'There is no data in %(dn)s for the time slice that starts on %(ts)s.') % {u'dn': self._Collection.DisplayName, u'ts': tCoords[t].strftime('%Y-%m-%d %H:%M:%S')})
+                elif tCornerCoordType == 'center':
+                    self._LogWarning(_(u'There is no data in %(dn)s for the time slice %(ts)s.') % {u'dn': self._Collection.DisplayName, u'ts': tCoords[t].strftime('%Y-%m-%d %H:%M:%S')})
+                else:
+                    self._LogWarning(_(u'There is no data in %(dn)s for the time slice that ends on %(ts)s.') % {u'dn': self._Collection.DisplayName, u'ts': tCoords[t].strftime('%Y-%m-%d %H:%M:%S')})
                 data[t] += self._CachedOldestGrid.UnscaledNoDataValue
             else:
-                self._LogWarning(_(u'There is no data in %(dn)s for the time slice %(ts)s but the datasets in this collection do not have a NoData value defined. The values of this time slice will be set to 0.') % {u'dn': self._Collection.DisplayName, u'ts': tCoords[t].strftime('%Y-%m-%d %H:%M:%S')})
+                if tCornerCoordType == 'min':
+                    self._LogWarning(_(u'There is no data in %(dn)s for the time slice that starts on %(ts)s but the datasets in this collection do not have a NoData value defined. The values of this time slice will be set to 0.') % {u'dn': self._Collection.DisplayName, u'ts': tCoords[t].strftime('%Y-%m-%d %H:%M:%S')})
+                elif tCornerCoordType == 'center':
+                    self._LogWarning(_(u'There is no data in %(dn)s for the time slice %(ts)s but the datasets in this collection do not have a NoData value defined. The values of this time slice will be set to 0.') % {u'dn': self._Collection.DisplayName, u'ts': tCoords[t].strftime('%Y-%m-%d %H:%M:%S')})
+                else:
+                    self._LogWarning(_(u'There is no data in %(dn)s for the time slice that ends on %(ts)s but the datasets in this collection do not have a NoData value defined. The values of this time slice will be set to 0.') % {u'dn': self._Collection.DisplayName, u'ts': tCoords[t].strftime('%Y-%m-%d %H:%M:%S')})
 
             t += 1

MGET/Branches/Jason/PythonPackage/src/GeoEco/__init__.py

@@ -1 @@
-__version__ = u'0.8a21'
+__version__ = u'0.8a22'