Changeset 927

Timestamp:

02/26/12 23:30:01 (15 months ago)

Author:

jjr8

Message:

Implemented the rest of #521: Tools that generate rasters should also (optionally) generate a raster catalog

Location:

MGET/Branches/Jason/PythonPackage/src/GeoEco

Files:

• DataProducts/Aviso.py (modified) (2 diffs)
• DataProducts/HYCOM.py (modified) (5 diffs)
• DataProducts/NASA/OceanColor.py (modified) (2 diffs)
• DataProducts/NASA/PODAAC.py (modified) (4 diffs)
• DataProducts/NOAA/NODC.py (modified) (3 diffs)
• DataProducts/NOAA/OSCAR.py (modified) (2 diffs)
• DataProducts/UMaineOMG.py (modified) (6 diffs)
• Datasets/ArcGIS.py (modified) (6 diffs)
• Datasets/Collections.py (modified) (2 diffs)
• Datasets/Virtual.py (modified) (1 diff)
• Logging.py (modified) (2 diffs)

MGET/Branches/Jason/PythonPackage/src/GeoEco/DataProducts/Aviso.py

@@ -280 +280 @@
 
     @classmethod
-    def _GetOtherTimeCoordinates(cls, queryableAttributeValues):
+    def _GetTimeCoordsFromQueryableAttributeValues(cls, queryableAttributeValues):
         if u'DateTime' not in queryableAttributeValues or not isinstance(queryableAttributeValues[u'DateTime'], datetime.datetime) or u'TemporalResolution' not in queryableAttributeValues or not isinstance(queryableAttributeValues[u'TemporalResolution'], basestring) or queryableAttributeValues[u'TemporalResolution'].lower() not in [u'daily', u'weekly']:
-            return [None, None]
+            return [None, None, None]
         dateTime = queryableAttributeValues[u'DateTime']
         if queryableAttributeValues[u'TemporalResolution'].lower() == u'daily':
-            return [dateTime - datetime.timedelta(0.5), dateTime + datetime.timedelta(0.5)]
-        return [dateTime - datetime.timedelta(3.5), dateTime + datetime.timedelta(3.5)]
+            return [dateTime - datetime.timedelta(0.5), dateTime, dateTime + datetime.timedelta(0.5) - datetime.timedelta(seconds=1)]
+        return [dateTime - datetime.timedelta(3.5), dateTime, dateTime + datetime.timedelta(3.5) - datetime.timedelta(seconds=1)]
 
     @classmethod
@@ -297 +297 @@
             workspace.ImportDatasets(GridSliceCollection(grid, tQACoordType=u'center').QueryDatasets(), mode, calculateStatistics=calculateStatistics, buildPyramids=buildPyramids)
             if rasterCatalog is not None:
-                workspace.ToRasterCatalog(rasterCatalog, grid.GetSpatialReference(u'ArcGIS'), tQACoordType=u'center', tCoordFunction=cls._GetOtherTimeCoordinates, overwriteExisting=True)
+                workspace.ToRasterCatalog(rasterCatalog, grid.GetSpatialReference(u'ArcGIS'), tQACoordType=u'center', tCoordFunction=cls._GetTimeCoordsFromQueryableAttributeValues, overwriteExisting=True)
         finally:
             grid.Close()

MGET/Branches/Jason/PythonPackage/src/GeoEco/DataProducts/HYCOM.py

@@ -268 +268 @@
             workspace.ImportDatasets(GridSliceCollection(grid, tQACoordType=u'center').QueryDatasets(), mode, calculateStatistics=calculateStatistics, buildPyramids=buildPyramids)
             if rasterCatalog is not None:
-                workspace.ToRasterCatalog(rasterCatalog, grid.GetSpatialReference(u'ArcGIS'), tQACoordType=u'center', tCoordFunction=lambda qav: [qav[u'DateTime'] - datetime.timedelta(0.5), qav[u'DateTime'] + datetime.timedelta(0.5)], overwriteExisting=True)
+                workspace.ToRasterCatalog(rasterCatalog, grid.GetSpatialReference(u'ArcGIS'), tQACoordType=u'center', tCoordFunction=lambda qav: [qav[u'DateTime'] - datetime.timedelta(0.5), qav[u'DateTime'], qav[u'DateTime'] + datetime.timedelta(0.5) - datetime.timedelta(seconds=1)], overwriteExisting=True)
         finally:
             grid.Close()
@@ -662 +662 @@
             workspace.ImportDatasets(grids, mode, calculateStatistics=calculateStatistics, buildPyramids=buildPyramids)
             if rasterCatalog is not None:
-                workspace.ToRasterCatalog(rasterCatalog, grid.GetSpatialReference(u'ArcGIS'), tQACoordType=u'center', tCoordFunction=lambda qav: [qav[u'DateTime'] - datetime.timedelta(0.5), qav[u'DateTime'] + datetime.timedelta(0.5)], overwriteExisting=True)
+                workspace.ToRasterCatalog(rasterCatalog, grid.GetSpatialReference(u'ArcGIS'), tQACoordType=u'center', tCoordFunction=lambda qav: [qav[u'DateTime'] - datetime.timedelta(0.5), qav[u'DateTime'], qav[u'DateTime'] + datetime.timedelta(0.5) - datetime.timedelta(seconds=1)], overwriteExisting=True)
             
         finally:
@@ -1307 +1307 @@
                 if j < len(gridTCoords) and gridTCoords[j] == tCoords[i]:
                     if datasetIs4D:
-                        for k in range(sliceList[1].start, sliceList[1].stop):
+                        for k in range(0, sliceList[1].stop - sliceList[1].start):
                             result[i, k, ySliceToWrite, xSliceToWrite] = numpy.take(data[i, k], indicesToTake)
                     else:
@@ -1398 +1398 @@
             workspace.ImportDatasets(GridSliceCollection(grid, tQACoordType=u'center').QueryDatasets(), mode, calculateStatistics=calculateStatistics, buildPyramids=buildPyramids)
             if rasterCatalog is not None:
-                workspace.ToRasterCatalog(rasterCatalog, grid.GetSpatialReference(u'ArcGIS'), tQACoordType=u'center', tCoordFunction=lambda qav: [qav[u'DateTime'] - datetime.timedelta(0.5), qav[u'DateTime'] + datetime.timedelta(0.5)], overwriteExisting=True)
+                workspace.ToRasterCatalog(rasterCatalog, grid.GetSpatialReference(u'ArcGIS'), tQACoordType=u'center', tCoordFunction=lambda qav: [qav[u'DateTime'] - datetime.timedelta(0.5), qav[u'DateTime'], qav[u'DateTime'] + datetime.timedelta(0.5) - datetime.timedelta(seconds=1)], overwriteExisting=True)
         finally:
             grid.Close()
@@ -1694 +1694 @@
             workspace.ImportDatasets(grids, mode, calculateStatistics=calculateStatistics, buildPyramids=buildPyramids)
             if rasterCatalog is not None:
-                workspace.ToRasterCatalog(rasterCatalog, grid.GetSpatialReference(u'ArcGIS'), tQACoordType=u'center', tCoordFunction=lambda qav: [qav[u'DateTime'] - datetime.timedelta(0.5), qav[u'DateTime'] + datetime.timedelta(0.5)], overwriteExisting=True)
+                workspace.ToRasterCatalog(rasterCatalog, grids[0].GetSpatialReference(u'ArcGIS'), tQACoordType=u'center', tCoordFunction=lambda qav: [qav[u'DateTime'] - datetime.timedelta(0.5), qav[u'DateTime'], qav[u'DateTime'] + datetime.timedelta(0.5) - datetime.timedelta(seconds=1)], overwriteExisting=True)
             
         finally:

MGET/Branches/Jason/PythonPackage/src/GeoEco/DataProducts/NASA/OceanColor.py

@@ -363 +363 @@
             return None
 
-        queryableAttributeValues['TemporalResolutionCode'] = components[0].upper()
+        queryableAttributeValues['TemporalResolutionCode'] = components[1].upper()
 
         if components[-1].lower() not in ['4', '9', '4km', '9km']:
@@ -506 +506 @@
         return self._DisplayName
 
-    def _GetOtherTimeCoordinates(self, queryableAttributeValues):
-        if u'DateTime' not in queryableAttributeValues or not isinstance(queryableAttributeValues[u'DateTime'], datetime.datetime):
-            return [None, None]
+    @classmethod
+    def _GetTimeCoordsFromQueryableAttributeValues(cls, queryableAttributeValues):
         try:
-            i = self.MinCoords['t', :].index(queryableAttributeValues[u'DateTime'])
+            if u'DateTime' in queryableAttributeValues and isinstance(queryableAttributeValues[u'DateTime'], datetime.datetime):
+                startDate = queryableAttributeValues[u'DateTime']
+                temporalResolution = None
+                
+                if u'TemporalResolution' in queryableAttributeValues and isinstance(queryableAttributeValues[u'TemporalResolution'], basestring) and queryableAttributeValues[u'TemporalResolution'] in [u'daily', '8day', 'monthly', 'annual']:
+                    temporalResolution = queryableAttributeValues[u'TemporalResolution']
+                elif u'TemporalResolutionCode' in queryableAttributeValues and isinstance(queryableAttributeValues[u'TemporalResolutionCode'], basestring) and queryableAttributeValues[u'TemporalResolutionCode'] in [u'DAY', '8D', 'MO', 'YR']:
+                    temporalResolution = queryableAttributeValues[u'TemporalResolutionCode']
+                else:
+                    return [startDate, None, None]
+                    
+                if temporalResolution in [u'daily', u'DAY']:
+                    return [startDate, startDate + datetime.timedelta(0.5), startDate + datetime.timedelta(1) - datetime.timedelta(seconds=1)]
+                
+                if temporalResolution in [u'8day', u'8D']:
+                    if startDate.month == 12 and startDate.day >= 24:
+                        endDate = datetime.datetime(startDate.year + 1, 1, 1)
+                        return [startDate, startDate + (endDate - startDate) / 2, endDate - datetime.timedelta(seconds=1)]
+                    else:
+                        return [startDate, startDate + datetime.timedelta(4), startDate + datetime.timedelta(8) - datetime.timedelta(seconds=1)]
+                    
+                if temporalResolution in [u'monthly', u'MO']:
+                    if startDate.month == 12:
+                        endDate = datetime.datetime(startDate.year + 1, 1, 1)
+                    else:
+                        endDate = datetime.datetime(startDate.year, startDate.month + 1, 1)
+                    return [startDate, startDate + (endDate - startDate) / 2, endDate - datetime.timedelta(seconds=1)]
+
+                if temporalResolution in [u'annual', u'YR']:
+                    return [startDate, datetime.datetime(startDate.year, 7, 1), datetime.datetime(startDate.year + 1, 1, 1) - datetime.timedelta(seconds=1)]
+
+            return [None, None, None]
         except:
-            return [None, None]
-        return [self.CenterCoords['t', i], self.MaxCoords['t', i]]
+            return [None, None, None]
 
     @classmethod
     def CreateArcGISRasters(cls, sensor, temporalResolution, spatialResolution, product,
-                            outputWorkspace, mode=u'add', rasterNameExpressions=[u'%(Sensor)s', u'%(TemporalResolution)s', u'%(SpatialResolution)s', u'%(ProductCode)s', u'%%Y', u'%(SensorCode)s%%Y%%j%(EndDate)s.L3m_%(TemporalResolutionCode)s_%(ProductCode)s_%(SpatialResolution)s.img'], rasterCatalog=None, cacheDirectory=None,
+                            outputWorkspace, mode=u'add', rasterNameExpressions=[u'%(Sensor)s', u'%(TemporalResolution)s', u'%(SpatialResolution)s', u'%(ProductCode)s', u'%%Y', u'%(SensorCode).1s%%Y%%j%(EndDate)s.L3m_%(TemporalResolutionCode).3s_%(ProductCode)s_%(SpatialResolution)s.img'], rasterCatalog=None, cacheDirectory=None,
                             rotationOffset=None, spatialExtent=None, startDate=None, endDate=None, 
                             timeout=60, maxRetryTime=300,
                             calculateStatistics=True, buildPyramids=False):
         cls.__doc__.Obj.ValidateMethodInvocation()
-        fullGrid = cls(sensor, temporalResolution, spatialResolution, product, timeout, maxRetryTime, cacheDirectory)
+        grid = cls(sensor, temporalResolution, spatialResolution, product, timeout, maxRetryTime, cacheDirectory)
         try:
             from GeoEco.Datasets.ArcGIS import ArcGISWorkspace, ArcGISRaster
-            grid = cls._RotateAndClip(fullGrid, rotationOffset, spatialExtent, startDate, endDate)
+            grid = cls._RotateAndClip(grid, rotationOffset, spatialExtent, startDate, endDate)
             workspace = ArcGISWorkspace(outputWorkspace, ArcGISRaster, pathCreationExpressions=rasterNameExpressions, cacheTree=True, queryableAttributes=tuple(grid.GetAllQueryableAttributes() + [QueryableAttribute(u'DateTime', _(u'Date'), DateTimeTypeMetadata())]))
             workspace.ImportDatasets(GridSliceCollection(grid).QueryDatasets(), mode, calculateStatistics=calculateStatistics, buildPyramids=buildPyramids)
             if rasterCatalog is not None:
-                workspace.ToRasterCatalog(rasterCatalog, grid.GetSpatialReference(u'ArcGIS'), tQACoordType=u'min', tCoordFunction=fullGrid._GetOtherTimeCoordinates, overwriteExisting=True)
+                workspace.ToRasterCatalog(rasterCatalog, grid.GetSpatialReference(u'ArcGIS'), tQACoordType=u'min', tCoordFunction=cls._GetTimeCoordsFromQueryableAttributeValues, overwriteExisting=True)
         finally:
-            fullGrid.Close()
+            grid.Close()
         return outputWorkspace
 

MGET/Branches/Jason/PythonPackage/src/GeoEco/DataProducts/NASA/PODAAC.py

@@ -104 +104 @@
 
     def __getattribute__(self, name):
-        if name.startswith('__') or name in ['_WrappedGrid', '_GetOtherTimeCoordinates', '_GetQueryableAttributesForTimeSlices', '_GetLazyPropertyValuesForTimeSlices', 'CreateArcGISRasters', 'InterpolateAtArcGISPoints', 'CreateCayulaCornillonFrontsAsArcGISRasters']:
+        if name.startswith('__') or name in ['_WrappedGrid', '_GetTimeCoordsFromQueryableAttributeValues', '_GetQueryableAttributesForTimeSlices', '_GetLazyPropertyValuesForTimeSlices', 'CreateArcGISRasters', 'InterpolateAtArcGISPoints', 'CreateCayulaCornillonFrontsAsArcGISRasters']:
             return object.__getattribute__(self, name)
         return object.__getattribute__(object.__getattribute__(self, '_WrappedGrid'), name)
 
     def __setattr__(self, name, value):
-        if name.startswith('__') or name in ['_WrappedGrid', '_GetOtherTimeCoordinates', '_GetQueryableAttributesForTimeSlices', '_GetLazyPropertyValuesForTimeSlices', 'CreateArcGISRasters', 'InterpolateAtArcGISPoints', 'CreateCayulaCornillonFrontsAsArcGISRasters']:
+        if name.startswith('__') or name in ['_WrappedGrid', '_GetTimeCoordsFromQueryableAttributeValues', '_GetQueryableAttributesForTimeSlices', '_GetLazyPropertyValuesForTimeSlices', 'CreateArcGISRasters', 'InterpolateAtArcGISPoints', 'CreateCayulaCornillonFrontsAsArcGISRasters']:
             object.__setattr__(self, name, value)
         else:
             setattr(object.__getattribute__(self, '_WrappedGrid'), name, value)
-
-    def _GetOtherTimeCoordinates(self, queryableAttributeValues):
-        if u'DateTime' not in queryableAttributeValues or not isinstance(queryableAttributeValues[u'DateTime'], datetime.datetime):
-            return [None, None]
-        try:
-            i = self.MinCoords['t', :].index(queryableAttributeValues[u'DateTime'])
-        except:
-            return [None, None]
-        return [self.CenterCoords['t', i], self.MaxCoords['t', i]]
 
     @classmethod
@@ -205 +196 @@
 
     @classmethod
+    def _GetTimeCoordsFromQueryableAttributeValues(cls, queryableAttributeValues):
+        try:
+            if u'DateTime' in queryableAttributeValues and isinstance(queryableAttributeValues[u'DateTime'], datetime.datetime):
+                startDate = queryableAttributeValues[u'DateTime']
+                temporalResolution = None
+
+                if u'GeophysicalParameter' in queryableAttributeValues and isinstance(queryableAttributeValues[u'GeophysicalParameter'], basestring) and queryableAttributeValues[u'GeophysicalParameter'].lower() in [u'nsst', u'sst4']:
+                    tOffset = datetime.timedelta(-0.5)
+                else:
+                    tOffset = datetime.timedelta(0)
+                
+                if u'TemporalResolution' in queryableAttributeValues and isinstance(queryableAttributeValues[u'TemporalResolution'], basestring) and queryableAttributeValues[u'TemporalResolution'] in [u'daily', '8day', 'monthly', 'annual']:
+                    temporalResolution = queryableAttributeValues[u'TemporalResolution']
+                elif u'TemporalResolutionCode' in queryableAttributeValues and isinstance(queryableAttributeValues[u'TemporalResolutionCode'], basestring) and queryableAttributeValues[u'TemporalResolutionCode'] in [u'DAY', '8D', 'MO', 'YR']:
+                    temporalResolution = queryableAttributeValues[u'TemporalResolutionCode']
+                else:
+                    return [startDate + tOffset, None, None]
+                    
+                if temporalResolution in [u'daily', u'DAY']:
+                    return [startDate + tOffset, startDate + datetime.timedelta(0.5) + tOffset, startDate + datetime.timedelta(1) + tOffset - datetime.timedelta(seconds=1)]
+                
+                if temporalResolution in [u'8day', u'8D']:
+                    if startDate.month == 12 and startDate.day >= 24:
+                        endDate = datetime.datetime(startDate.year + 1, 1, 1)
+                        return [startDate + tOffset, startDate + (endDate - startDate) / 2 + tOffset, endDate + tOffset - datetime.timedelta(seconds=1)]
+                    else:
+                        return [startDate + tOffset, startDate + datetime.timedelta(4) + tOffset, startDate + datetime.timedelta(8) + tOffset - datetime.timedelta(seconds=1)]
+                    
+                if temporalResolution in [u'monthly', u'MO']:
+                    if startDate.month == 12:
+                        endDate = datetime.datetime(startDate.year + 1, 1, 1)
+                    else:
+                        endDate = datetime.datetime(startDate.year, startDate.month + 1, 1)
+                    return [startDate + tOffset, startDate + (endDate - startDate) / 2 + tOffset, endDate + tOffset - datetime.timedelta(seconds=1)]
+
+                if temporalResolution in [u'annual', u'YR']:
+                    return [startDate + tOffset, datetime.datetime(startDate.year, 7, 1) + tOffset, datetime.datetime(startDate.year + 1, 1, 1) + tOffset - datetime.timedelta(seconds=1)]
+
+            return [None, None, None]
+        except:
+            return [None, None, None]
+
+    @classmethod
     def CreateArcGISRasters(cls, satellite, temporalResolution, spatialResolution, geophysicalParameter, variableName,
-                            outputWorkspace, mode=u'add', rasterNameExpressions=[u'%(Satellite)s', u'%(Wavelength)s', u'%(SpatialResolution)s', u'%(TemporalResolution)s', u'%%Y', u'%(SatelliteCode)s%%Y%%j%(EndDate)s.L3m_%(TemporalResolutionCode)s_%(GeophysicalParameter)s_%(SpatialResolutionCode)s_%(VariableName)s.img'], rasterCatalog=None,
+                            outputWorkspace, mode=u'add', rasterNameExpressions=[u'%(Satellite)s', u'%(Wavelength)s', u'%(SpatialResolution)s', u'%(TemporalResolution)s', u'%%Y', u'%(SatelliteCode).1s%%Y%%j%(EndDate)s.L3m_%(TemporalResolutionCode).3s_%(GeophysicalParameter)s_%(SpatialResolutionCode).1s_%(VariableName)s.img'], rasterCatalog=None,
                             qualityLevel=0,
                             rotationOffset=None, spatialExtent=None, startDate=None, endDate=None, 
@@ -216 +250 @@
         # clip it as requested.
 
-        fullGrid = MODISL3SSTTimeSeries(satellite, temporalResolution, spatialResolution, geophysicalParameter, variableName, qualityLevel, timeout, maxRetryTime, cacheDirectory)
+        grid = MODISL3SSTTimeSeries(satellite, temporalResolution, spatialResolution, geophysicalParameter, variableName, qualityLevel, timeout, maxRetryTime, cacheDirectory)
         
         try:
-            grid = fullGrid
             if rotationOffset is not None:
                 grid = RotatedGlobalGrid(grid, rotationOffset, u'Map units')
@@ -245 +278 @@
 
             if rasterCatalog is not None:
-                workspace.ToRasterCatalog(rasterCatalog, grid.GetSpatialReference(u'ArcGIS'), tQACoordType=u'min', tCoordFunction=fullGrid._GetOtherTimeCoordinates, overwriteExisting=True)
+                workspace.ToRasterCatalog(rasterCatalog, grid.GetSpatialReference(u'ArcGIS'), tQACoordType=u'min', tCoordFunction=cls._GetTimeCoordsFromQueryableAttributeValues, overwriteExisting=True)
 
         finally:
-            fullGrid.Close()
+            grid.Close()
 
         # Return successfully.

MGET/Branches/Jason/PythonPackage/src/GeoEco/DataProducts/NOAA/NODC.py

@@ -575 +575 @@
 
     def __getattribute__(self, name):
-        if name.startswith('__') or name in ['_WrappedGrid', '_GetOtherTimeCoordinates', 'CreateArcGISRasters', 'CreateClimatologicalArcGISRasters', 'InterpolateAtArcGISPoints', 'CreateCayulaCornillonFrontsAsArcGISRasters']:
+        if name.startswith('__') or name in ['_WrappedGrid', '_GetTimeCoordsFromQueryableAttributeValues', 'CreateArcGISRasters', 'CreateClimatologicalArcGISRasters', 'InterpolateAtArcGISPoints', 'CreateCayulaCornillonFrontsAsArcGISRasters']:
             return object.__getattribute__(self, name)
         return object.__getattribute__(object.__getattribute__(self, '_WrappedGrid'), name)
 
     def __setattr__(self, name, value):
-        if name.startswith('__') or name in ['_WrappedGrid', '_GetOtherTimeCoordinates', 'CreateArcGISRasters', 'CreateClimatologicalArcGISRasters', 'InterpolateAtArcGISPoints', 'CreateCayulaCornillonFrontsAsArcGISRasters']:
+        if name.startswith('__') or name in ['_WrappedGrid', '_GetTimeCoordsFromQueryableAttributeValues', 'CreateArcGISRasters', 'CreateClimatologicalArcGISRasters', 'InterpolateAtArcGISPoints', 'CreateCayulaCornillonFrontsAsArcGISRasters']:
             object.__setattr__(self, name, value)
         else:
             setattr(object.__getattribute__(self, '_WrappedGrid'), name, value)
 
-    def _GetOtherTimeCoordinates(self, queryableAttributeValues):
-        if u'DateTime' not in queryableAttributeValues or not isinstance(queryableAttributeValues[u'DateTime'], datetime.datetime):
-            return [None, None]
+    @classmethod
+    def _GetTimeCoordsFromQueryableAttributeValues(cls, queryableAttributeValues):
         try:
-            i = self.MinCoords['t', :].index(queryableAttributeValues[u'DateTime'])
+            if u'DateTime' in queryableAttributeValues and isinstance(queryableAttributeValues[u'DateTime'], datetime.datetime) and u'TemporalResolution' in queryableAttributeValues and isinstance(queryableAttributeValues[u'TemporalResolution'], basestring) and queryableAttributeValues[u'TemporalResolution'] in [u'daily', '5day', '8day', 'monthly', 'yearly']:
+                startDate = queryableAttributeValues[u'DateTime']
+                temporalResolution = queryableAttributeValues[u'TemporalResolution']
+
+                if u'ObservationTime' in queryableAttributeValues and isinstance(queryableAttributeValues[u'ObservationTime'], basestring) and queryableAttributeValues[u'ObservationTime'].lower() == u'nighttime':
+                    tOffset = datetime.timedelta(-0.5)
+                else:
+                    tOffset = datetime.timedelta(0)
+                    
+                if temporalResolution == u'daily':
+                    return [startDate + tOffset, startDate + datetime.timedelta(0.5) + tOffset, startDate + datetime.timedelta(1) + tOffset - datetime.timedelta(seconds=1)]
+                
+                if temporalResolution == u'5day':
+                    if startDate.month == 12 and startDate.day >= 24:
+                        endDate = datetime.datetime(startDate.year + 1, 1, 1)
+                        return [startDate + tOffset, startDate + (endDate - startDate) / 2 + tOffset, endDate + tOffset - datetime.timedelta(seconds=1)]
+                    else:
+                        return [startDate + tOffset, startDate + datetime.timedelta(2.5) + tOffset, startDate + datetime.timedelta(5) + tOffset - datetime.timedelta(seconds=1)]
+                
+                if temporalResolution == u'8day':
+                    if startDate.month == 12 and startDate.day >= 24:
+                        endDate = datetime.datetime(startDate.year + 1, 1, 1)
+                        return [startDate + tOffset, startDate + (endDate - startDate) / 2 + tOffset, endDate + tOffset - datetime.timedelta(seconds=1)]
+                    else:
+                        return [startDate + tOffset, startDate + datetime.timedelta(4) + tOffset, startDate + datetime.timedelta(8) + tOffset - datetime.timedelta(seconds=1)]
+                    
+                if temporalResolution == u'monthly':
+                    if startDate.month == 12:
+                        endDate = datetime.datetime(startDate.year + 1, 1, 1)
+                    else:
+                        endDate = datetime.datetime(startDate.year, startDate.month + 1, 1)
+                    return [startDate + tOffset, startDate + (endDate - startDate) / 2 + tOffset, endDate + tOffset - datetime.timedelta(seconds=1)]
+
+                if temporalResolution == u'yearly':
+                    return [startDate + tOffset, datetime.datetime(startDate.year, 7, 1) + tOffset, datetime.datetime(startDate.year + 1, 1, 1) + tOffset - datetime.timedelta(seconds=1)]
+
+            return [None, None, None]
         except:
-            return [None, None]
-        return [self.CenterCoords['t', i], self.MaxCoords['t', i]]
+            return [None, None, None]
 
     @classmethod
@@ -606 +640 @@
         # and clip it as requested.
 
-        fullGrid = AVHRRPathfinderSSTTimeSeries(temporalResolution, observationTime, parameter, maskLand, minOverallQualityFlag, maskIfBrightnessTempTestFailed, maskIfCloudTestFailed, maskIfUniformityTest1Failed, maskIfUniformityTest2Failed, maskIfZenithAngleTest1Failed, maskIfZenithAngleTest2Failed, maskIfReferenceTestFailed, maskIfStraySunlightTestFailed, maskIfEdgeTestFailed, maskIfGlintTestFailed, maskIfSSTTestFailed, timeout, maxRetryTime, cacheDirectory)
+        grid = AVHRRPathfinderSSTTimeSeries(temporalResolution, observationTime, parameter, maskLand, minOverallQualityFlag, maskIfBrightnessTempTestFailed, maskIfCloudTestFailed, maskIfUniformityTest1Failed, maskIfUniformityTest2Failed, maskIfZenithAngleTest1Failed, maskIfZenithAngleTest2Failed, maskIfReferenceTestFailed, maskIfStraySunlightTestFailed, maskIfEdgeTestFailed, maskIfGlintTestFailed, maskIfSSTTestFailed, timeout, maxRetryTime, cacheDirectory)
         
         try:
-            grid = fullGrid
             if rotationOffset is not None:
                 grid = RotatedGlobalGrid(grid, rotationOffset, u'Map units')
@@ -635 +668 @@
 
             if rasterCatalog is not None:
-                workspace.ToRasterCatalog(rasterCatalog, grid.GetSpatialReference(u'ArcGIS'), tQACoordType=u'min', tCoordFunction=fullGrid._GetOtherTimeCoordinates, overwriteExisting=True)
+                workspace.ToRasterCatalog(rasterCatalog, grid.GetSpatialReference(u'ArcGIS'), tQACoordType=u'min', tCoordFunction=cls._GetTimeCoordsFromQueryableAttributeValues, overwriteExisting=True)
 
         finally:
-            fullGrid.Close()
+            grid.Close()
 
         # Return successfully.

MGET/Branches/Jason/PythonPackage/src/GeoEco/DataProducts/NOAA/OSCAR.py

@@ -211 +211 @@
         return data.reshape(data.shape[0], data.shape[2], data.shape[3]), noDataValue
 
-    def _GetOtherTimeCoordinates(self, queryableAttributeValues):
+    def _GetTimeCoordsFromQueryableAttributeValues(self, queryableAttributeValues):
         if u'DateTime' not in queryableAttributeValues or not isinstance(queryableAttributeValues[u'DateTime'], datetime.datetime):
-            return [None, None]
+            return [None, None, None]
+        import numpy
         try:
-            i = self.CenterCoords['t', :].index(queryableAttributeValues[u'DateTime'])
+            i = numpy.where(self.CenterCoords['t', :] == queryableAttributeValues[u'DateTime'])[0][0]
         except:
-            return [None, None]
-        return [self.MinCoords['t', i], self.MaxCoords['t', i]]
+            return [None, None, None]
+        return [self.MinCoords['t', i], queryableAttributeValues[u'DateTime'], self.MaxCoords['t', i] - datetime.timedelta(seconds=1)]
 
     @classmethod
@@ -322 +323 @@
             workspace.ImportDatasets(GridSliceCollection(grid, tQACoordType=u'center').QueryDatasets(), mode, calculateStatistics=calculateStatistics, buildPyramids=buildPyramids)
             if rasterCatalog is not None:
-                workspace.ToRasterCatalog(rasterCatalog, grid.GetSpatialReference(u'ArcGIS'), tQACoordType=u'center', tCoordFunction=self._GetOtherTimeCoordinates, overwriteExisting=True)
+                workspace.ToRasterCatalog(rasterCatalog, grid.GetSpatialReference(u'ArcGIS'), tQACoordType=u'center', tCoordFunction=fullGrid._GetTimeCoordsFromQueryableAttributeValues, overwriteExisting=True)
         finally:
             fullGrid.Close()

MGET/Branches/Jason/PythonPackage/src/GeoEco/DataProducts/UMaineOMG.py

@@ -310 +310 @@
         return mean, self.NoDataValue
 
-    def _GetOtherTimeCoordinates(self, queryableAttributeValues):
+    def _GetTimeCoordsFromQueryableAttributeValues(self, queryableAttributeValues):
         if u'DateTime' not in queryableAttributeValues or not isinstance(queryableAttributeValues[u'DateTime'], datetime.datetime):
-            return [None, None]
+            return [None, None, None]
+        import numpy
         try:
-            i = self.CenterCoords['t', :].index(queryableAttributeValues[u'DateTime'])
+            i = numpy.where(self.CenterCoords['t', :] == queryableAttributeValues[u'DateTime'])[0][0]
         except:
-            return [None, None]
-        return [self.MinCoords['t', i], self.MaxCoords['t', i]]
+            return [None, None, None]
+        return [self.MinCoords['t', i], queryableAttributeValues[u'DateTime'], self.MaxCoords['t', i] - datetime.timedelta(seconds=1)]
 
     @classmethod
@@ -380 +381 @@
             from GeoEco.Datasets.ArcGIS import ArcGISWorkspace, ArcGISRaster
             from GeoEco.Datasets.Virtual import GridSliceCollection
+            fullGrid = grid
             grid = cls._Clip(grid, spatialExtent, linearUnit, startDate=startDate, endDate=endDate)
             workspace = ArcGISWorkspace(outputWorkspace, ArcGISRaster, pathCreationExpressions=rasterNameExpressions, cacheTree=True, queryableAttributes=tuple(grid.GetAllQueryableAttributes() + [QueryableAttribute(u'DateTime', _(u'Date'), DateTimeTypeMetadata())]))
             workspace.ImportDatasets(GridSliceCollection(grid, tQACoordType=u'center').QueryDatasets(), mode, calculateStatistics=calculateStatistics, buildPyramids=buildPyramids)
             if rasterCatalog is not None:
-                workspace.ToRasterCatalog(rasterCatalog, grid.GetSpatialReference(u'ArcGIS'), tQACoordType=u'center', tCoordFunction=grid._GetOtherTimeCoordinates, overwriteExisting=True)
+                workspace.ToRasterCatalog(rasterCatalog, fullGrid.GetSpatialReference(u'ArcGIS'), tQACoordType=u'center', tCoordFunction=fullGrid._GetTimeCoordsFromQueryableAttributeValues, overwriteExisting=True)
         finally:
             grid.Close()
@@ -942 +944 @@
         return mean, self.NoDataValue
 
-    def _GetOtherTimeCoordinates(self, queryableAttributeValues):
+    def _GetTimeCoordsFromQueryableAttributeValues(self, queryableAttributeValues):
         if u'DateTime' not in queryableAttributeValues or not isinstance(queryableAttributeValues[u'DateTime'], datetime.datetime):
-            return [None, None]
+            return [None, None, None]
+        import numpy
         try:
-            i = self.CenterCoords['t', :].index(queryableAttributeValues[u'DateTime'])
+            i = numpy.where(self.CenterCoords['t', :] == queryableAttributeValues[u'DateTime'])[0][0]
         except:
-            return [None, None]
-        return [self.MinCoords['t', i], self.MaxCoords['t', i]]
+            return [None, None, None]
+        return [self.MinCoords['t', i], queryableAttributeValues[u'DateTime'], self.MaxCoords['t', i] - datetime.timedelta(seconds=1)]
 
     @classmethod
     def CreateArcGISRasters(cls, variableName, depths,
-                            outputWorkspace, mode=u'add', rasterNameExpressions=['%(VariableName)s', '%%Y', 'Depth_%(Depth)gm', '%(VariableName)s_%%Y%%j_%%H%%M_%(Depth)gm.img'], rasterCatalog=None,
+                            outputWorkspace, mode=u'add', rasterNameExpressions=['%(VariableName)s', '%%Y', 'Depth_%(Depth)04.0fm', '%(VariableName)s_%%Y%%j_%%H%%M_%(Depth)04.0fm.img'], rasterCatalog=None,
                             spatialExtent=None, linearUnit=u'Degrees', startDate=None, endDate=None,
                             url=u'http://viz.clusters.umaine.edu:8080/thredds/dodsC/pacific/1991-2008', timeout=60, maxRetryTime=120, cacheDirectory=None,
@@ -963 +966 @@
         grid = ROMSCoSiNE4D(variableName, url, timeout, maxRetryTime, cacheDirectory)
         try:
+            fullGrid = grid
             grid = ROMSCoSiNE3D._Clip(grid, spatialExtent, linearUnit, startDate=startDate, endDate=endDate)
 
@@ -995 +999 @@
 
                 if rasterCatalog is not None:
-                    workspace.ToRasterCatalog(rasterCatalog, grid.GetSpatialReference(u'ArcGIS'), tQACoordType=u'center', tCoordFunction=grid._GetOtherTimeCoordinates, overwriteExisting=True)
+                    workspace.ToRasterCatalog(rasterCatalog, fullGrid.GetSpatialReference(u'ArcGIS'), tQACoordType=u'center', tCoordFunction=fullGrid._GetTimeCoordsFromQueryableAttributeValues, overwriteExisting=True)
             
             finally:
@@ -1009 +1013 @@
     def CreateClimatologicalArcGISRasters(cls, variableName, depths,
                                           statistic, binType,
-                                          outputWorkspace, mode=u'add', rasterNameExpressions=[u'%(VariableName)s', u'%(ClimatologyBinType)s_Climatology', 'Depth_%(Depth)gm', u'%(VariableName)s_%(Depth)gm_%(ClimatologyBinName)s_%(Statistic)s.img'],
+                                          outputWorkspace, mode=u'add', rasterNameExpressions=[u'%(VariableName)s', u'%(ClimatologyBinType)s_Climatology', 'Depth_%(Depth)04.0fm', u'%(VariableName)s_%(Depth)04.0fm_%(ClimatologyBinName)s_%(Statistic)s.img'],
                                           binDuration=1, startDayOfYear=1,
                                           spatialExtent=None, linearUnit=u'Degrees', startDate=None, endDate=None,

MGET/Branches/Jason/PythonPackage/src/GeoEco/Datasets/ArcGIS.py

@@ -467 +467 @@
             for item in level:
                 qa = item[1]
-                if qa.Name not in qaToFieldName:
+                if qa.Name not in qaToFieldName and qa.Name.lower() not in ['startdate', 'centerdate', 'enddate']:
                     qaToFieldName[qa.Name] = qa.Name
                     
@@ -592 +592 @@
                     # Note that not all printf syntax is supported.
 
-                    elif re.match(r'%\([^\)]+\)[#0\- +]?[\d]*(?:\\\.[\d]+)?[hlL]?[diouxXeEfFgGcrs]', pce[i:]) is not None:
+                    elif re.match(r'%\([^\)]+\)[#0\- +]?[\d]*(?:\.[\d]+)?[hlL]?[diouxXeEfFgGcrs]', pce[i:]) is not None:
 
                         # Parse components of the formatter.
@@ -644 +644 @@
 
                         elif conversion in 'fF':
-                            ppe += r'(?P<' + qaName + '>[+\- ]*\d+\.\d*)'
+                            if precision is not None:
+                                if precision > 0:
+                                    ppe += r'(?P<' + qaName + '>[+\- ]*\d+\.\d{' + str(precision) + '})'
+                                else:
+                                    ppe += r'(?P<' + qaName + '>[+\- ]*\d+)'
+                            else:
+                                ppe += r'(?P<' + qaName + '>[+\- ]*\d+\.?\d*)'
 
                         elif conversion in 's':
-                            ppe += r'(?P<' + qaName + '>.+)'
+                            if precision is not None:       # For NASA OceanColor L3 data, we need the ability to specify the maximum string length
+                                ppe += r'(?P<' + qaName + '>.{0,' + str(precision) + '})'
+                            else:
+                                ppe += r'(?P<' + qaName + '>.*)'
 
                         else:
@@ -716 +725 @@
             # the queryable attribute dictionaries.
 
-            pathsAndQAsList = self.QueryDatasets(getQueryableAttributesOnly=True)
+            pathsAndQAVsList = self.QueryDatasets(getQueryableAttributesOnly=True)
             
-            pathToQAs = {}
-            for result in pathsAndQAsList:
-                pathToQAs[os.path.join(self.Path, *result[0]).lower()] = result[1]
+            pathToQAVs = {}
+            for result in pathsAndQAVsList:
+                pathToQAVs[os.path.join(self.Path, *result[0]).lower()] = result[1]
 
             # Open an update cursor on the raster catalog. For each
@@ -728 +737 @@
             self._LogInfo(_(u'Updating the fields of each row of the raster catalog.'))
 
-            otherFields = None
             cur = catalogTable.OpenUpdateCursor(rowCount=catalogTable.GetRowCount(), rowDescriptionSingular=_(u'row'), rowDescriptionPlural=_(u'rows'))
             try:
                 while cur.NextRow():
                     path = oidToPath[cur.GetOID()]
-                    if path not in pathToQAs:
+                    if path not in pathToQAVs:
                         continue
 
-                    qas = pathToQAs[path]
+                    qav = pathToQAVs[path]
                     for qaName, fieldName in qaToFieldName.items():
-                        cur.SetValue(fieldName, qas[qaName])
 
                         # If this is queryable attribute is named
                         # 'DateTime' and the caller provided a
-                        # function that can calculate the other two
-                        # dates given this one, calculate those dates
-                        # and set those fields.
+                        # function that can calculate the dates from
+                        # the queryable attribute values, calculate
+                        # the dates and set the fields.
 
                         if qaName.lower() == u'datetime' and tCoordFunction is not None:
-                            if otherFields is None:
-                                otherFields = [u'StartDate', u'CenterDate', u'EndDate']
-                                otherFields.remove(fieldName)
-
-                            values = tCoordFunction(qas)
-                            cur.SetValue(otherFields[0], values[0])
-                            cur.SetValue(otherFields[1], values[1])
+                            values = tCoordFunction(qav)
+                            cur.SetValue(u'StartDate', values[0])
+                            cur.SetValue(u'CenterDate', values[1])
+                            cur.SetValue(u'EndDate', values[2])
+                        else:
+                            cur.SetValue(fieldName, qav[qaName])
 
                     cur.UpdateRow()
@@ -1545 +1551 @@
 
         if progressReporter is not None:
-            progressReporter.ReportProgress()
+            progressReporter.ReportProgress(reinitializeArcGISProgressor=True)
 
     @classmethod

MGET/Branches/Jason/PythonPackage/src/GeoEco/Datasets/Collections.py

@@ -211 +211 @@
         # us, so we do not pass them through to _ConstructFoundObject.
 
+        closeDatasets = False
+        getQueryableAttributesOnly = False
+
         if options is not None:
             constructOptions = {}
@@ -225 +228 @@
                     raise RuntimeError(u'Programming error in this tool: DatasetCollectionTree._QueryRecursive(): queryType must be \'normal\' when getQueryableAttributesOnly is True. Please contact the author of this tool for assistance.')
         else:
-            closeDatasets = False
-            getQueryableAttributesOnly = False
             constructOptions = options
 

MGET/Branches/Jason/PythonPackage/src/GeoEco/Datasets/Virtual.py

@@ -1335 +1335 @@
                 
                 for i in range(len(mask.Dimensions)):
-                    gridCoords = numpy.cast['float32'](self.CenterCoords[mask.Dimensions[i]])
-                    maskCoords = numpy.cast['float32'](mask.CenterCoords[mask.Dimensions[i]])
+                    if mask.Dimensions[i] != 't':
+                        gridCoords = numpy.cast['float32'](self.CenterCoords[mask.Dimensions[i]])
+                        maskCoords = numpy.cast['float32'](mask.CenterCoords[mask.Dimensions[i]])
+                    else:
+                        gridCoords = self.CenterCoords[mask.Dimensions[i]]
+                        maskCoords = mask.CenterCoords[mask.Dimensions[i]]
 
                     if gridCoords[0] < maskCoords[0] or gridCoords[-1] > maskCoords[-1]:

MGET/Branches/Jason/PythonPackage/src/GeoEco/Logging.py

@@ -587 +587 @@
             self.Stop()
 
-    def ReportProgress(self, operationsCompleted=1):
+    def ReportProgress(self, operationsCompleted=1, reinitializeArcGISProgressor=False):
         assert self.HasStarted, u'This ProgressReporter has not been started'
         
@@ -646 +646 @@
                 self._ClockNextReportTime = clockNow + 60.0
 
-        if self._UseArcGISProgressor() and int(math.floor(float(self._OperationsCompleted) / float(self._TotalOperations) * 1000.)) > int(math.floor(float(self._OperationsCompleted - 1) / float(self._TotalOperations) * 1000.)):
+        if self._UseArcGISProgressor() and (reinitializeArcGISProgressor or int(math.floor(float(self._OperationsCompleted) / float(self._TotalOperations) * 1000.)) > int(math.floor(float(self._OperationsCompleted - 1) / float(self._TotalOperations) * 1000.))):
             from GeoEco.ArcGIS import GeoprocessorManager
             gp = GeoprocessorManager.GetWrappedGeoprocessor()
             try:
+                if reinitializeArcGISProgressor:
+                    gp.SetProgressor('step', self.ArcGISProgressorLabel, 0, 1000, 1)
                 gp.SetProgressorPosition(min(1000, int(math.floor(float(self._OperationsCompleted) / float(self._TotalOperations) * 1000.))))
             except: