Changeset 1019

Timestamp:

08/02/12 12:49:20 (10 months ago)

Author:

jjr8

Message:

* Incremented build number
* Fixed #550: For binary classification models, Predict ... From Rasters tools should output continuous probabilities unless a cutoff is specified
* Fixed random other bugs

Location:

MGET/Branches/Jason/PythonPackage/src/GeoEco

Files:

• FisheryAnalysis/Spatiotemporal.py (modified) (3 diffs)
• SpatialAnalysis/Interpolation.py (modified) (2 diffs)
• Statistics/Modeling.py (modified) (2 diffs)
• Statistics/PredictModel.r (modified) (9 diffs)
• __init__.py (modified) (1 diff)

MGET/Branches/Jason/PythonPackage/src/GeoEco/FisheryAnalysis/Spatiotemporal.py

@@ -528 +528 @@
 
 The spectral power is a measure of the strength of the contribution of
-the wave's contribute to the overall pattern. Important waves appear
-as peaks in the periodogram. When you find a peak, the value of the x
-axis gives the period (in days) of the cycle. In marine ecological
-data, patterns are often observed in relation to solar, lunar, or
-tidal cycles. Solar cycles are indicated by peaks at 365 days (an
-annual cycle) and at 24 hours (a diurnal cycle). Lunar cycles are
-indicated by a peak at 29.5 days. Tidal cycles are more complicated to
-observe, and depend on tidal patterns in the region of interest.
+the wave to the overall pattern. Important waves appear as peaks in
+the periodogram. When you find a peak, the value of the x axis gives
+the period (in days) of the wave. In marine ecological data, patterns
+are often observed in relation to solar, lunar, or tidal cycles. Solar
+cycles are indicated by peaks at 365 days (an annual cycle) and at 24
+hours (a diurnal cycle). Lunar cycles are indicated by a peak at 29.5
+days. Tidal cycles are more complicated to observe, and depend on
+tidal patterns in the region of interest.
 
 To interpret the periodogram, look for high spikes at 365 days, 29.5
-days, 1 day, with very low values elsewhere. If you see high spikes at
-one or more of those locations, a solar or lunar cycle is indicated.
-But if those spikes are not very high relative to other periods--if
-the periodogram displays a generally jittery or noisy pattern--then
-solar and lunar cycles may not exist.
+days, or 1 day. If you see high spikes at one or more of those
+locations with very low values everywhere else, a solar or lunar cycle
+is indicated. But if those spikes are not very high relative to other
+periods--if the periodogram displays a generally jittery or noisy
+pattern--then solar and lunar cycles may not exist.
 
 In order for Fourier analysis to detect such patterns, the data must
@@ -879 +879 @@
 
 AddMethodMetadata(FisheryPeriodicity.AnalyzeCPUEInTable,
-    shortDescription=_(u'Creates several plots that analyze the temporal periodicity of catch per unit effort (FPUE) calculated from fishing records in a Table.'),
+    shortDescription=_(u'Creates several plots that analyze the temporal periodicity of catch per unit effort (CPUE) calculated from fishing records in a Table.'),
     longDescription=_(
 u"""The first plot is a bar plot showing the mean CPUE for a series of
@@ -976 +976 @@
 
 AddMethodMetadata(FisheryPeriodicity.AnalyzeCPUEInArcGISTable,
-    shortDescription=_(u'Creates several plots that analyze the temporal periodicity of catch per unit effort (FPUE) calculated from fishing records in a table.'),
+    shortDescription=_(u'Creates several plots that analyze the temporal periodicity of catch per unit effort (CPUE) calculated from fishing records in a table.'),
     longDescription=FisheryPeriodicity.AnalyzeCPUEInTable.__doc__.Obj.LongDescription,
     isExposedToPythonCallers=True,

MGET/Branches/Jason/PythonPackage/src/GeoEco/SpatialAnalysis/Interpolation.py

@@ -288 +288 @@
                     coordinateFields[1] = zField
                 elif table.GeometryType != u'Point25D' and zValue is None:
-                    raise ValueError(_(u'%(grid)s has a z dimension but %(table)s has %(gt)s geometry and a z-coordinate field was not specified. It must have Point25D geometry or you must specify a z-coordinate field or specify the value of the z coordinate.') % {u'g': grids[0].DisplayName, u'table': table.DisplayName, u'gt': table.GeometryType})
+                    raise ValueError(_(u'%(grid)s has a z dimension but %(table)s has %(gt)s geometry and a z-coordinate field was not specified. It must have Point25D geometry or you must specify a z-coordinate field or specify the value of the z coordinate.') % {u'grid': grids[0].DisplayName, u'table': table.DisplayName, u'gt': table.GeometryType})
 
             if grids[0].Dimensions[0] == u't':
@@ -294 +294 @@
                     coordinateFields[0] = tField
                 elif tValue is None:
-                    raise ValueError(_(u'%(grid)s has a t dimension but a t-coordinate field of %(table)s was not specified. You must specify a t-coordinate field or the value of the t coordinate.') % {u'g': grids[0].DisplayName, u'table': table.DisplayName})
+                    raise ValueError(_(u'%(grid)s has a t dimension but a t-coordinate field of %(table)s was not specified. You must specify a t-coordinate field or the value of the t coordinate.') % {u'grid': grids[0].DisplayName, u'table': table.DisplayName})
             else:
                 tField = None

MGET/Branches/Jason/PythonPackage/src/GeoEco/Statistics/Modeling.py

@@ -4251 +4251 @@
 CopyArgumentMetadata(GLM.PredictFromArcGISRasters, u'constantPredictorNames', TreeModel.PredictFromArcGISRasters, u'constantPredictorNames')
 CopyArgumentMetadata(GLM.PredictFromArcGISRasters, u'constantPredictorValues', TreeModel.PredictFromArcGISRasters, u'constantPredictorValues')
-
-AddArgumentMetadata(TreeModel.PredictFromArcGISRasters, u'cutoff',
-    typeMetadata=FloatTypeMetadata(minValue=0., maxValue=1., canBeNone=True),
-    description=_(
-u"""Cutoff to use when classifying the continuous probability output
-by a binary classification model into a binary result (0 or 1). This
-parameter should not be specified for regression models or
-classification models that have more than two classes.
-
-Probabilities greater than or equal to the cutoff will be classified
-as 1; probabilities less than the cutoff will be classified as 0. If a
-cutoff is not provided, the classification will be performed
-automatically by R."""),
-    arcGISDisplayName=_(u'Binary classification cutoff'))       # TODO: improve "automatically classified by R"
-
+CopyArgumentMetadata(GLM.PredictFromArcGISRasters, u'cutoff', TreeModel.PredictFromArcGISRasters, u'cutoff')
 CopyArgumentMetadata(GLM.PredictFromArcGISRasters, u'resamplingTechniques', TreeModel.PredictFromArcGISRasters, u'resamplingTechniques')
 CopyArgumentMetadata(GLM.PredictFromArcGISRasters, u'ignoreOutOfRangeValues', TreeModel.PredictFromArcGISRasters, u'ignoreOutOfRangeValues')
@@ -5047 +5033 @@
 CopyArgumentMetadata(GLM.PredictFromArcGISRasters, u'constantPredictorNames', RandomForestModel.PredictFromArcGISRasters, u'constantPredictorNames')
 CopyArgumentMetadata(GLM.PredictFromArcGISRasters, u'constantPredictorValues', RandomForestModel.PredictFromArcGISRasters, u'constantPredictorValues')
-
-AddArgumentMetadata(RandomForestModel.PredictFromArcGISRasters, u'cutoff',
-    typeMetadata=FloatTypeMetadata(minValue=0., maxValue=1., canBeNone=True),
-    description=_(
-u"""Cutoff to use when classifying the continuous probability output
-by a binary classification model into a binary result (0 or 1). This
-parameter should not be specified for regression models or
-classification models with more than two classes.
-
-Probabilities greater than or equal to the cutoff will be classified
-as 1; probabilities less than the cutoff will be classified as 0. If a
-cutoff is not provided, 0.5 will be implicitly used, by virtue of the
-default behavior of random forests, in which the prediction is decided
-by majority vote between all of the trees in the forest."""),
-    arcGISDisplayName=_(u'Binary classification cutoff'))
-
+CopyArgumentMetadata(GLM.PredictFromArcGISRasters, u'cutoff', RandomForestModel.PredictFromArcGISRasters, u'cutoff')
 CopyArgumentMetadata(GLM.PredictFromArcGISRasters, u'resamplingTechniques', RandomForestModel.PredictFromArcGISRasters, u'resamplingTechniques')
 CopyArgumentMetadata(GLM.PredictFromArcGISRasters, u'ignoreOutOfRangeValues', RandomForestModel.PredictFromArcGISRasters, u'ignoreOutOfRangeValues')

MGET/Branches/Jason/PythonPackage/src/GeoEco/Statistics/PredictModel.r

@@ -89 +89 @@
         maxFittedPredictorValues[[predictor]] <- max(c(trainingData[[predictor]]), na.rm=TRUE)
     }
+    
+    isBinaryClassification <- minResponseValue == 0 && maxResponseValue == 1 &&
+                              (!is.null(rPackage) && rPackage == "rpart" && model$method == "class" ||
+                               !is.null(rPackage) && rPackage == "randomForest" && model$type == "classification" ||
+                               !is.null(rPackage) && rPackage == "party" && (model@responses@is_nominal[1] || model@responses@is_ordinal[1]) ||
+                               (!is.null(rPackage) && rPackage %in% c("mgcv", "gam") || is.null(rPackage)) && model$family$family %in% c("binomial", "quasibinomial", "negbin"))
+    
+    isNonbinaryClassification <- !isBinaryClassification &&
+                                 (!is.null(rPackage) && rPackage == "rpart" && model$method == "class" ||
+                                 !is.null(rPackage) && rPackage == "randomForest" && model$type == "classification" ||
+                                 !is.null(rPackage) && rPackage == "party" && (model@responses@is_nominal[1] || model@responses@is_ordinal[1]))
 
     # Build a vector listing the categorical predictors.
@@ -124 +135 @@
     allValuesAreNA <- TRUE
     
-    return(list(responseVariable, allPredictors, minResponseValue, maxResponseValue, minFittedPredictorValues, maxFittedPredictorValues, categoricalPredictors, categoricalPredictorLevels, warnedAboutMissingLevels, allValuesForPredictorAreNA, allValuesAreNA))
+    return(list(responseVariable, allPredictors, minResponseValue, maxResponseValue, isBinaryClassification, isNonbinaryClassification, minFittedPredictorValues, maxFittedPredictorValues, categoricalPredictors, categoricalPredictorLevels, warnedAboutMissingLevels, allValuesForPredictorAreNA, allValuesAreNA))
 }
 
 
-PredictModel <- function(model, rPackage, cutoff, ignoreOutOfRangeValues, returnPredictedErrors, allPredictors, responseVariable, minResponseValue, maxResponseValue, minFittedPredictorValues, maxFittedPredictorValues, categoricalPredictors, categoricalPredictorLevels, warnedAboutMissingLevels, allValuesForPredictorAreNA, allValuesAreNA, predictorValues, forceBinaryProbabilities=FALSE)
+PredictModel <- function(model, rPackage, cutoff, ignoreOutOfRangeValues, returnPredictedErrors, allPredictors, responseVariable, minResponseValue, maxResponseValue, minFittedPredictorValues, maxFittedPredictorValues, categoricalPredictors, categoricalPredictorLevels, warnedAboutMissingLevels, allValuesForPredictorAreNA, allValuesAreNA, predictorValues)
 {
     # Determine which records have NA for one or more predictors.
@@ -193 +204 @@
 
         if (model$method == "class")
-            if (minResponseValue == 0 && maxResponseValue == 1 && (!is.null(cutoff) || forceBinaryProbabilities))
+            if (minResponseValue == 0 && maxResponseValue == 1)
             {
                 predictedResponse <- as.vector(suppressWarnings(predict(model, newdata=predictorValues, type="prob")[,2]))
@@ -221 +232 @@
         # the second class and apply the cutoff.
         
-        if (rPackage == "randomForest" && model$type == "classification" && minResponseValue == 0 && maxResponseValue == 1 && (!is.null(cutoff) || forceBinaryProbabilities))
+        if (rPackage == "randomForest" && model$type == "classification" && minResponseValue == 0 && maxResponseValue == 1)
         {
             predictedResponse <- as.vector(suppressWarnings(predict(model, newdata=predictorValues, type="prob")[,2]))
@@ -227 +238 @@
                 predictedResponse <- as.integer(predictedResponse >= cutoff)
         }
-        else if (rPackage == "party" && (model@responses@is_nominal[1] || model@responses@is_ordinal[1]) && minResponseValue == 0 && maxResponseValue == 1 && !is.null(cutoff))
-            predictedResponse <- as.integer(sapply(suppressWarnings(predict(model, newdata=predictorValues, type="prob")), "[", i = 2) >= cutoff)
+        else if (rPackage == "party" && (model@responses@is_nominal[1] || model@responses@is_ordinal[1]) && minResponseValue == 0 && maxResponseValue == 1)
+        {
+            predictedResponse <- sapply(suppressWarnings(predict(model, newdata=predictorValues, type="prob")), "[", i = 2)
+            if (!is.null(cutoff))
+                predictedResponse <- as.integer(predictedResponse >= cutoff)
+        }
 
         # For other models (regression forests or classification forests with
@@ -355 +370 @@
     minResponseValue <- variables[[3]]
     maxResponseValue <- variables[[4]]
-    minFittedPredictorValues <- variables[[5]]
-    maxFittedPredictorValues <- variables[[6]]
-    categoricalPredictors <- variables[[7]]
-    categoricalPredictorLevels <- variables[[8]]
-    warnedAboutMissingLevels <- variables[[9]]
-    allValuesForPredictorAreNA <- variables[[10]]
-    allValuesAreNA <- variables[[11]]
+    isBinaryClassification <- variables[[5]]
+    isNonbinaryClassification <- variables[[6]]
+    minFittedPredictorValues <- variables[[7]]
+    maxFittedPredictorValues <- variables[[8]]
+    categoricalPredictors <- variables[[9]]
+    categoricalPredictorLevels <- variables[[10]]
+    warnedAboutMissingLevels <- variables[[11]]
+    allValuesForPredictorAreNA <- variables[[12]]
+    allValuesAreNA <- variables[[13]]
     
     # Verify that the caller provided all of the necessary predictors.
@@ -398 +415 @@
     }, finally=close(rasterInfo))
 
-    # Determine the data type and NoData value of the output rasters. For
-    # regression models, use 32-bit floating point as the data type and
-    # -3.4028235e+038 as the NoData value (the 32-bit IEEE-754 floating-point
-    # negative number that is farthest from 0; traditionally used by ArcGIS for
-    # NoData). For classification models, use the most compact integer data type
-    # and corresponding NoData value.
-
-    if (is.null(cutoff) && (is.null(rPackage) || !(rPackage == "rpart" && model$method == "class" || rPackage == "randomForest" && model$type == "classification" || rPackage == "party" && (model@responses@is_nominal[1] || model@responses@is_ordinal[1]))))
-    {
+    # Determine the data type and NoData value of the output rasters.
+    # If a cutoff is provided (indicating that the output should be a
+    # binary classification) or the model is a classification with
+    # more than two classes, use the most compact integer data type
+    # and corresponding NoData value. Otherwise (for regression models
+    # or classification models with two classes that but no cutoff was
+    # provided), use 32-bit floating point as the data type and
+    # -3.4028235e+038 as the NoData value (the 32-bit IEEE-754
+    # floating-point negative number that is farthest from 0;
+    # traditionally used by ArcGIS for NoData).
+
+    if (!is.null(cutoff) || isNonbinaryClassification)
+    {
+        if (minResponseValue > -128 && maxResponseValue <= 127)
+        {
+            dataType <- "signedint"
+            nbytes <- 1
+            noDataValue <- as.integer(-128)
+        }
+        else if (minResponseValue >= -128 && maxResponseValue < 127)
+        {
+            dataType <- "signedint"
+            nbytes <- 1
+            noDataValue <- as.integer(127)
+        }
+        else if (minResponseValue > 0 && maxResponseValue <= 255)
+        {
+            dataType <- "unsignedint"
+            nbytes <- 1
+            noDataValue <- as.integer(0)
+        }
+        else if (minResponseValue >= 0 && maxResponseValue < 255)
+        {
+            dataType <- "unsignedint"
+            nbytes <- 1
+            noDataValue <- as.integer(255)
+        }
+        else if (minResponseValue > -32768 && maxResponseValue <= 32767)
+        {
+            dataType <- "signedint"
+            nbytes <- 2
+            noDataValue <- as.integer(-32768)
+        }
+        else if (minResponseValue >= -32768 && maxResponseValue < 32767)
+        {
+            dataType <- "signedint"
+            nbytes <- 2
+            noDataValue <- as.integer(32767)
+        }
+        else if (minResponseValue > 0 && maxResponseValue <= 65535)
+        {
+            dataType <- "unsignedint"
+            nbytes <- 2
+            noDataValue <- as.integer(0)
+        }
+        else if (minResponseValue >= 0 && maxResponseValue < 65535)
+        {
+            dataType <- "unsignedint"
+            nbytes <- 2
+            noDataValue <- as.integer(65535)
+        }
+        else if (minResponseValue > -2147483648 && maxResponseValue <= 2147483647)
+        {
+            dataType <- "signedint"
+            nbytes <- 4
+            noDataValue <- as.integer(-2147483648)
+        }
+        else if (minResponseValue >= -2147483648 && maxResponseValue < 2147483647)
+        {
+            dataType <- "signedint"
+            nbytes <- 4
+            noDataValue <- as.integer(2147483647)
+        }
+        else if (minResponseValue > 0)
+        {
+            dataType <- "unsignedint"
+            nbytes <- 4
+            noDataValue <- as.integer(0)
+        }
+        else
+        {
+            dataType <- "unsignedint"
+            nbytes <- 4
+            noDataValue <- as.integer(4294967296)
+        }
+    }
+    else
+    {
+        if (isBinaryClassification && is.null(cutoff))
+            warning("This appears to be a binary classification model but no cutoff value was provided. The output will be a continuous floating-point value ranging from 0 to 1. To obtain a binary integer value (0 or 1), please provide a cutoff and try again.", call.=FALSE)
+
         dataType <- "float"
         nbytes <- 4
         noDataValue <- -3.4028235e+038
-    }
-    else
-    {
-        if (minResponseValue > -128 && maxResponseValue <= 127)
-        {
-            dataType <- "signedint"
-            nbytes <- 1
-            noDataValue <- as.integer(-128)
-        }
-        else if (minResponseValue >= -128 && maxResponseValue < 127)
-        {
-            dataType <- "signedint"
-            nbytes <- 1
-            noDataValue <- as.integer(127)
-        }
-        else if (minResponseValue > 0 && maxResponseValue <= 255)
-        {
-            dataType <- "unsignedint"
-            nbytes <- 1
-            noDataValue <- as.integer(0)
-        }
-        else if (minResponseValue >= 0 && maxResponseValue < 255)
-        {
-            dataType <- "unsignedint"
-            nbytes <- 1
-            noDataValue <- as.integer(255)
-        }
-        else if (minResponseValue > -32768 && maxResponseValue <= 32767)
-        {
-            dataType <- "signedint"
-            nbytes <- 2
-            noDataValue <- as.integer(-32768)
-        }
-        else if (minResponseValue >= -32768 && maxResponseValue < 32767)
-        {
-            dataType <- "signedint"
-            nbytes <- 2
-            noDataValue <- as.integer(32767)
-        }
-        else if (minResponseValue > 0 && maxResponseValue <= 65535)
-        {
-            dataType <- "unsignedint"
-            nbytes <- 2
-            noDataValue <- as.integer(0)
-        }
-        else if (minResponseValue >= 0 && maxResponseValue < 65535)
-        {
-            dataType <- "unsignedint"
-            nbytes <- 2
-            noDataValue <- as.integer(65535)
-        }
-        else if (minResponseValue > -2147483648 && maxResponseValue <= 2147483647)
-        {
-            dataType <- "signedint"
-            nbytes <- 4
-            noDataValue <- as.integer(-2147483648)
-        }
-        else if (minResponseValue >= -2147483648 && maxResponseValue < 2147483647)
-        {
-            dataType <- "signedint"
-            nbytes <- 4
-            noDataValue <- as.integer(2147483647)
-        }
-        else if (minResponseValue > 0)
-        {
-            dataType <- "unsignedint"
-            nbytes <- 4
-            noDataValue <- as.integer(0)
-        }
-        else
-        {
-            dataType <- "unsignedint"
-            nbytes <- 4
-            noDataValue <- as.integer(4294967296)
-        }
     }
 
@@ -629 +653 @@
     minResponseValue <- variables[[3]]
     maxResponseValue <- variables[[4]]
-    minFittedPredictorValues <- variables[[5]]
-    maxFittedPredictorValues <- variables[[6]]
-    categoricalPredictors <- variables[[7]]
-    categoricalPredictorLevels <- variables[[8]]
-    warnedAboutMissingLevels <- variables[[9]]
-    allValuesForPredictorAreNA <- variables[[10]]
-    allValuesAreNA <- variables[[11]]
-    
-    isBinaryClassification <- minResponseValue == 0 && maxResponseValue == 1 &&
-                              (!is.null(rPackage) && rPackage == "rpart" && model$method == "class" ||
-                               !is.null(rPackage) && rPackage == "randomForest" && model$type == "classification" ||
-                               !is.null(rPackage) && rPackage == "party" && (model@responses@is_nominal[1] || model@responses@is_ordinal[1]) ||
-                               (!is.null(rPackage) && rPackage %in% c("mgcv", "gam") || is.null(rPackage)) && model$family$family %in% c("binomial", "quasibinomial", "negbin"))
-    
-    isNonbinaryClassification <- !isBinaryClassification &&
-                              (!is.null(rPackage) && rPackage == "rpart" && model$method == "class" ||
-                               !is.null(rPackage) && rPackage == "randomForest" && model$type == "classification" ||
-                               !is.null(rPackage) && rPackage == "party" && (model@responses@is_nominal[1] || model@responses@is_ordinal[1]))
+    isBinaryClassification <- variables[[5]]
+    isNonbinaryClassification <- variables[[6]]
+    minFittedPredictorValues <- variables[[7]]
+    maxFittedPredictorValues <- variables[[8]]
+    categoricalPredictors <- variables[[9]]
+    categoricalPredictorLevels <- variables[[10]]
+    warnedAboutMissingLevels <- variables[[11]]
+    allValuesForPredictorAreNA <- variables[[12]]
+    allValuesAreNA <- variables[[13]]
     
     # Verify that the caller provided all of the necessary predictors.
@@ -661 +676 @@
     # Do the prediction. For binary models, force the probability to be returned.
 
-    predictedResponse <- PredictModel(model, rPackage, NULL, ignoreOutOfRangeValues, FALSE, allPredictors, responseVariable, minResponseValue, maxResponseValue, minFittedPredictorValues, maxFittedPredictorValues, categoricalPredictors, categoricalPredictorLevels, warnedAboutMissingLevels, allValuesForPredictorAreNA, allValuesAreNA, newData, TRUE)[[1]]
+    predictedResponse <- PredictModel(model, rPackage, NULL, ignoreOutOfRangeValues, FALSE, allPredictors, responseVariable, minResponseValue, maxResponseValue, minFittedPredictorValues, maxFittedPredictorValues, categoricalPredictors, categoricalPredictorLevels, warnedAboutMissingLevels, allValuesForPredictorAreNA, allValuesAreNA, newData)[[1]]
     
     # If the data include the response variable, compute summary statistics.

MGET/Branches/Jason/PythonPackage/src/GeoEco/__init__.py

@@ -1 @@
-__version__ = u'0.8a42'
+__version__ = u'0.8a43'