| 1 | # DiGIR/__init__.py - Provides methods for retrieving biogeographic
|
|---|
| 2 | # data using the DiGIR protocol.
|
|---|
| 3 | #
|
|---|
| 4 | # Copyright (C) 2009 Jason J. Roberts
|
|---|
| 5 | #
|
|---|
| 6 | # This program is free software; you can redistribute it and/or
|
|---|
| 7 | # modify it under the terms of the GNU General Public License
|
|---|
| 8 | # as published by the Free Software Foundation; either version 2
|
|---|
| 9 | # of the License, or (at your option) any later version.
|
|---|
| 10 | #
|
|---|
| 11 | # This program is distributed in the hope that it will be useful,
|
|---|
| 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|---|
| 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|---|
| 14 | # GNU General Public License (available in the file LICENSE.TXT)
|
|---|
| 15 | # for more details.
|
|---|
| 16 | #
|
|---|
| 17 | # You should have received a copy of the GNU General Public License
|
|---|
| 18 | # along with this program; if not, write to the Free Software
|
|---|
| 19 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
|---|
| 20 |
|
|---|
| 21 | import copy
|
|---|
| 22 | import datetime
|
|---|
| 23 | import os
|
|---|
| 24 | import re
|
|---|
| 25 | import StringIO
|
|---|
| 26 | import sys
|
|---|
| 27 | import time
|
|---|
| 28 | import types
|
|---|
| 29 | import urllib2
|
|---|
| 30 |
|
|---|
| 31 | from GeoEco.DataManagement.ArcGISRasters import ArcGISRaster
|
|---|
| 32 | from GeoEco.DynamicDocString import DynamicDocString
|
|---|
| 33 | from GeoEco.httplib2 import Http
|
|---|
| 34 | from GeoEco.Internationalization import _
|
|---|
| 35 | from GeoEco.Logging import Logger, ProgressReporter
|
|---|
| 36 |
|
|---|
| 37 | class DiGIR(object):
|
|---|
| 38 | __doc__ = DynamicDocString()
|
|---|
| 39 |
|
|---|
| 40 | @classmethod
|
|---|
| 41 | def SendRequestToServer(cls, url, requestXML=None, resource=None, httpobj=None, timeout=None, maxRetryTime=None):
|
|---|
| 42 | cls.__doc__.Obj.ValidateMethodInvocation()
|
|---|
| 43 | try:
|
|---|
| 44 |
|
|---|
| 45 | # Perform additional validation and, if necessary, parse the
|
|---|
| 46 | # caller's requestXML into an lxml Element.
|
|---|
| 47 |
|
|---|
| 48 | from lxml import etree
|
|---|
| 49 |
|
|---|
| 50 | etreeElementType = type(etree.Element('foo'))
|
|---|
| 51 | if not isinstance(requestXML, (types.NoneType, basestring, etreeElementType)):
|
|---|
| 52 | Logger.RaiseException(TypeError(_(u'The value provided for the requestXML is an invalid type %(badType)s. Please provide a value having one of these types: %(goodTypes)s.') % {u'badType' : type(requestXML), u'goodTypes' : u', '.join(map(unicode, [types.NoneType, types.StringType, types.UnicodeType, etreeElementType]))}))
|
|---|
| 53 |
|
|---|
| 54 | if requestXML is None:
|
|---|
| 55 | requestElement = None
|
|---|
| 56 | requestType = u'metadata'
|
|---|
| 57 | elif isinstance(requestXML, etreeElementType):
|
|---|
| 58 | requestElement = requestXML
|
|---|
| 59 | else:
|
|---|
| 60 | requestElement = etree.fromstring(requestXML)
|
|---|
| 61 |
|
|---|
| 62 | if requestElement is not None:
|
|---|
| 63 | requestType = requestElement.tag
|
|---|
| 64 | if not requestType.startswith(u'{'):
|
|---|
| 65 | Logger.RaiseException(ValueError(_(u'The root element of the requestXML, <%(tag)s>, does not have an XML namespace. This element must come from the DiGIR protocol namespace "http://digir.net/schema/protocol/2003/1.0". To fix this, just add the attribute xmlns to the root element, with the value "http://digir.net/schema/protocol/2003/1.0".') % {u'tag': requestElement.tag}))
|
|---|
| 66 | if requestType.split(u'}')[0][1:] != u'http://digir.net/schema/protocol/2003/1.0':
|
|---|
| 67 | Logger.RaiseException(ValueError(_(u'The root element of the requestXML, <%(tag)s>, is from the "%(ns)s" XML namespace. This element must come from the DiGIR protocol namespace "http://digir.net/schema/protocol/2003/1.0". Please change the namespace for this element, and any children that should be from the DiGIR protocol namespace.') % {u'tag': requestType.split(u'}')[1], u'ns': requestType.split(u'}')[0][1:]}))
|
|---|
| 68 | requestType = requestType.split(u'}')[1]
|
|---|
| 69 | if requestType not in [u'inventory', u'search']:
|
|---|
| 70 | Logger.RaiseException(ValueError(_(u'The root element of the requestXML, <%(tag)s>, is not allowed because it does not appear in the DiGIR protocol schema. The root element must be either <inventory> or <search>.') % {u'tag': requestType}))
|
|---|
| 71 | if resource is None:
|
|---|
| 72 | Logger.Debug(_(u'WARNING: DiGIR.SendRequestToServer() called with requestType=%(type)s but resource=None. Many DiGIR servers require a resource to be provided for this type of request. The request may fail.') % {u'type': requestType})
|
|---|
| 73 |
|
|---|
| 74 | # Build an lxml Element representing the root element of the
|
|---|
| 75 | # XML request we will send to the server.
|
|---|
| 76 |
|
|---|
| 77 | import time
|
|---|
| 78 | import socket
|
|---|
| 79 |
|
|---|
| 80 | digirNS = u'http://digir.net/schema/protocol/2003/1.0'
|
|---|
| 81 | digirNSPrefix = u'{%s}' % digirNS
|
|---|
| 82 |
|
|---|
| 83 | rootElement = etree.Element(digirNSPrefix + u'request', nsmap={None: digirNS})
|
|---|
| 84 |
|
|---|
| 85 | headerElement = etree.SubElement(rootElement, digirNSPrefix + u'header')
|
|---|
| 86 | versionElement = etree.SubElement(headerElement, digirNSPrefix + u'version')
|
|---|
| 87 | versionElement.text = u'1.0.0'
|
|---|
| 88 | sendTimeElement = etree.SubElement(headerElement, digirNSPrefix + u'sendTime')
|
|---|
| 89 | sendTimeElement.text = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
|---|
| 90 | sourceElement = etree.SubElement(headerElement, digirNSPrefix + u'source')
|
|---|
| 91 | try:
|
|---|
| 92 | sourceElement.text = socket.gethostbyaddr(socket.gethostname())[2][0]
|
|---|
| 93 | except:
|
|---|
| 94 | sourceElement.text = u'255.255.255.255' # Should only happen when machine has no IP address, but we don't want to fail now; we'd rather fail trying to connect to the server.
|
|---|
| 95 | destinationElement = etree.SubElement(headerElement, digirNSPrefix + u'destination')
|
|---|
| 96 | destinationElement.text = url
|
|---|
| 97 | if resource is not None:
|
|---|
| 98 | destinationElement.attrib[u'resource'] = resource
|
|---|
| 99 | typeElement = etree.SubElement(headerElement, digirNSPrefix + u'type')
|
|---|
| 100 | typeElement.text = requestType
|
|---|
| 101 |
|
|---|
| 102 | if requestElement is not None:
|
|---|
| 103 | rootElement.append(requestElement)
|
|---|
| 104 |
|
|---|
| 105 | # Build an XML document from the root element, encoded in
|
|---|
| 106 | # UTF-8. This code was adapted from the implementation of
|
|---|
| 107 | # lxml.etree.tostring().
|
|---|
| 108 |
|
|---|
| 109 | class dummy:
|
|---|
| 110 | pass
|
|---|
| 111 | data = []
|
|---|
| 112 | f = dummy()
|
|---|
| 113 | f.write = data.append
|
|---|
| 114 | etree.ElementTree(rootElement).write(f, encoding='UTF-8', xml_declaration=True, method='xml')
|
|---|
| 115 | xmlDoc = ''.join(data)
|
|---|
| 116 |
|
|---|
| 117 | # Allocate a GeoEco.httplib2.Http instance, if the caller
|
|---|
| 118 | # did not provide one.
|
|---|
| 119 |
|
|---|
| 120 | if httpobj is None or (httpobj.timeout != timeout):
|
|---|
| 121 | httpobj = Http(timeout=timeout)
|
|---|
| 122 | httpobj.follow_all_redirects = True
|
|---|
| 123 | httpobj.force_exception_to_status_code = False
|
|---|
| 124 |
|
|---|
| 125 | # Send the request to the server.
|
|---|
| 126 |
|
|---|
| 127 | import logging
|
|---|
| 128 | import urllib
|
|---|
| 129 | import time
|
|---|
| 130 |
|
|---|
| 131 | Logger.Debug(_('Sending request to DiGIR server %(url)s:\n%(request)s') % {u'url': url, u'request': etree.tostring(rootElement, pretty_print=True)})
|
|---|
| 132 | started = time.clock()
|
|---|
| 133 |
|
|---|
| 134 | try:
|
|---|
| 135 | response, content = httpobj.request_with_retry(url, method='POST', body=urllib.urlencode({'request': xmlDoc}), headers={'Content-Type': 'application/x-www-form-urlencoded', 'Accept': 'text/plain'}, max_retry_time=maxRetryTime, logger=logging.getLogger('GeoEco'), message=_(u'The DiGIR %(req)s request to the server %(url)s failed due to %%(e)s: %%(msg)s. Retrying...') % {u'req': requestType, u'url': url})
|
|---|
| 136 | except Exception, e:
|
|---|
| 137 | if e.__class__.__name__ == 'ExecuteAbort' and str(e).lower() == 'cancelled function':
|
|---|
| 138 | raise
|
|---|
| 139 | if self._MaxRetryTime is not None:
|
|---|
| 140 | raise RuntimeError(_(u'The DiGIR %(req)s request to the server %(url)s failed. The request was retried for %(retry)i seconds without success. Verify that the server\'s URL is correct. If it is, check that the server is operating properly and that your computer can connect to it. If necessary, contact the server\'s operator for assistance. Error details: %(e)s: %(msg)s') % {u'req': requestType, u'url': url, u'retry': maxRetryTime, u'e': e.__class__.__name__, u'msg': unicode(e)})
|
|---|
| 141 | raise RuntimeError(_(u'The DiGIR %(req)s request to the server %(url)s failed. Verify that the server\'s URL is correct. If it is, check that the server is operating properly and that your computer can connect to it. If necessary, contact the server\'s operator for assistance. Error details: %(e)s: %(msg)s') % {u'req': requestType, u'url': url, u'e': e.__class__.__name__, u'msg': unicode(e)})
|
|---|
| 142 |
|
|---|
| 143 | # TODO: Validate the returned content against the DiGIR
|
|---|
| 144 | # protocol schema?
|
|---|
| 145 |
|
|---|
| 146 | # Return successfully.
|
|---|
| 147 |
|
|---|
| 148 | Logger.Debug(_('The server returned %(len)i bytes in %(sec).1f seconds.') % {u'len': len(content), u'sec': time.clock() - started})
|
|---|
| 149 | return content, httpobj
|
|---|
| 150 |
|
|---|
| 151 | except:
|
|---|
| 152 | Logger.LogExceptionAsError()
|
|---|
| 153 | raise
|
|---|
| 154 |
|
|---|
| 155 | @classmethod
|
|---|
| 156 | def _SendRequestToServerAndParseXMLResponse(cls, url, requestXML=None, resource=None, httpobj=None, timeout=None, maxRetryTime=None):
|
|---|
| 157 |
|
|---|
| 158 | # The OBIS server sometimes returns incomplete XML documents
|
|---|
| 159 | # when it is under heavy load. When this happens, the HTTP
|
|---|
| 160 | # request succeeds but the resulting document is incomplete
|
|---|
| 161 | # and will fail to parse. If this happens, try the request
|
|---|
| 162 | # again, up to three times.
|
|---|
| 163 |
|
|---|
| 164 | from lxml import etree
|
|---|
| 165 | retries = 0
|
|---|
| 166 | while True:
|
|---|
| 167 | response, httpobj = cls.SendRequestToServer(url, requestXML, resource, httpobj, timeout, maxRetryTime)
|
|---|
| 168 | try:
|
|---|
| 169 | return etree.parse(StringIO.StringIO(response)).getroot(), httpobj
|
|---|
| 170 | except etree.XMLSyntaxError, e:
|
|---|
| 171 | if (unicode(e).lower().startswith('premature end of data') or unicode(e).lower().startswith('couldn\'t find end of start tag')) and retries < 3:
|
|---|
| 172 | Logger.Debug(_(u'The data returned from the server appears to be an incomplete XML document; the parser raised %(e)s: %(msg)s. This can happen if the server is under heavy load. Retrying the request...') % {u'e': e.__class__.__name__, u'msg': unicode(e)})
|
|---|
| 173 | retries += 1
|
|---|
| 174 | continue
|
|---|
| 175 | raise
|
|---|
| 176 |
|
|---|
| 177 | @classmethod
|
|---|
| 178 | def GetResourcesAsArcGISTable(cls, url, resourcesTable, relatedInfoTable=None, contactsTable=None, schemasTable=None, dataElementsTable=None, timeout=None, maxRetryTime=None, overwriteExisting=False):
|
|---|
| 179 | cls.__doc__.Obj.ValidateMethodInvocation()
|
|---|
| 180 | try:
|
|---|
| 181 |
|
|---|
| 182 | # Get the metadata XML.
|
|---|
| 183 |
|
|---|
| 184 | from lxml import etree
|
|---|
| 185 |
|
|---|
| 186 | Logger.Info(_(u'Requesting DiGIR metadata from server %(url)s.') % {u'url': url})
|
|---|
| 187 | metadataRoot = cls._SendRequestToServerAndParseXMLResponse(url, timeout=timeout, maxRetryTime=maxRetryTime)[0]
|
|---|
| 188 |
|
|---|
| 189 | # Create temporary directories and/or personal
|
|---|
| 190 | # geodatabases to hold the output tables while we are
|
|---|
| 191 | # constructing them.
|
|---|
| 192 |
|
|---|
| 193 | from GeoEco.DataManagement.Directories import TemporaryDirectory
|
|---|
| 194 | tempDir = TemporaryDirectory()
|
|---|
| 195 |
|
|---|
| 196 | from GeoEco.ArcGIS import GeoprocessorManager
|
|---|
| 197 | gp = GeoprocessorManager.GetWrappedGeoprocessor()
|
|---|
| 198 |
|
|---|
| 199 | resourcesTableIsDBF = os.path.isdir(os.path.dirname(resourcesTable)) and gp.Describe(os.path.dirname(resourcesTable)).DataType.lower() == u'folder'
|
|---|
| 200 | relatedInfoTableIsDBF = relatedInfoTable is None or (os.path.isdir(os.path.dirname(relatedInfoTable)) and gp.Describe(os.path.dirname(relatedInfoTable)).DataType.lower() == u'folder')
|
|---|
| 201 | contactsTableIsDBF = contactsTable is None or (os.path.isdir(os.path.dirname(contactsTable)) and gp.Describe(os.path.dirname(contactsTable)).DataType.lower() == u'folder')
|
|---|
| 202 | schemasTableIsDBF = schemasTable is None or (os.path.isdir(os.path.dirname(schemasTable)) and gp.Describe(os.path.dirname(schemasTable)).DataType.lower() == u'folder')
|
|---|
| 203 | dataElementsTableIsDBF = dataElementsTable is None or (os.path.isdir(os.path.dirname(dataElementsTable)) and gp.Describe(os.path.dirname(dataElementsTable)).DataType.lower() == u'folder')
|
|---|
| 204 |
|
|---|
| 205 | if not resourcesTableIsDBF or not relatedInfoTableIsDBF or not contactsTableIsDBF or not schemasTableIsDBF or not dataElementsTableIsDBF:
|
|---|
| 206 | gp.CreatePersonalGDB_management(tempDir.Path, u'Temp.mdb')
|
|---|
| 207 | tempGeoDB = os.path.join(tempDir.Path, u'Temp.mdb')
|
|---|
| 208 | else:
|
|---|
| 209 | tempGeoDB = None
|
|---|
| 210 |
|
|---|
| 211 | try:
|
|---|
| 212 | if resourcesTableIsDBF:
|
|---|
| 213 | tempResourcesTable = os.path.join(tempDir.Path, u'resources.dbf')
|
|---|
| 214 | else:
|
|---|
| 215 | tempResourcesTable = os.path.join(tempGeoDB, u'resources')
|
|---|
| 216 |
|
|---|
| 217 | if relatedInfoTable is not None:
|
|---|
| 218 | if relatedInfoTableIsDBF:
|
|---|
| 219 | tempRelatedInfoTable = os.path.join(tempDir.Path, u'related.dbf')
|
|---|
| 220 | else:
|
|---|
| 221 | tempRelatedInfoTable = os.path.join(tempGeoDB, u'related')
|
|---|
| 222 |
|
|---|
| 223 | if contactsTable is not None:
|
|---|
| 224 | if contactsTableIsDBF:
|
|---|
| 225 | tempContactsTable = os.path.join(tempDir.Path, u'contacts.dbf')
|
|---|
| 226 | else:
|
|---|
| 227 | tempContactsTable = os.path.join(tempGeoDB, u'contacts')
|
|---|
| 228 |
|
|---|
| 229 | if schemasTable is not None:
|
|---|
| 230 | if schemasTableIsDBF:
|
|---|
| 231 | tempSchemasTable = os.path.join(tempDir.Path, u'schemas.dbf')
|
|---|
| 232 | else:
|
|---|
| 233 | tempSchemasTable = os.path.join(tempGeoDB, u'schemas')
|
|---|
| 234 |
|
|---|
| 235 | if dataElementsTable is not None:
|
|---|
| 236 | if dataElementsTableIsDBF:
|
|---|
| 237 | tempDataElementsTable = os.path.join(tempDir.Path, u'dataElem.dbf')
|
|---|
| 238 | else:
|
|---|
| 239 | tempDataElementsTable = os.path.join(tempGeoDB, u'dataElem')
|
|---|
| 240 |
|
|---|
| 241 | # Create the temporary tables.
|
|---|
| 242 |
|
|---|
| 243 | cls._CreateArcGISTableFromXMLResponses([metadataRoot],
|
|---|
| 244 | 'GetResourcesFromMetadataResponse.xslt',
|
|---|
| 245 | tempResourcesTable,
|
|---|
| 246 | None,
|
|---|
| 247 | ['Code', 'NumRecords', 'LastUpdate', 'Name', 'Abstract', 'Keywords', 'Citation', 'UseRestr', 'RecordID', 'RecBasis'],
|
|---|
| 248 | ['TEXT', 'LONG', 'DATE', 'TEXT', 'TEXT', 'TEXT', 'TEXT', 'TEXT', 'TEXT', 'TEXT'],
|
|---|
| 249 | _(u'Creating the response table.'))
|
|---|
| 250 |
|
|---|
| 251 | if relatedInfoTable is not None:
|
|---|
| 252 | cls._CreateArcGISTableFromXMLResponses([metadataRoot],
|
|---|
| 253 | 'GetRelatedInfoFromMetadataResponse.xslt',
|
|---|
| 254 | tempRelatedInfoTable,
|
|---|
| 255 | None,
|
|---|
| 256 | ['Code', 'RelInfo'],
|
|---|
| 257 | ['TEXT', 'TEXT'],
|
|---|
| 258 | _(u'Creating the related information table.'))
|
|---|
| 259 |
|
|---|
| 260 | if contactsTable is not None:
|
|---|
| 261 | cls._CreateArcGISTableFromXMLResponses([metadataRoot],
|
|---|
| 262 | 'GetContactsFromMetadataResponse.xslt',
|
|---|
| 263 | tempContactsTable,
|
|---|
| 264 | None,
|
|---|
| 265 | ['Code', 'Name', 'Title', 'Email', 'Phone', 'Type'],
|
|---|
| 266 | ['TEXT', 'TEXT', 'TEXT', 'TEXT', 'TEXT', 'TEXT'],
|
|---|
| 267 | _(u'Creating the contacts table.'))
|
|---|
| 268 |
|
|---|
| 269 | if schemasTable is not None:
|
|---|
| 270 | cls._CreateArcGISTableFromXMLResponses([metadataRoot],
|
|---|
| 271 | 'GetSchemasFromMetadataResponse.xslt',
|
|---|
| 272 | tempSchemasTable,
|
|---|
| 273 | None,
|
|---|
| 274 | ['Code', 'Namespace', 'Location'],
|
|---|
| 275 | ['TEXT', 'TEXT', 'TEXT'],
|
|---|
| 276 | _(u'Creating the conceptual schemas table.'))
|
|---|
| 277 |
|
|---|
| 278 | # The data elements table requires special handling.
|
|---|
| 279 |
|
|---|
| 280 | if dataElementsTable is not None:
|
|---|
| 281 | namespaces = []
|
|---|
| 282 | locations = []
|
|---|
| 283 | for conceptualSchema in metadataRoot.xpath('/digir:response/digir:content/digir:metadata/digir:provider/digir:resource/digir:conceptualSchema[text() and @schemaLocation]', namespaces={'digir': 'http://digir.net/schema/protocol/2003/1.0'}):
|
|---|
| 284 | namespaces.append(conceptualSchema.text)
|
|---|
| 285 | locations.append(conceptualSchema.get('schemaLocation'))
|
|---|
| 286 | elementsRoot = DiGIR._GetDataElementsFromSchemas(namespaces, locations)[0]
|
|---|
| 287 |
|
|---|
| 288 | cls._CreateArcGISTableFromXMLResponses([elementsRoot],
|
|---|
| 289 | 'GetFieldsFromSubstitutableDataElements.xslt',
|
|---|
| 290 | tempDataElementsTable,
|
|---|
| 291 | None,
|
|---|
| 292 | ['Name', 'XSDType', 'ArcGISType', 'Searchable', 'Returnable', 'Namespace', 'Location', 'Descr'],
|
|---|
| 293 | ['TEXT', 'TEXT', 'TEXT', 'SHORT', 'SHORT', 'TEXT', 'TEXT', 'TEXT'],
|
|---|
| 294 | _(u'Creating the data elements table.'))
|
|---|
| 295 |
|
|---|
| 296 | # Extract the temporary tables to the caller's
|
|---|
| 297 | # destinations.
|
|---|
| 298 |
|
|---|
| 299 | oldLogInfoAsDebug = Logger.GetLogInfoAsDebug()
|
|---|
| 300 | Logger.SetLogInfoAsDebug(True)
|
|---|
| 301 | try:
|
|---|
| 302 | GeoprocessorManager.CopyArcGISObject(tempResourcesTable, resourcesTable, overwriteExisting, [u'table', u'DBaseTable'], _(u'table'))
|
|---|
| 303 | if relatedInfoTable is not None:
|
|---|
| 304 | GeoprocessorManager.CopyArcGISObject(tempRelatedInfoTable, relatedInfoTable, overwriteExisting, [u'table', u'DBaseTable'], _(u'table'))
|
|---|
| 305 | if contactsTable is not None:
|
|---|
| 306 | GeoprocessorManager.CopyArcGISObject(tempContactsTable, contactsTable, overwriteExisting, [u'table', u'DBaseTable'], _(u'table'))
|
|---|
| 307 | if schemasTable is not None:
|
|---|
| 308 | GeoprocessorManager.CopyArcGISObject(tempSchemasTable, schemasTable, overwriteExisting, [u'table', u'DBaseTable'], _(u'table'))
|
|---|
| 309 | if dataElementsTable is not None:
|
|---|
| 310 | GeoprocessorManager.CopyArcGISObject(tempDataElementsTable, dataElementsTable, overwriteExisting, [u'table', u'DBaseTable'], _(u'table'))
|
|---|
| 311 | finally:
|
|---|
| 312 | Logger.SetLogInfoAsDebug(oldLogInfoAsDebug)
|
|---|
| 313 |
|
|---|
| 314 | finally:
|
|---|
| 315 | # If we created a temporary personal geodatabase,
|
|---|
| 316 | # delete it now. If we do not explicitly delete it
|
|---|
| 317 | # here, the geoprocessor will keep a handle open
|
|---|
| 318 | # through MS Access APIs, and the temporary directory
|
|---|
| 319 | # will not be able to be deleted.
|
|---|
| 320 |
|
|---|
| 321 | if tempGeoDB is not None:
|
|---|
| 322 | try:
|
|---|
| 323 | gp.Delete_management(tempGeoDB)
|
|---|
| 324 | except:
|
|---|
| 325 | pass
|
|---|
| 326 |
|
|---|
| 327 | except:
|
|---|
| 328 | Logger.LogExceptionAsError()
|
|---|
| 329 | raise
|
|---|
| 330 |
|
|---|
| 331 | @classmethod
|
|---|
| 332 | def SearchAndCreateArcGISPoints(cls, url, pointFeatureClass, filterExpression=None, resources=None, dataElementNames=None, xCoordinateDataElementName=None, yCoordinateDataElementName=None, maxRecords=None, timeout=None, maxRetryTime=None, overwriteExisting=False):
|
|---|
| 333 | cls.__doc__.Obj.ValidateMethodInvocation()
|
|---|
| 334 | try:
|
|---|
| 335 |
|
|---|
| 336 | # Perform additional validation.
|
|---|
| 337 |
|
|---|
| 338 | if resources is not None and len(resources) != len(set(resources)):
|
|---|
| 339 | Logger.RaiseException(ValueError(_(u'The resources parameter must not contain duplicate entries.')))
|
|---|
| 340 |
|
|---|
| 341 | if dataElementNames is not None and len(dataElementNames) != len(set(dataElementNames)):
|
|---|
| 342 | Logger.RaiseException(ValueError(_(u'The data element names parameter must not contain duplicate entries.')))
|
|---|
| 343 |
|
|---|
| 344 | # Get metadata from the server.
|
|---|
| 345 |
|
|---|
| 346 | Logger.Info(_(u'Requesting DiGIR metadata from server %(url)s.') % {u'url': url})
|
|---|
| 347 | metadataRoot, httpobj = cls._SendRequestToServerAndParseXMLResponse(url, timeout=timeout, maxRetryTime=maxRetryTime)
|
|---|
| 348 |
|
|---|
| 349 | # Enumerate the data elements available on the server.
|
|---|
| 350 |
|
|---|
| 351 | namespaces = []
|
|---|
| 352 | locations = []
|
|---|
| 353 | for conceptualSchema in metadataRoot.xpath('/digir:response/digir:content/digir:metadata/digir:provider/digir:resource/digir:conceptualSchema[text() and @schemaLocation]', namespaces={'digir': 'http://digir.net/schema/protocol/2003/1.0'}):
|
|---|
| 354 | namespaces.append(conceptualSchema.text)
|
|---|
| 355 | locations.append(conceptualSchema.get('schemaLocation'))
|
|---|
| 356 | elementsRoot, namespaces = DiGIR._GetDataElementsFromSchemas(namespaces, locations)
|
|---|
| 357 | elementsRoot = cls._TransformXMLDocs([elementsRoot], 'GetFieldsFromSubstitutableDataElements.xslt')[0]
|
|---|
| 358 |
|
|---|
| 359 | searchableDataElementNamesOnServer = [element.findtext('name') for element in elementsRoot.xpath('/rows/row[searchable/text()="1"]')]
|
|---|
| 360 | returnableDataElementNamesOnServer = [element.findtext('name') for element in elementsRoot.xpath('/rows/row[returnable/text()="1"]')]
|
|---|
| 361 |
|
|---|
| 362 | # If the caller did not specify which data elements he
|
|---|
| 363 | # wants to create as fields in the feature class, use all
|
|---|
| 364 | # those available on the server.
|
|---|
| 365 |
|
|---|
| 366 | if dataElementNames is None:
|
|---|
| 367 | dataElementNames = returnableDataElementNamesOnServer
|
|---|
| 368 |
|
|---|
| 369 | dataElementNamespaces = []
|
|---|
| 370 | dataElementArcGISTypes = []
|
|---|
| 371 | for name in dataElementNames:
|
|---|
| 372 | elements = elementsRoot.xpath('/rows/row[name/text()="%s"]' % name)
|
|---|
| 373 | if len(elements) <= 0:
|
|---|
| 374 | Logger.RaiseException(ValueError(_(u'The conceptual schemas used by the resources offered by the DiGIR server %(url)s do not contain a data element named "%(name)s". Please remove this name from the list of data elements to retrieve. For a list of valid names, use the Get DiGIR Resources tool to create a table of data elements available from the server. Note that names are case sensitive.') % {u'name': name, u'url': url}))
|
|---|
| 375 | element = elements[0]
|
|---|
| 376 | if element.findtext('returnable') != '1':
|
|---|
| 377 | Logger.RaiseException(ValueError(_(u'The DiGIR data element named "%(name)s" is not marked as being "returnable" by the conceptual schema %(namespace)s (located at %(location)s). Please remove this name from the list of data elements to retrieve.') % {u'name': name, u'namespace': element.findtext('namespace'), u'location': element.findtext('location')}))
|
|---|
| 378 | dataElementNamespaces.append(element.findtext('namespace'))
|
|---|
| 379 | dataElementArcGISTypes.append(element.findtext('arcGISType'))
|
|---|
| 380 |
|
|---|
| 381 | # Build the DiGIR <structure> element from the list of
|
|---|
| 382 | # data elements, instructing the server to return a set of
|
|---|
| 383 | # <record> elements.
|
|---|
| 384 |
|
|---|
| 385 | namespaceForPrefix = {None: 'http://digir.net/schema/protocol/2003/1.0', 'xsd': 'http://www.w3.org/2001/XMLSchema'}
|
|---|
| 386 | prefixForNamespace = {'http://digir.net/schema/protocol/2003/1.0': None, 'http://www.w3.org/2001/XMLSchema': 'xsd'}
|
|---|
| 387 | for i in range(len(namespaces)):
|
|---|
| 388 | if not prefixForNamespace.has_key(namespaces[i]):
|
|---|
| 389 | namespaceForPrefix['ns' + str(i)] = namespaces[i]
|
|---|
| 390 | prefixForNamespace[namespaces[i]] = 'ns' + str(i)
|
|---|
| 391 |
|
|---|
| 392 | from lxml import etree
|
|---|
| 393 |
|
|---|
| 394 | structureElement = etree.Element('{http://digir.net/schema/protocol/2003/1.0}structure', nsmap=namespaceForPrefix)
|
|---|
| 395 | xsdElementElement = etree.SubElement(structureElement, '{http://www.w3.org/2001/XMLSchema}element')
|
|---|
| 396 | xsdElementElement.set('name', 'record')
|
|---|
| 397 | xsdComplexTypeElement = etree.SubElement(xsdElementElement, '{http://www.w3.org/2001/XMLSchema}complexType')
|
|---|
| 398 | xsdSequenceElement = etree.SubElement(xsdComplexTypeElement, '{http://www.w3.org/2001/XMLSchema}sequence')
|
|---|
| 399 |
|
|---|
| 400 | # Add a <coordinates> element to the <record> element. If
|
|---|
| 401 | # the caller specified names of data elements to use for
|
|---|
| 402 | # longitude and latitude, use them. If not, try the common
|
|---|
| 403 | # names "Longitude" and "DecimalLongitude", and "Latitude"
|
|---|
| 404 | # and "DecimalLatitude".
|
|---|
| 405 |
|
|---|
| 406 | xsdElementElement = etree.SubElement(xsdSequenceElement, '{http://www.w3.org/2001/XMLSchema}element')
|
|---|
| 407 | xsdElementElement.set('name', 'coordinates')
|
|---|
| 408 | xsdComplexTypeElement = etree.SubElement(xsdElementElement, '{http://www.w3.org/2001/XMLSchema}complexType')
|
|---|
| 409 | xsdSequenceElement2 = etree.SubElement(xsdComplexTypeElement, '{http://www.w3.org/2001/XMLSchema}sequence')
|
|---|
| 410 |
|
|---|
| 411 | xsdElementElement = etree.SubElement(xsdSequenceElement2, '{http://www.w3.org/2001/XMLSchema}element') # X coordinate
|
|---|
| 412 | if xCoordinateDataElementName is not None:
|
|---|
| 413 | xCoordinateDataElementNamespace = cls._ValidateCoordinateName(xCoordinateDataElementName, elementsRoot, url, 'X')
|
|---|
| 414 | else:
|
|---|
| 415 | try:
|
|---|
| 416 | xCoordinateDataElementName = u'Longitude'
|
|---|
| 417 | xCoordinateDataElementNamespace = cls._ValidateCoordinateName(xCoordinateDataElementName, elementsRoot, url, 'X')
|
|---|
| 418 | except:
|
|---|
| 419 | try:
|
|---|
| 420 | xCoordinateDataElementName = u'DecimalLongitude'
|
|---|
| 421 | xCoordinateDataElementNamespace = cls._ValidateCoordinateName(xCoordinateDataElementName, elementsRoot, url, 'X')
|
|---|
| 422 | except:
|
|---|
| 423 | xCoordinateDataElementName = None
|
|---|
| 424 | xCoordinateDataElementNamespace = None
|
|---|
| 425 | if xCoordinateDataElementNamespace is None:
|
|---|
| 426 | Logger.RaiseException(ValueError(_(u'You must supply the name of the DiGIR data element representing the X coordinate. Neither Longitude nor DecimalLongitude could be used because the conceptual schemas used by the resources offered by the DiGIR server %(url)s either do not contain those data elements or do not define them in a way that allows them to be used.')))
|
|---|
| 427 | xsdElementElement.set('ref', prefixForNamespace[xCoordinateDataElementNamespace] + ':' + xCoordinateDataElementName)
|
|---|
| 428 |
|
|---|
| 429 | xsdElementElement = etree.SubElement(xsdSequenceElement2, '{http://www.w3.org/2001/XMLSchema}element') # Y coordinate
|
|---|
| 430 | if yCoordinateDataElementName is not None:
|
|---|
| 431 | yCoordinateDataElementNamespace = cls._ValidateCoordinateName(yCoordinateDataElementName, elementsRoot, url, 'Y')
|
|---|
| 432 | else:
|
|---|
| 433 | try:
|
|---|
| 434 | yCoordinateDataElementName = u'Latitude'
|
|---|
| 435 | yCoordinateDataElementNamespace = cls._ValidateCoordinateName(yCoordinateDataElementName, elementsRoot, url, 'Y')
|
|---|
| 436 | except:
|
|---|
| 437 | try:
|
|---|
| 438 | yCoordinateDataElementName = u'DecimalLatitude'
|
|---|
| 439 | yCoordinateDataElementNamespace = cls._ValidateCoordinateName(yCoordinateDataElementName, elementsRoot, url, 'Y')
|
|---|
| 440 | except:
|
|---|
| 441 | yCoordinateDataElementName = None
|
|---|
| 442 | yCoordinateDataElementNamespace = None
|
|---|
| 443 | if yCoordinateDataElementName is None:
|
|---|
| 444 | Logger.RaiseException(ValueError(_(u'You must supply the name of the DiGIR data element representing the Y coordinate. Neither Latitude nor DecimalLatitude could be used because the conceptual schemas used by the resources offered by the DiGIR server %(url)s either do not contain those data elements or do not define them in a way that allows them to be used.')))
|
|---|
| 445 | xsdElementElement.set('ref', prefixForNamespace[yCoordinateDataElementNamespace] + ':' + yCoordinateDataElementName)
|
|---|
| 446 |
|
|---|
| 447 | # Add an <attributes> element to the <record> element, for
|
|---|
| 448 | # the data elements requested by the caller.
|
|---|
| 449 |
|
|---|
| 450 | if len(dataElementNames) > 0:
|
|---|
| 451 | xsdElementElement = etree.SubElement(xsdSequenceElement, '{http://www.w3.org/2001/XMLSchema}element')
|
|---|
| 452 | xsdElementElement.set('name', 'attributes')
|
|---|
| 453 | xsdComplexTypeElement = etree.SubElement(xsdElementElement, '{http://www.w3.org/2001/XMLSchema}complexType')
|
|---|
| 454 | xsdSequenceElement2 = etree.SubElement(xsdComplexTypeElement, '{http://www.w3.org/2001/XMLSchema}sequence')
|
|---|
| 455 |
|
|---|
| 456 | for i in range(len(dataElementNames)):
|
|---|
| 457 | xsdElementElement = etree.SubElement(xsdSequenceElement2, '{http://www.w3.org/2001/XMLSchema}element')
|
|---|
| 458 | xsdElementElement.set('ref', prefixForNamespace[dataElementNamespaces[i]] + ':' + dataElementNames[i])
|
|---|
| 459 |
|
|---|
| 460 | # Build the <filter> element.
|
|---|
| 461 |
|
|---|
| 462 | filterElement = cls._CreateFilterFromExpression(filterExpression,
|
|---|
| 463 | dict(zip([element.findtext('name') for element in elementsRoot.xpath('/rows/row')], [element.findtext('arcGISType') for element in elementsRoot.xpath('/rows/row')])),
|
|---|
| 464 | dict(zip([element.findtext('name') for element in elementsRoot.xpath('/rows/row')], [element.findtext('namespace') for element in elementsRoot.xpath('/rows/row')])),
|
|---|
| 465 | namespaceForPrefix,
|
|---|
| 466 | xCoordinateDataElementName,
|
|---|
| 467 | yCoordinateDataElementName)
|
|---|
| 468 |
|
|---|
| 469 | # If the caller did not specify any resources, we will
|
|---|
| 470 | # query all of the resources offered by the server. If the
|
|---|
| 471 | # caller did specify resources, verify that they are all
|
|---|
| 472 | # offered by the server.
|
|---|
| 473 |
|
|---|
| 474 | resourcesOnServer = map(str, metadataRoot.xpath('/digir:response/digir:content/digir:metadata/digir:provider/digir:resource/digir:code/text()', namespaces={'digir': 'http://digir.net/schema/protocol/2003/1.0'}))
|
|---|
| 475 | if resourcesOnServer is None or len(resourcesOnServer) <= 0:
|
|---|
| 476 | Logger.Info(_(u'The metadata response from the DiGIR server %(url)s does not contain any resources and therefore cannot be searched. Please specify a different server or contact the server operator and ask why this one does not contain any resources.') % {u'url': url})
|
|---|
| 477 |
|
|---|
| 478 | if resources is None:
|
|---|
| 479 | resources = copy.deepcopy(resourcesOnServer)
|
|---|
| 480 |
|
|---|
| 481 | for resource in resources:
|
|---|
| 482 | if resource not in resourcesOnServer:
|
|---|
| 483 | Logger.RaiseException(ValueError(_(u'The resource "%(res)s" is not offered by the DiGIR server %(url)s. Please remove this resource from your list of requested resources or specify a different server.') % {u'res': resource, u'url': url}))
|
|---|
| 484 |
|
|---|
| 485 | # For each resource, issue a search requesting the count
|
|---|
| 486 | # of records available for the filter.
|
|---|
| 487 | #
|
|---|
| 488 | # Note: it used to be that we could issue a <search>
|
|---|
| 489 | # request that only contained <filter> and <count>
|
|---|
| 490 | # elements. This was done in accordance with the
|
|---|
| 491 | # documentation for the <records> element in
|
|---|
| 492 | # http://digir.net/schema/protocol/2003/1.0/digir.xsd,
|
|---|
| 493 | # which said "Specifies the columns and rows of the
|
|---|
| 494 | # results to be returned. The element is optional because
|
|---|
| 495 | # some requests may simply be for the count, which is
|
|---|
| 496 | # specified by setting the count element to true."
|
|---|
| 497 | #
|
|---|
| 498 | # Apparently not all DiGIR server implementations respect
|
|---|
| 499 | # this. This includes the OBIS-SEAMAP DiGIR server
|
|---|
| 500 | # deployed by Duke. To work around it, we send a <records>
|
|---|
| 501 | # element with a limit of 1 record requesting only the
|
|---|
| 502 | # longitude element:
|
|---|
| 503 | #
|
|---|
| 504 | # <search>
|
|---|
| 505 | # <filter>...</filter>
|
|---|
| 506 | # <records limit="1" start="0">
|
|---|
| 507 | # <structure>
|
|---|
| 508 | # <xsd:element name="record">
|
|---|
| 509 | # <xsd:complexType>
|
|---|
| 510 | # <xsd:sequence>
|
|---|
| 511 | # <xsd:element ref="ns0:Longitude"/>
|
|---|
| 512 | # </xsd:sequence>
|
|---|
| 513 | # </xsd:complexType>
|
|---|
| 514 | # </xsd:element>
|
|---|
| 515 | # </structure>
|
|---|
| 516 | # </records>
|
|---|
| 517 | # <count>True</count>
|
|---|
| 518 | # </search>
|
|---|
| 519 | #
|
|---|
| 520 | # The limit attribute does not appear to affect the count
|
|---|
| 521 | # that is returned. Even though we only get one record, we
|
|---|
| 522 | # still get an accurate count.
|
|---|
| 523 | #
|
|---|
| 524 | # See MGET ticket #431 for more information.
|
|---|
| 525 |
|
|---|
| 526 | searchElement = etree.Element('{http://digir.net/schema/protocol/2003/1.0}search', nsmap=namespaceForPrefix)
|
|---|
| 527 | searchElement.append(filterElement)
|
|---|
| 528 |
|
|---|
| 529 | recordsElement = etree.SubElement(searchElement, '{http://digir.net/schema/protocol/2003/1.0}records')
|
|---|
| 530 | recordsElement.attrib[u'limit'] = '1'
|
|---|
| 531 | recordsElement.attrib[u'start'] = '0'
|
|---|
| 532 | tempStructureElement = etree.SubElement(recordsElement, '{http://digir.net/schema/protocol/2003/1.0}structure')
|
|---|
| 533 | xsdElementElement = etree.SubElement(tempStructureElement, '{http://www.w3.org/2001/XMLSchema}element')
|
|---|
| 534 | xsdElementElement.set('name', 'record')
|
|---|
| 535 | xsdComplexTypeElement = etree.SubElement(xsdElementElement, '{http://www.w3.org/2001/XMLSchema}complexType')
|
|---|
| 536 | xsdSequenceElement = etree.SubElement(xsdComplexTypeElement, '{http://www.w3.org/2001/XMLSchema}sequence')
|
|---|
| 537 | xsdElementElement = etree.SubElement(xsdSequenceElement, '{http://www.w3.org/2001/XMLSchema}element')
|
|---|
| 538 | xsdElementElement.set('ref', prefixForNamespace[xCoordinateDataElementNamespace] + ':' + xCoordinateDataElementName)
|
|---|
| 539 |
|
|---|
| 540 | countElement = etree.SubElement(searchElement, '{http://digir.net/schema/protocol/2003/1.0}count')
|
|---|
| 541 | countElement.text = 'True'
|
|---|
| 542 |
|
|---|
| 543 | findMatchCounts = etree.XPath('/digir:response/digir:diagnostics/digir:diagnostic[@code="MATCH_COUNT"]', namespaces={'digir': 'http://digir.net/schema/protocol/2003/1.0'})
|
|---|
| 544 | resourceRecordCounts = []
|
|---|
| 545 |
|
|---|
| 546 | Logger.Info(_(u'Retrieving the counts of matching records for %(count)i DiGIR resources.') % {u'count': len(resources)})
|
|---|
| 547 | progressReporter = ProgressReporter(progressMessage1=_(u'Still retrieving counts: %(elapsed)s elapsed, %(opsCompleted)i resources queried, %(perOp)s per resource, %(opsRemaining)i remaining, estimated completion time: %(etc)s.'),
|
|---|
| 548 | completionMessage=_(u'Finished retrieving counts: %(elapsed)s elapsed, %(opsCompleted)i resources queried, %(perOp)s per resource.'))
|
|---|
| 549 | progressReporter.Start(len(resources))
|
|---|
| 550 |
|
|---|
| 551 | for resource in resources:
|
|---|
| 552 | responseRoot, httpobj = cls._SendRequestToServerAndParseXMLResponse(url, searchElement, resource, httpobj, timeout, maxRetryTime)
|
|---|
| 553 | elements = findMatchCounts(responseRoot)
|
|---|
| 554 | if len(elements) <= 0:
|
|---|
| 555 | Logger.Warning(_(u'For the resource %(name)s, failed to retrieve a count of records that match the search filter from the DiGIR server. It will be assumed that this resource has 0 records matching the search filter.') % {u'name': resource, u'url': url})
|
|---|
| 556 | resourceRecordCounts.append(0)
|
|---|
| 557 | else:
|
|---|
| 558 | try:
|
|---|
| 559 | count = int(elements[0].text)
|
|---|
| 560 | Logger.Debug(_(u'The resource %(name)s has %(count)i records matching the search filter.') % {u'name': resource, u'count': count})
|
|---|
| 561 | resourceRecordCounts.append(count)
|
|---|
| 562 | except:
|
|---|
| 563 | Logger.Warning(_(u'For the resource %(name)s, failed to parse the count of records that match the search filter from the DiGIR server. It will be assumed that this resource has 0 records matching the search filter.') % {u'name': resource, u'url': url})
|
|---|
| 564 | resourceRecordCounts.append(0)
|
|---|
| 565 | progressReporter.ReportProgress()
|
|---|
| 566 |
|
|---|
| 567 | # Create a temporary directory to hold temporary files and
|
|---|
| 568 | # the output feature class while we are constructing it.
|
|---|
| 569 |
|
|---|
| 570 | from GeoEco.DataManagement.Directories import TemporaryDirectory
|
|---|
| 571 | tempDir = TemporaryDirectory()
|
|---|
| 572 |
|
|---|
| 573 | # If the total record count is greater than zero, download the records.
|
|---|
| 574 |
|
|---|
| 575 | recordXMLDocFiles = []
|
|---|
| 576 | resourceForXMLDoc = []
|
|---|
| 577 | totalRecordCount = sum(resourceRecordCounts)
|
|---|
| 578 |
|
|---|
| 579 | if totalRecordCount <= 0:
|
|---|
| 580 | Logger.Info(_(u'The total number of matching records is 0. The output feature class will be empty.'))
|
|---|
| 581 | else:
|
|---|
| 582 |
|
|---|
| 583 | # For each resource, issue a search request for the
|
|---|
| 584 | # records. If the count of records exceeds the allowed
|
|---|
| 585 | # number of responses, issue multiple sequential requests.
|
|---|
| 586 |
|
|---|
| 587 | if maxRecords is not None and totalRecordCount > maxRecords:
|
|---|
| 588 | Logger.Warning(_(u'The total number of matching records (%(count)i) exceeds the maximum number of records to download (%(max)i). Only the first %(max)i records will be downloaded.') % {u'count': totalRecordCount, u'max': maxRecords})
|
|---|
| 589 | totalRecordCount = maxRecords
|
|---|
| 590 |
|
|---|
| 591 | totalRecordsRetrieved = 0
|
|---|
| 592 | findEndOfRecords = etree.XPath('/digir:response/digir:diagnostics/digir:diagnostic[@code="END_OF_RECORDS" and text()="true"]', namespaces={'digir': 'http://digir.net/schema/protocol/2003/1.0'})
|
|---|
| 593 |
|
|---|
| 594 | Logger.Info(_(u'Downloading %(count)i matching records.') % {u'count': totalRecordCount})
|
|---|
| 595 | progressReporter = ProgressReporter(progressMessage1=_(u'Still downloading records: %(elapsed)s elapsed, %(opsCompleted)i records downloaded, %(perOp)s per record, %(opsRemaining)i remaining, estimated completion time: %(etc)s.'),
|
|---|
| 596 | completionMessage=_(u'Finished downloading records: %(elapsed)s elapsed, %(opsCompleted)i records downloaded, %(perOp)s per record.'))
|
|---|
| 597 | progressReporter.Start(totalRecordCount)
|
|---|
| 598 |
|
|---|
| 599 | for i in range(len(resources)):
|
|---|
| 600 |
|
|---|
| 601 | # If this resource has no matching records, skip it.
|
|---|
| 602 |
|
|---|
| 603 | if resourceRecordCounts[i] <= 0:
|
|---|
| 604 | continue
|
|---|
| 605 |
|
|---|
| 606 | # Determine the maximum number of records we can
|
|---|
| 607 | # obtain from this resource in one request.
|
|---|
| 608 |
|
|---|
| 609 | maxSearchResponseRecords = 2147483647
|
|---|
| 610 | elements = metadataRoot.xpath('/digir:response/digir:content/digir:metadata/digir:provider/digir:resource[digir:code/text()="' + resources[i] + '"]/digir:maxSearchResponseRecords[1]/text()', namespaces={'digir': 'http://digir.net/schema/protocol/2003/1.0'})
|
|---|
| 611 | if len(elements) > 0:
|
|---|
| 612 | try:
|
|---|
| 613 | maxSearchResponseRecords = int(elements[0])
|
|---|
| 614 | except:
|
|---|
| 615 | Logger.Warning(_(u'Failed to parse an integer from the value of the <maxSearchResponseRecords> element of the DiGIR metadata for resource %(name)s. It will be assumed that the value of this element is 1000.') % {u'name': resources[i]})
|
|---|
| 616 | maxSearchResponseRecords = 1000
|
|---|
| 617 |
|
|---|
| 618 | # Retrieve all of the records, in chunks of the max
|
|---|
| 619 | # number.
|
|---|
| 620 |
|
|---|
| 621 | maxSearchResponseRecords = 250 # TODO: REMOVE AFTER TESTING IS COMPLETE
|
|---|
| 622 |
|
|---|
| 623 | resourceRecordsRetrieved = 0
|
|---|
| 624 |
|
|---|
| 625 | while resourceRecordsRetrieved < resourceRecordCounts[i] and totalRecordsRetrieved < totalRecordCount:
|
|---|
| 626 | recordsToGet = resourceRecordCounts[i] - resourceRecordsRetrieved
|
|---|
| 627 | if recordsToGet > maxSearchResponseRecords:
|
|---|
| 628 | recordsToGet = maxSearchResponseRecords
|
|---|
| 629 | if recordsToGet > totalRecordCount - totalRecordsRetrieved:
|
|---|
| 630 | recordsToGet = totalRecordCount - totalRecordsRetrieved
|
|---|
| 631 |
|
|---|
| 632 | # Build the <search> element.
|
|---|
| 633 |
|
|---|
| 634 | searchElement = etree.Element('{http://digir.net/schema/protocol/2003/1.0}search', nsmap=namespaceForPrefix)
|
|---|
| 635 | searchElement.append(filterElement)
|
|---|
| 636 | recordsElement = etree.SubElement(searchElement, '{http://digir.net/schema/protocol/2003/1.0}records')
|
|---|
| 637 | recordsElement.set('start', str(resourceRecordsRetrieved))
|
|---|
| 638 | recordsElement.set('limit', str(recordsToGet))
|
|---|
| 639 | recordsElement.append(structureElement)
|
|---|
| 640 |
|
|---|
| 641 | # Send the request to the server.
|
|---|
| 642 | #
|
|---|
| 643 | # The OBIS server sometimes returns incomplete
|
|---|
| 644 | # XML documents when it is under heavy load.
|
|---|
| 645 | # When this happens, the HTTP request succeeds
|
|---|
| 646 | # but the resulting document is incomplete and
|
|---|
| 647 | # will fail to parse. If this happens, try the
|
|---|
| 648 | # request again, up to three times.
|
|---|
| 649 |
|
|---|
| 650 | Logger.Debug(_(u'Requesting %(count)i records for resouce %(name)s.') % {u'count': recordsToGet, u'name': resources[i]})
|
|---|
| 651 | retries = 0
|
|---|
| 652 | while True:
|
|---|
| 653 | try:
|
|---|
| 654 | responseXML = cls.SendRequestToServer(url, searchElement, resources[i], httpobj, timeout, maxRetryTime)[0]
|
|---|
| 655 |
|
|---|
| 656 | # Before parsing the XML, check whether the
|
|---|
| 657 | # <content> element contains xmlns definitions for
|
|---|
| 658 | # the namespaces we submitted. The main DiGIR
|
|---|
| 659 | # server PHP implementation omits these, which is
|
|---|
| 660 | # technically invalid. It will cause etree.parse
|
|---|
| 661 | # to fail. If necessary, stick these in before we
|
|---|
| 662 | # parse the string.
|
|---|
| 663 |
|
|---|
| 664 | contentStart = responseXML.find('<content')
|
|---|
| 665 | if contentStart >= 0:
|
|---|
| 666 | contentEnd = responseXML.find('>', contentStart)
|
|---|
| 667 | toAppend = ''
|
|---|
| 668 | for prefix, namespace in namespaceForPrefix.items():
|
|---|
| 669 | if prefix is not None and responseXML.find('xmlns:' + prefix + '=') < 0:
|
|---|
| 670 | toAppend += ' xmlns:' + prefix + '="' + namespace + '"'
|
|---|
| 671 | if len(toAppend) > 0:
|
|---|
| 672 | responseXML = responseXML.replace('<content', '<content' + toAppend, 1)
|
|---|
| 673 |
|
|---|
| 674 | # Now parse the XML.
|
|---|
| 675 |
|
|---|
| 676 | responseRoot = etree.parse(StringIO.StringIO(responseXML)).getroot()
|
|---|
| 677 |
|
|---|
| 678 | except etree.XMLSyntaxError, e:
|
|---|
| 679 | if (unicode(e).lower().startswith('premature end of data') or unicode(e).lower().startswith('couldn\'t find end of start tag')) and retries < 3:
|
|---|
| 680 | Logger.Debug(_(u'The data returned from the server appears to be an incomplete XML document; the parser raised %(e)s: %(msg)s. This can happen if the server is under heavy load. Retrying the request...') % {u'e': e.__class__.__name__, u'msg': unicode(e)})
|
|---|
| 681 | retries += 1
|
|---|
| 682 | continue
|
|---|
| 683 | raise
|
|---|
| 684 |
|
|---|
| 685 | break
|
|---|
| 686 |
|
|---|
| 687 | # Count the number of records we actually did
|
|---|
| 688 | # receive and update our progress.
|
|---|
| 689 |
|
|---|
| 690 | recordsReceived = len(responseRoot.find('{http://digir.net/schema/protocol/2003/1.0}content'))
|
|---|
| 691 | resourceRecordsRetrieved += recordsReceived
|
|---|
| 692 | totalRecordsRetrieved += recordsReceived
|
|---|
| 693 | Logger.Debug(_(u'The server returned %(count)i records.') % {u'count': recordsReceived})
|
|---|
| 694 | progressReporter.ReportProgress(recordsReceived) # Assumes the server will not return more records than the limit we specified
|
|---|
| 695 |
|
|---|
| 696 | # If we got some records, append this response to
|
|---|
| 697 | # our list of responses to write to the output
|
|---|
| 698 | # feature class.
|
|---|
| 699 |
|
|---|
| 700 | if recordsReceived > 0:
|
|---|
| 701 | responseXMLTempFile = os.path.join(tempDir.Path, 'response%i' % len(recordXMLDocFiles))
|
|---|
| 702 | Logger.Debug(_(u'Writing the XML response from the server to temporary file %(name)s.') % {u'name': responseXMLTempFile})
|
|---|
| 703 | f = open(responseXMLTempFile, 'wb')
|
|---|
| 704 | try:
|
|---|
| 705 | f.write(responseXML)
|
|---|
| 706 | finally:
|
|---|
| 707 | try:
|
|---|
| 708 | f.close()
|
|---|
| 709 | except:
|
|---|
| 710 | pass
|
|---|
| 711 | recordXMLDocFiles.append(responseXMLTempFile)
|
|---|
| 712 | resourceForXMLDoc.append(resources[i])
|
|---|
| 713 |
|
|---|
| 714 | # Break out of the loop for this resource if we
|
|---|
| 715 | # reached the end of the records.
|
|---|
| 716 |
|
|---|
| 717 | if len(findEndOfRecords(responseRoot)) > 0 or resourceRecordsRetrieved >= resourceRecordCounts[i]:
|
|---|
| 718 | if resourceRecordsRetrieved < resourceRecordCounts[i]:
|
|---|
| 719 | Logger.Warning(_(u'Although the server initially reported that resource %(name)s had %(count1)i matching records, it only returned %(count2)i records.') % {u'name': resources[i], u'count1': resourceRecordCounts[i], u'count2': resourceRecordsRetrieved})
|
|---|
| 720 | progressReporter.TotalOperations -= resourceRecordCounts[i] - resourceRecordsRetrieved
|
|---|
| 721 | resourceRecordCounts[i] = resourceRecordsRetrieved
|
|---|
| 722 | break
|
|---|
| 723 |
|
|---|
| 724 | # TODO: Parse errors from the server.
|
|---|
| 725 |
|
|---|
| 726 | # If we have recieved all of the records, break out of
|
|---|
| 727 | # the for loop. This will only happen prematurely if
|
|---|
| 728 | # the caller specified a maxRecords that is less than
|
|---|
| 729 | # sum(resourceRecordCounts).
|
|---|
| 730 |
|
|---|
| 731 | if totalRecordsRetrieved >= totalRecordCount:
|
|---|
| 732 | break
|
|---|
| 733 |
|
|---|
| 734 | # If the output feature class will be stored in a
|
|---|
| 735 | # geodatabase, create a temporary personal geodatabase to
|
|---|
| 736 | # hold the feature class while we are constructing it.
|
|---|
| 737 |
|
|---|
| 738 | from GeoEco.ArcGIS import GeoprocessorManager
|
|---|
| 739 | gp = GeoprocessorManager.GetWrappedGeoprocessor()
|
|---|
| 740 |
|
|---|
| 741 | outputIsShapefile = os.path.isdir(os.path.dirname(pointFeatureClass)) and pointFeatureClass.lower().endswith('.shp')
|
|---|
| 742 |
|
|---|
| 743 | if not outputIsShapefile:
|
|---|
| 744 | gp.CreatePersonalGDB_management(tempDir.Path, u'Temp.mdb')
|
|---|
| 745 | tempGeoDB = os.path.join(tempDir.Path, u'Temp.mdb')
|
|---|
| 746 | else:
|
|---|
| 747 | tempGeoDB = None
|
|---|
| 748 |
|
|---|
| 749 | try:
|
|---|
| 750 | if outputIsShapefile:
|
|---|
| 751 | tempFC = os.path.join(tempDir.Path, u'data.shp')
|
|---|
| 752 | else:
|
|---|
| 753 | tempFC = os.path.join(tempGeoDB, u'data')
|
|---|
| 754 |
|
|---|
| 755 | # Create the output feature class in the temp
|
|---|
| 756 | # location.
|
|---|
| 757 |
|
|---|
| 758 | cls._CreateArcGISTableFromXMLResponses(recordXMLDocFiles,
|
|---|
| 759 | 'GetPointsFromSearchResponse.xslt',
|
|---|
| 760 | tempFC,
|
|---|
| 761 | 'POINT',
|
|---|
| 762 | ['ResourceCode'] + dataElementNames,
|
|---|
| 763 | ['TEXT'] + dataElementArcGISTypes,
|
|---|
| 764 | _(u'Creating an empty point feature class and adding %(count)i fields.' % {u'count': len(dataElementNames) + 1}),
|
|---|
| 765 | True,
|
|---|
| 766 | xsltParameters=map(lambda r: {'digirResourceCode': "'" + r + "'"}, resourceForXMLDoc))
|
|---|
| 767 |
|
|---|
| 768 | # Extract the temporary tables to the caller's
|
|---|
| 769 | # destinations.
|
|---|
| 770 |
|
|---|
| 771 | oldLogInfoAsDebug = Logger.GetLogInfoAsDebug()
|
|---|
| 772 | Logger.SetLogInfoAsDebug(True)
|
|---|
| 773 | try:
|
|---|
| 774 | GeoprocessorManager.CopyArcGISObject(tempFC, pointFeatureClass, overwriteExisting, [u'ShapeFile', u'FeatureClass'], _(u'feature class'))
|
|---|
| 775 | finally:
|
|---|
| 776 | Logger.SetLogInfoAsDebug(oldLogInfoAsDebug)
|
|---|
| 777 |
|
|---|
| 778 | finally:
|
|---|
| 779 | # If we created a temporary personal geodatabase,
|
|---|
| 780 | # delete it now. If we do not explicitly delete it
|
|---|
| 781 | # here, the geoprocessor will keep a handle open
|
|---|
| 782 | # through MS Access APIs, and the temporary directory
|
|---|
| 783 | # will not be able to be deleted.
|
|---|
| 784 |
|
|---|
| 785 | if tempGeoDB is not None:
|
|---|
| 786 | try:
|
|---|
| 787 | gp.Delete_management(tempGeoDB)
|
|---|
| 788 | except:
|
|---|
| 789 | pass
|
|---|
| 790 |
|
|---|
| 791 | except:
|
|---|
| 792 | Logger.LogExceptionAsError()
|
|---|
| 793 | raise
|
|---|
| 794 |
|
|---|
| 795 | @classmethod
|
|---|
| 796 | def _ValidateCoordinateName(cls, dataElementName, elementsRoot, url, displayName):
|
|---|
| 797 | elements = elementsRoot.xpath('/rows/row[name/text()="%s"]' % dataElementName)
|
|---|
| 798 | if len(elements) <= 0:
|
|---|
| 799 | raise ValueError(_(u'Cannot use the data element named "%(name)s" as the %(coord)s coordinate because the conceptual schemas used by the resources offered by the DiGIR server %(url)s do not contain a data element with this name. For a list of valid names, use the Get DiGIR Resources tool to create a table of data elements available from the server. Note that names are case sensitive.') % {u'name': dataElementName, u'coord': displayName, u'url': url})
|
|---|
| 800 | element = elements[0]
|
|---|
| 801 | if element.findtext('returnable') != '1':
|
|---|
| 802 | raise ValueError(_(u'Cannot use the DiGIR data element named "%(name)s" as the %(coord)s coordinate because it is not marked as being "returnable" by the conceptual schema %(namespace)s (located at %(location)s). Please specify a returnable data element.') % {u'name': dataElementName, u'coord': displayName, u'namespace': element.findtext('namespace'), u'location': element.findtext('location')})
|
|---|
| 803 | if element.findtext('arcGISType') not in ['SHORT', 'LONG', 'FLOAT', 'DOUBLE']:
|
|---|
| 804 | raise ValueError(_(u'Cannot use the DiGIR data element named "%(name)s" as the %(coord)s coordinate because it is not defined as having a numeric data type by the conceptual schema %(namespace)s (located at %(location)s). Please specify a data element with an appropriate data type (i.e. ArcGIS type SHORT, LONG, FLOAT, or DOUBLE).') % {u'name': dataElementName, u'coord': displayName, u'namespace': element.findtext('namespace'), u'location': element.findtext('location')})
|
|---|
| 805 | return element.findtext('namespace')
|
|---|
| 806 |
|
|---|
| 807 | @classmethod
|
|---|
| 808 | def _CreateFilterFromExpression(cls, filterExpression, typeForName, namespaceForName, namespaceForPrefix, xCoordinateName, yCoordinateName):
|
|---|
| 809 |
|
|---|
| 810 | # If the caller provided coordinate names, it means that he
|
|---|
| 811 | # intends to save the returned records as points. For that to
|
|---|
| 812 | # work, all of the returned records must contain coordinates.
|
|---|
| 813 | # To ensure this, we append to the filter comparisons against
|
|---|
| 814 | # the coordinates that will always be true, except if the
|
|---|
| 815 | # coordinates are NULL in the server's database. This works
|
|---|
| 816 | # because, in SQL and most other database systems, comparing
|
|---|
| 817 | # anything to NULL is always false.
|
|---|
| 818 |
|
|---|
| 819 | from lxml import etree
|
|---|
| 820 |
|
|---|
| 821 | if xCoordinateName is not None and yCoordinateName is not None:
|
|---|
| 822 | coordinatesElement = etree.Element('{http://digir.net/schema/protocol/2003/1.0}and')
|
|---|
| 823 | element = etree.SubElement(coordinatesElement, '{http://digir.net/schema/protocol/2003/1.0}greaterThanOrEquals')
|
|---|
| 824 | element = etree.SubElement(element, '{' + namespaceForName[xCoordinateName] + '}' + xCoordinateName)
|
|---|
| 825 | element.text = '-999999999999.0'
|
|---|
| 826 | element = etree.SubElement(coordinatesElement, '{http://digir.net/schema/protocol/2003/1.0}greaterThanOrEquals')
|
|---|
| 827 | element = etree.SubElement(element, '{' + namespaceForName[yCoordinateName] + '}' + yCoordinateName)
|
|---|
| 828 | element.text = '-999999999999.0'
|
|---|
| 829 |
|
|---|
| 830 | # If the caller did not provide a filter expression, just
|
|---|
| 831 | # return a <filter> with the coordinate comparisions.
|
|---|
| 832 |
|
|---|
| 833 | if filterExpression is None:
|
|---|
| 834 | filterElement = etree.Element('{http://digir.net/schema/protocol/2003/1.0}filter', nsmap=namespaceForPrefix)
|
|---|
| 835 | filterElement.append(coordinatesElement)
|
|---|
| 836 | return filterElement
|
|---|
| 837 |
|
|---|
| 838 | # Otherwise, turn the caller's expression into a tree of DiGIR
|
|---|
| 839 | # XML, attach the coordinate comparisions, and return.
|
|---|
| 840 | #
|
|---|
| 841 | # First: build the parser using the pyparsing module.
|
|---|
| 842 |
|
|---|
| 843 | from pyparsing import CaselessLiteral, Word, delimitedList, Optional, Combine, Group, alphas, nums, alphanums, ParseException, Forward, oneOf, quotedString, ZeroOrMore, Keyword, Regex
|
|---|
| 844 |
|
|---|
| 845 | filterExpr = Forward()
|
|---|
| 846 | and_ = Keyword('and', caseless=True)
|
|---|
| 847 | or_ = Keyword('or', caseless=True)
|
|---|
| 848 | in_ = Keyword('in', caseless=True)
|
|---|
| 849 |
|
|---|
| 850 | columnName = Word(alphas, alphanums).setName('column')
|
|---|
| 851 | binop = oneOf('= != <> < > >= <= like', caseless=True)
|
|---|
| 852 | arithSign = Word('+-', exact=1)
|
|---|
| 853 | realNum = Combine(Optional(arithSign) + (Word(nums) + '.' + Optional(Word(nums)) | ('.' + Word(nums))) + Optional(CaselessLiteral('E') + Optional(arithSign) + Word(nums)))
|
|---|
| 854 | intNum = Combine(Optional(arithSign) + Word(nums))
|
|---|
| 855 | dateLiteral = Combine('#' + (Regex('[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}') | Regex('[0-9]{4}-[0-9]{2}-[0-9]{2}')) + '#')
|
|---|
| 856 | columnRval = realNum | intNum | quotedString | dateLiteral
|
|---|
| 857 |
|
|---|
| 858 | filterCond = Group((columnName + binop + columnRval) | (columnName + in_ + "(" + delimitedList(columnRval) + ")") | ('(' + filterExpr + ')'))
|
|---|
| 859 | filterExpr << filterCond + ZeroOrMore((and_ | or_) + filterExpr)
|
|---|
| 860 |
|
|---|
| 861 | # Parse the caller's expression and turn it into DiGIR XML.
|
|---|
| 862 |
|
|---|
| 863 | try:
|
|---|
| 864 | tokens = filterExpr.parseString(filterExpression, parseAll=True)
|
|---|
| 865 | except pyparsing.ParseException, e:
|
|---|
| 866 | Logger.RaiseException(ValueError(_(u'The filter expression could not be parsed. Please review the documentation for this parameter and try again. The filter expression syntax is similar to a SQL where clause or ArcGIS "SQL expression" but does not support all of the constructs found in those syntaxes. Error details: %(e)s: %(msg)s') % {u'e': e.__class__.__name__, u'msg': unicode(e)}))
|
|---|
| 867 |
|
|---|
| 868 | filterExpressionElement = cls._CreateXMLFromTokens(tokens, typeForName, namespaceForName)
|
|---|
| 869 |
|
|---|
| 870 | # Create and return a <filter> that combines the XML for the
|
|---|
| 871 | # caller's expression with the coordinate comparisions.
|
|---|
| 872 |
|
|---|
| 873 | andElement = etree.Element('{http://digir.net/schema/protocol/2003/1.0}and')
|
|---|
| 874 | andElement.append(filterExpressionElement)
|
|---|
| 875 | andElement.append(coordinatesElement)
|
|---|
| 876 |
|
|---|
| 877 | filterElement = etree.Element('{http://digir.net/schema/protocol/2003/1.0}filter', nsmap=namespaceForPrefix)
|
|---|
| 878 | filterElement.append(andElement)
|
|---|
| 879 | return filterElement
|
|---|
| 880 |
|
|---|
| 881 | @classmethod
|
|---|
| 882 | def _CreateXMLFromTokens(cls, tokens, typeForName, namespaceForName):
|
|---|
| 883 |
|
|---|
| 884 | from lxml import etree
|
|---|
| 885 |
|
|---|
| 886 | if isinstance(tokens[0], basestring):
|
|---|
| 887 | if tokens[0] == '(':
|
|---|
| 888 | return cls._CreateXMLFromTokens(tokens[1:-1], typeForName, namespaceForName)
|
|---|
| 889 |
|
|---|
| 890 | else:
|
|---|
| 891 | name = tokens[0]
|
|---|
| 892 | if name not in typeForName.keys():
|
|---|
| 893 | Logger.RaiseException(ValueError(_('Syntax error in the filter expression: "%(name)s" is not the name of a searchable DiGIR data element in the conceptual schemas used by the resources available on this DiGIR server. For a list of valid names, use the Get DiGIR Resources tool to create a table of data elements available from the server. Note that names are case sensitive.') % {u'name': name}))
|
|---|
| 894 |
|
|---|
| 895 | comparison = tokens[1].lower()
|
|---|
| 896 | if comparison == 'like' and typeForName[name] != 'TEXT':
|
|---|
| 897 | Logger.RaiseException(ValueError(_('Syntax error in the filter expression: The LIKE operator cannot be applied to the DiGIR data element %(name)s because that data element is not a string data type.') % {u'name': name}))
|
|---|
| 898 |
|
|---|
| 899 | if comparison != 'in':
|
|---|
| 900 | value = cls._ValidateFilterExpressionValue(name, typeForName[name], tokens[2])
|
|---|
| 901 | if comparison == '=':
|
|---|
| 902 | comparisonElement = etree.Element('{http://digir.net/schema/protocol/2003/1.0}equals')
|
|---|
| 903 | elif comparison in ['!=', '<>']:
|
|---|
| 904 | comparisonElement = etree.Element('{http://digir.net/schema/protocol/2003/1.0}notEquals')
|
|---|
| 905 | elif comparison == '<':
|
|---|
| 906 | comparisonElement = etree.Element('{http://digir.net/schema/protocol/2003/1.0}lessThan')
|
|---|
| 907 | elif comparison == '>':
|
|---|
| 908 | comparisonElement = etree.Element('{http://digir.net/schema/protocol/2003/1.0}greaterThan')
|
|---|
| 909 | elif comparison == '<=':
|
|---|
| 910 | comparisonElement = etree.Element('{http://digir.net/schema/protocol/2003/1.0}lessThanOrEquals')
|
|---|
| 911 | elif comparison == '>=':
|
|---|
| 912 | comparisonElement = etree.Element('{http://digir.net/schema/protocol/2003/1.0}greaterThanOrEquals')
|
|---|
| 913 | elif comparison == 'like':
|
|---|
| 914 | comparisonElement = etree.Element('{http://digir.net/schema/protocol/2003/1.0}like')
|
|---|
| 915 | else:
|
|---|
| 916 | Logger.RaiseException(RuntimeError(_(u'Programming error in this tool: %(op)s is not a recognized comparison operator. Please contact the author of this tool for assistance.') % {u'op': comparison}))
|
|---|
| 917 | subElement = etree.SubElement(comparisonElement, '{' + namespaceForName[name] + '}' + name)
|
|---|
| 918 | subElement.text = value
|
|---|
| 919 | return comparisonElement
|
|---|
| 920 |
|
|---|
| 921 | else:
|
|---|
| 922 | values = []
|
|---|
| 923 | for i in range(3, len(tokens) - 1):
|
|---|
| 924 | values.append(cls._ValidateFilterExpressionValue(name, typeForName[name], tokens[i]))
|
|---|
| 925 | comparisonElement = etree.Element('{http://digir.net/schema/protocol/2003/1.0}in')
|
|---|
| 926 | for value in values:
|
|---|
| 927 | subElement = etree.SubElement(comparisonElement, '{' + namespaceForName[name] + '}' + name)
|
|---|
| 928 | subElement.text = value
|
|---|
| 929 | return comparisonElement
|
|---|
| 930 |
|
|---|
| 931 | else:
|
|---|
| 932 | pass1 = []
|
|---|
| 933 | for i in range(len(tokens)):
|
|---|
| 934 | if i % 2 == 0:
|
|---|
| 935 | pass1.append(cls._CreateXMLFromTokens(tokens[i], typeForName, namespaceForName))
|
|---|
| 936 | else:
|
|---|
| 937 | pass1.append(tokens[i].lower()) # Guaranteed to be either 'and' or 'or'
|
|---|
| 938 |
|
|---|
| 939 | pass2 = []
|
|---|
| 940 | andElement = pass1[0]
|
|---|
| 941 | i = 1
|
|---|
| 942 | while True:
|
|---|
| 943 | if i >= len(pass1):
|
|---|
| 944 | pass2.append(andElement)
|
|---|
| 945 | break
|
|---|
| 946 | if pass1[i] == 'or':
|
|---|
| 947 | pass2.append(andElement)
|
|---|
| 948 | andElement = pass1[i+1]
|
|---|
| 949 | else:
|
|---|
| 950 | newAndElement = etree.Element('{http://digir.net/schema/protocol/2003/1.0}and')
|
|---|
| 951 | newAndElement.append(pass1[i+1])
|
|---|
| 952 | newAndElement.append(andElement)
|
|---|
| 953 | andElement = newAndElement
|
|---|
| 954 | i += 2
|
|---|
| 955 |
|
|---|
| 956 | orElement = pass2[0]
|
|---|
| 957 | for i in range(1, len(pass2)):
|
|---|
| 958 | newOrElement = etree.Element('{http://digir.net/schema/protocol/2003/1.0}or')
|
|---|
| 959 | newOrElement.append(pass2[i])
|
|---|
| 960 | newOrElement.append(orElement)
|
|---|
| 961 | orElement = newOrElement
|
|---|
| 962 |
|
|---|
| 963 | return orElement
|
|---|
| 964 |
|
|---|
| 965 | @classmethod
|
|---|
| 966 | def _ValidateFilterExpressionValue(cls, name, type_, value):
|
|---|
| 967 | if value[0] == '#':
|
|---|
| 968 | value = value[1:-1]
|
|---|
| 969 | if len(value) == 10:
|
|---|
| 970 | value += 'T00:00:00'
|
|---|
| 971 | else:
|
|---|
| 972 | value = value[:10] + 'T' + value[11:]
|
|---|
| 973 | value += 'Z'
|
|---|
| 974 | if type_ != 'DATE':
|
|---|
| 975 | Logger.RaiseException(ValueError(_('Syntax error in the filter expression: The DiGIR data element %(name)s cannot be compared to the date %(value)s because that data element is not a date/time data type.') % {u'name': name, u'value': value}))
|
|---|
| 976 | elif value[0] in ["'", '"']:
|
|---|
| 977 | if type_ != 'TEXT':
|
|---|
| 978 | Logger.RaiseException(ValueError(_('Syntax error in the filter expression: The DiGIR data element %(name)s cannot be compared to the string %(value)s because that data element is not a string data type.') % {u'name': name, u'value': value}))
|
|---|
| 979 | value = value[1:-1]
|
|---|
| 980 | else:
|
|---|
| 981 | if type_ not in ['SHORT', 'LONG', 'FLOAT', 'DOUBLE']:
|
|---|
| 982 | Logger.RaiseException(ValueError(_('Syntax error in the filter expression: The DiGIR data element %(name)s cannot be compared to the number %(value)s because that data element is not a numeric data type.') % {u'name': name, u'value': value}))
|
|---|
| 983 | return value
|
|---|
| 984 |
|
|---|
| 985 | @classmethod
|
|---|
| 986 | def _CreateArcGISTableFromXMLResponses(cls, xmlResponseDocs, xsltFile, table, geometryType, fieldNames, fieldDataTypes, creationMessage, logAddFields=False, xsltParameters=None):
|
|---|
| 987 | if isinstance(geometryType, basestring):
|
|---|
| 988 | geometryType = geometryType.upper()
|
|---|
| 989 |
|
|---|
| 990 | # Determine the maximum length required to hold all string
|
|---|
| 991 | # fields, so we can create the fields with a sufficient
|
|---|
| 992 | # length. Also count the number of rows.
|
|---|
| 993 |
|
|---|
| 994 | Logger.Debug(_(u'Calculating maximum lengths of string fields.'))
|
|---|
| 995 |
|
|---|
| 996 | from lxml import etree
|
|---|
| 997 |
|
|---|
| 998 | maxLengths = [None] * len(fieldDataTypes)
|
|---|
| 999 | for i in range(len(fieldDataTypes)):
|
|---|
| 1000 | if fieldDataTypes[i] == 'TEXT':
|
|---|
| 1001 | maxLengths[i] = 1
|
|---|
| 1002 |
|
|---|
| 1003 | xmlResponseDocsArePaths = len(xmlResponseDocs) > 0 and isinstance(xmlResponseDocs[0], basestring) and os.path.isfile(xmlResponseDocs[0])
|
|---|
| 1004 | rowCount = 0
|
|---|
| 1005 | for doc in xmlResponseDocs:
|
|---|
| 1006 | if xmlResponseDocsArePaths:
|
|---|
| 1007 | Logger.Debug(_(u'Reading and parsing XML from temporary file %(name)s.') % {u'name': doc})
|
|---|
| 1008 | f = open(doc)
|
|---|
| 1009 | try:
|
|---|
| 1010 | doc = etree.parse(f)
|
|---|
| 1011 | finally:
|
|---|
| 1012 | try:
|
|---|
| 1013 | f.close()
|
|---|
| 1014 | except:
|
|---|
| 1015 | pass
|
|---|
| 1016 |
|
|---|
| 1017 | rows = cls._TransformXMLDocs([doc], xsltFile, xsltParameters)[0]
|
|---|
| 1018 |
|
|---|
| 1019 | rowCount += len(rows)
|
|---|
| 1020 | for row in rows:
|
|---|
| 1021 | for i in range(len(fieldDataTypes)):
|
|---|
| 1022 | if fieldDataTypes[i] == 'TEXT' and row[i].text is not None and len(row[i].text) > maxLengths[i]:
|
|---|
| 1023 | maxLengths[i] = len(row[i].text)
|
|---|
| 1024 |
|
|---|
| 1025 | # Create the table.
|
|---|
| 1026 |
|
|---|
| 1027 | Logger.Info(creationMessage)
|
|---|
| 1028 |
|
|---|
| 1029 | from GeoEco.DatabaseAccess.ArcGIS import ArcGIS91DatabaseConnection
|
|---|
| 1030 | conn = ArcGIS91DatabaseConnection()
|
|---|
| 1031 |
|
|---|
| 1032 | if geometryType is None:
|
|---|
| 1033 | table = conn.CreateTable(table)
|
|---|
| 1034 | else:
|
|---|
| 1035 | from GeoEco.ArcGIS import GeoprocessorManager
|
|---|
| 1036 | gp = GeoprocessorManager.GetWrappedGeoprocessor()
|
|---|
| 1037 | table = gp.CreateFeatureClass_management(os.path.dirname(table), os.path.basename(table), geometryType, None, None, None, u"GEOGCS['GCS_WGS_1984',DATUM['D_WGS_1984',SPHEROID['WGS_1984',6378137.0,298.257223563]],PRIMEM['Greenwich',0.0],UNIT['Degree',0.0174532925199433]]")
|
|---|
| 1038 | shapeFieldName = gp.Describe(table).ShapeFieldName
|
|---|
| 1039 |
|
|---|
| 1040 | isDBFTable = os.path.isfile(table) and (table.lower().endswith('.dbf') or table.lower().endswith('.shp'))
|
|---|
| 1041 |
|
|---|
| 1042 | if logAddFields:
|
|---|
| 1043 | progressReporter = ProgressReporter(progressMessage1=_(u'Still adding fields: %(elapsed)s elapsed, %(opsCompleted)i fields added, %(perOp)s per field, %(opsRemaining)i remaining, estimated completion time: %(etc)s.'),
|
|---|
| 1044 | completionMessage=_(u'Finished adding fields: %(elapsed)s elapsed, %(opsCompleted)i fields added, %(perOp)s per field.'))
|
|---|
| 1045 | progressReporter.Start(len(fieldNames))
|
|---|
| 1046 |
|
|---|
| 1047 | for i in range(len(fieldNames)):
|
|---|
| 1048 | fieldNames[i] = conn._AddField(table, fieldNames[i], fieldDataTypes[i], length=maxLengths[i]) # Call _AddField directly instead of AddField, to skip the expense of checking that the table and field do not exist every time.
|
|---|
| 1049 | if isDBFTable and fieldDataTypes[i] and maxLengths[i] > 254:
|
|---|
| 1050 | Logger.Warning(_(u'Some values of the %(field)s field are longer than the DBase limit of 254 characters and will be truncated (the longest value is %(long)i characters). You can avoid this problem by storing the output table or feature class in a geodatabase.') % {u'field': fieldNames[i], u'long': maxLengths[i]})
|
|---|
| 1051 | if logAddFields:
|
|---|
| 1052 | progressReporter.ReportProgress()
|
|---|
| 1053 |
|
|---|
| 1054 | if conn.FieldExists(table, u'Field1'): # ArcGIS may create this bogus field in DBF files that are not associated with a shapefile.
|
|---|
| 1055 | conn.DeleteField(table, u'Field1')
|
|---|
| 1056 | if conn.FieldExists(table, u'Id'): # ArcGIS may create this bogus field in shapefiles.
|
|---|
| 1057 | conn.DeleteField(table, u'Id')
|
|---|
| 1058 |
|
|---|
| 1059 | # Insert the rows.
|
|---|
| 1060 |
|
|---|
| 1061 | Logger.Info(_(u'Inserting %(rows)i records.') % {u'rows': rowCount})
|
|---|
| 1062 |
|
|---|
| 1063 | if rowCount > 0:
|
|---|
| 1064 | import codecs, locale
|
|---|
| 1065 | codec = codecs.getencoder(locale.getpreferredencoding())
|
|---|
| 1066 |
|
|---|
| 1067 | cursor = conn.OpenInsertCursor(table, rowCount=rowCount)
|
|---|
| 1068 | rowNum = 0
|
|---|
| 1069 | try:
|
|---|
| 1070 | for doc in xmlResponseDocs:
|
|---|
| 1071 | if xmlResponseDocsArePaths:
|
|---|
| 1072 | Logger.Debug(_(u'Reading and parsing XML from temporary file %(name)s.') % {u'name': doc})
|
|---|
| 1073 | f = open(doc)
|
|---|
| 1074 | try:
|
|---|
| 1075 | doc = etree.parse(f)
|
|---|
| 1076 | finally:
|
|---|
| 1077 | try:
|
|---|
| 1078 | f.close()
|
|---|
| 1079 | except:
|
|---|
| 1080 | pass
|
|---|
| 1081 |
|
|---|
| 1082 | rows = cls._TransformXMLDocs([doc], xsltFile, xsltParameters)[0] # It is expensive to transform twice, but probably cheaper than caching the transformed doc on disk. We can't cache in memory or we might run out of memory.
|
|---|
| 1083 |
|
|---|
| 1084 | for row in rows:
|
|---|
| 1085 | rowNum += 1
|
|---|
| 1086 | for i in range(len(fieldNames)):
|
|---|
| 1087 |
|
|---|
| 1088 | if fieldDataTypes[i] == 'TEXT':
|
|---|
| 1089 | if row[i].text is None or len(row[i].text) <= 0:
|
|---|
| 1090 | if isDBFTable:
|
|---|
| 1091 | cursor.SetValue(fieldNames[i], '')
|
|---|
| 1092 | else:
|
|---|
| 1093 | cursor.SetValue(fieldNames[i], None)
|
|---|
| 1094 | else:
|
|---|
| 1095 | cursor.SetValue(fieldNames[i], codec(row[i].text, 'replace')[0])
|
|---|
| 1096 |
|
|---|
| 1097 | if fieldDataTypes[i] == 'SHORT' or fieldDataTypes[i] == 'LONG':
|
|---|
| 1098 | value = None
|
|---|
| 1099 | if row[i].text is not None and len(row[i].text) > 0:
|
|---|
| 1100 | try:
|
|---|
| 1101 | value = int(row[i].text)
|
|---|
| 1102 | except:
|
|---|
| 1103 | Logger.Warning(_(u'Failed to parse an integer from the string "%(s)s". This is a schematic error in the data returned by the DiGIR server; to fix it, please contact the server administrator. As a result, the %(field)s field of row %(row)i will be set to NULL (or -999 if the output table is a shapefile or DBase table).') % {u's': row[i].text, u'field': fieldNames[i], u'row': rowNum})
|
|---|
| 1104 | if value is None and isDBFTable:
|
|---|
| 1105 | value = -999
|
|---|
| 1106 | cursor.SetValue(fieldNames[i], value)
|
|---|
| 1107 |
|
|---|
| 1108 | if fieldDataTypes[i] == 'FLOAT' or fieldDataTypes[i] == 'DOUBLE':
|
|---|
| 1109 | value = None
|
|---|
| 1110 | if row[i].text is not None and len(row[i].text) > 0:
|
|---|
| 1111 | try:
|
|---|
| 1112 | value = float(row[i].text)
|
|---|
| 1113 | except:
|
|---|
| 1114 | Logger.Warning(_(u'Failed to parse a floating-point number from the string "%(s)s". This is a schematic error in the data returned by the DiGIR server; to fix it, please contact the server administrator. As a result, the %(field)s field of row %(row)i will be set to NULL (or -999.0 if the output table is a shapefile or DBase table).') % {u's': row[i].text, u'field': fieldNames[i], u'row': rowNum})
|
|---|
| 1115 | if value is None and isDBFTable:
|
|---|
| 1116 | value = -999.0
|
|---|
| 1117 | cursor.SetValue(fieldNames[i], value)
|
|---|
| 1118 |
|
|---|
| 1119 | if fieldDataTypes[i] == 'DATE':
|
|---|
| 1120 | value = None
|
|---|
| 1121 | if row[i].text is not None and len(row[i].text) > 0:
|
|---|
| 1122 | try:
|
|---|
| 1123 | value = cls._ParseDiGIRDate(row[i].text)
|
|---|
| 1124 | except ValueError, e:
|
|---|
| 1125 | Logger.Warning(_(u'Failed to parse a date from the string "%(s)s". The string may contain a date in an unknown format. This is a schematic error in the data returned by the DiGIR server; to fix it, please contact the server administrator. As a result, the %(field)s field of row %(row)i will be set to NULL.') % {u's': row[i].text, u'field': fieldNames[i], u'row': rowNum})
|
|---|
| 1126 | cursor.SetValue(fieldNames[i], value)
|
|---|
| 1127 |
|
|---|
| 1128 | if geometryType == 'POINT':
|
|---|
| 1129 | point = gp.CreateObject('Point')
|
|---|
| 1130 | point.X = float(row[i+1].text)
|
|---|
| 1131 | point.Y = float(row[i+2].text)
|
|---|
| 1132 | cursor.SetValue(shapeFieldName, point)
|
|---|
| 1133 |
|
|---|
| 1134 | cursor.InsertRow()
|
|---|
| 1135 | finally:
|
|---|
| 1136 | del cursor
|
|---|
| 1137 |
|
|---|
| 1138 | _MonthMap = {'jan': 1, 'feb': 2, 'mar': 3, 'apr': 4, 'may': 5, 'jun': 6, 'jul': 7, 'aug': 8, 'sep': 9, 'oct': 10, 'nov': 11, 'dec': 12}
|
|---|
| 1139 |
|
|---|
| 1140 | @classmethod
|
|---|
| 1141 | def _ParseDiGIRDate(cls, s):
|
|---|
| 1142 | if s is None or s == 'None':
|
|---|
| 1143 | return None
|
|---|
| 1144 |
|
|---|
| 1145 | m = re.match('(\d{4})-(\d{2})-(\d{2})[\sT](\d{2}):(\d{2}):(\d{2}).*', s)
|
|---|
| 1146 | if m is not None:
|
|---|
| 1147 | return datetime.datetime(int(m.group(1)), int(m.group(2)), int(m.group(3)), int(m.group(4)), int(m.group(5)), int(m.group(6)))
|
|---|
| 1148 |
|
|---|
| 1149 | m = re.match('(\d{2})-(\d{2})-(\d{4})[\sT](\d{2}):(\d{2}):(\d{2}).*', s)
|
|---|
| 1150 | if m is not None:
|
|---|
| 1151 | return datetime.datetime(int(m.group(3)), int(m.group(2)), int(m.group(1)), int(m.group(4)), int(m.group(5)), int(m.group(6)))
|
|---|
| 1152 |
|
|---|
| 1153 | m = re.match('(\d{2})-([a-zA-Z]{3})-(\d{2})', s)
|
|---|
| 1154 | if m is not None:
|
|---|
| 1155 | month = m.group(2).lower()
|
|---|
| 1156 | if not DiGIR._MonthMap.has_key(month):
|
|---|
| 1157 | return None
|
|---|
| 1158 | return datetime.datetime(int(m.group(3)), DiGIR._MonthMap[month], int(m.group(1)))
|
|---|
| 1159 |
|
|---|
| 1160 | m = re.match('(\d{4})-(\d{2})-(\d{2})', s)
|
|---|
| 1161 | if m is not None:
|
|---|
| 1162 | return datetime.datetime(int(m.group(1)), int(m.group(2)), int(m.group(3)))
|
|---|
| 1163 |
|
|---|
| 1164 | raise ValueError('Unknown date format.')
|
|---|
| 1165 |
|
|---|
| 1166 | @classmethod
|
|---|
| 1167 | def _GetDataElementsFromSchemas(cls, namespaces, locations):
|
|---|
| 1168 |
|
|---|
| 1169 | # Create an empty XML document to hold the <xsd:element>
|
|---|
| 1170 | # elements.
|
|---|
| 1171 |
|
|---|
| 1172 | from lxml import etree
|
|---|
| 1173 | elementsRoot = etree.Element('Elements')
|
|---|
| 1174 |
|
|---|
| 1175 | namespacesToProcess = copy.deepcopy(namespaces)
|
|---|
| 1176 | locationsToProcess = copy.deepcopy(locations)
|
|---|
| 1177 | processedNamespaces = {}
|
|---|
| 1178 | processedLocations = {}
|
|---|
| 1179 | processedElementNames = {}
|
|---|
| 1180 |
|
|---|
| 1181 | while len(namespacesToProcess) > 0:
|
|---|
| 1182 | namespace = namespacesToProcess.pop(0)
|
|---|
| 1183 | location = locationsToProcess.pop(0)
|
|---|
| 1184 |
|
|---|
| 1185 | # If the namespace or location is bogus, skip it. (For
|
|---|
| 1186 | # example, OBIS has some resources with the namespace
|
|---|
| 1187 | # "OBIS Schema Version 1.0").
|
|---|
| 1188 |
|
|---|
| 1189 | if not isinstance(namespace, basestring) or not isinstance(location, basestring) or not namespace.startswith('http:') or not location.startswith('http:'):
|
|---|
| 1190 | continue
|
|---|
| 1191 |
|
|---|
| 1192 | # If we have already processed this namespace, skip it. We
|
|---|
| 1193 | # assume that all namespaces resolve to equivalent XSDs.
|
|---|
| 1194 |
|
|---|
| 1195 | if processedNamespaces.has_key(namespace):
|
|---|
| 1196 | continue
|
|---|
| 1197 |
|
|---|
| 1198 | # Download and parse the XSD.
|
|---|
| 1199 |
|
|---|
| 1200 | Logger.Debug(_('Downloading and parsing XML schema %(url)s') % {u'url': location})
|
|---|
| 1201 |
|
|---|
| 1202 | try:
|
|---|
| 1203 | f = urllib2.urlopen(location)
|
|---|
| 1204 | except Exception, e:
|
|---|
| 1205 | raise RuntimeError(_(u'Failed to download XML schema %(url)s due to %(e)s: %(msg)s') % {u'url': location, u'e': e.__class__.__name__, u'msg': unicode(e)})
|
|---|
| 1206 | try:
|
|---|
| 1207 | schemaRoot = etree.parse(f).getroot()
|
|---|
| 1208 | finally:
|
|---|
| 1209 | try:
|
|---|
| 1210 | f.close()
|
|---|
| 1211 | except:
|
|---|
| 1212 | pass
|
|---|
| 1213 |
|
|---|
| 1214 | # Add <xsd:element> elements that have unique names to our
|
|---|
| 1215 | # XML document.
|
|---|
| 1216 |
|
|---|
| 1217 | for e in schemaRoot.xpath('/xsd:schema/xsd:element[@substitutionGroup="digir:searchableData" or @substitutionGroup="digir:returnableData" or @substitutionGroup="digir:searchableReturnableData"]', namespaces={'xsd': 'http://www.w3.org/2001/XMLSchema'}):
|
|---|
| 1218 | if not processedElementNames.has_key(e.get('name')):
|
|---|
| 1219 | processedElementNames[e.get('name')] = None
|
|---|
| 1220 | newElement = copy.deepcopy(e)
|
|---|
| 1221 | newElement.set('originalNamespace', namespace)
|
|---|
| 1222 | newElement.set('originalLocation', location)
|
|---|
| 1223 | elementsRoot.append(newElement)
|
|---|
| 1224 |
|
|---|
| 1225 | # Add any imported schemas to our lists to process.
|
|---|
| 1226 |
|
|---|
| 1227 | for e in schemaRoot.xpath('/xsd:schema/xsd:import[@namespace and @schemaLocation]', namespaces={'xsd': 'http://www.w3.org/2001/XMLSchema'}):
|
|---|
| 1228 | namespacesToProcess.append(e.get('namespace'))
|
|---|
| 1229 | locationsToProcess.append(e.get('schemaLocation'))
|
|---|
| 1230 |
|
|---|
| 1231 | processedNamespaces[namespace] = None
|
|---|
| 1232 | processedLocations[location] = None
|
|---|
| 1233 |
|
|---|
| 1234 | # Return the XML document and a list of processed namespaces.
|
|---|
| 1235 |
|
|---|
| 1236 | return elementsRoot, processedNamespaces.keys()
|
|---|
| 1237 |
|
|---|
| 1238 | _XSLTCache = {}
|
|---|
| 1239 |
|
|---|
| 1240 | @classmethod
|
|---|
| 1241 | def _TransformXMLDocs(cls, xmlDocs, xsltFile, xsltParameters=None):
|
|---|
| 1242 |
|
|---|
| 1243 | # Load the XSL transform.
|
|---|
| 1244 |
|
|---|
| 1245 | from lxml import etree
|
|---|
| 1246 |
|
|---|
| 1247 | if DiGIR._XSLTCache.has_key(xsltFile):
|
|---|
| 1248 | transform = DiGIR._XSLTCache[xsltFile]
|
|---|
| 1249 | else:
|
|---|
| 1250 | if not os.path.isfile(xsltFile):
|
|---|
| 1251 | xsltFile = os.path.join(os.path.dirname(sys.modules[DiGIR.__module__].__file__), xsltFile)
|
|---|
| 1252 | f = open(xsltFile)
|
|---|
| 1253 | try:
|
|---|
| 1254 | transform = etree.XSLT(etree.parse(f))
|
|---|
| 1255 | finally:
|
|---|
| 1256 | try:
|
|---|
| 1257 | f.close()
|
|---|
| 1258 | except:
|
|---|
| 1259 | pass
|
|---|
| 1260 | DiGIR._XSLTCache[xsltFile] = transform
|
|---|
| 1261 |
|
|---|
| 1262 | # Transform the XML documents.
|
|---|
| 1263 |
|
|---|
| 1264 | transformedDocs = []
|
|---|
| 1265 | for i in range(len(xmlDocs)):
|
|---|
| 1266 | if xsltParameters is not None:
|
|---|
| 1267 | transformedDocs.append(transform(xmlDocs[i], **xsltParameters[i]).getroot())
|
|---|
| 1268 | else:
|
|---|
| 1269 | transformedDocs.append(transform(xmlDocs[i]).getroot())
|
|---|
| 1270 |
|
|---|
| 1271 | return transformedDocs
|
|---|
| 1272 |
|
|---|
| 1273 | @classmethod
|
|---|
| 1274 | def GetOBISResourcesAsArcGISTable(cls, resourcesTable, relatedInfoTable=None, contactsTable=None, schemasTable=None, dataElementsTable=None, url=u'http://iobis.rutgers.edu/digir2/DiGIR.php', timeout=120, maxRetryTime=300, overwriteExisting=False):
|
|---|
| 1275 | cls.__doc__.Obj.ValidateMethodInvocation()
|
|---|
| 1276 | cls.GetResourcesAsArcGISTable(url, resourcesTable, relatedInfoTable, contactsTable, schemasTable, dataElementsTable, timeout, maxRetryTime, overwriteExisting)
|
|---|
| 1277 |
|
|---|
| 1278 | @classmethod
|
|---|
| 1279 | def SearchOBISAndCreateArcGISPoints(cls, pointFeatureClass, filterExpression=None, resources=None, dataElementNames=None, maxRecords=None, url=u'http://iobis.rutgers.edu/digir2/DiGIR.php', timeout=120, maxRetryTime=300, overwriteExisting=False):
|
|---|
| 1280 | cls.__doc__.Obj.ValidateMethodInvocation()
|
|---|
| 1281 | cls.SearchAndCreateArcGISPoints(url, pointFeatureClass, filterExpression, resources, dataElementNames, xCoordinateDataElementName=u'Longitude', yCoordinateDataElementName=u'Latitude', maxRecords=None, timeout=timeout, maxRetryTime=maxRetryTime, overwriteExisting=False)
|
|---|
| 1282 |
|
|---|
| 1283 | @classmethod
|
|---|
| 1284 | def GetOBISSEAMAPResourcesAsArcGISTable(cls, resourcesTable, relatedInfoTable=None, contactsTable=None, schemasTable=None, dataElementsTable=None, url=u'http://seamap.env.duke.edu/digir/DiGIR.php', timeout=120, maxRetryTime=300, overwriteExisting=False):
|
|---|
| 1285 | cls.__doc__.Obj.ValidateMethodInvocation()
|
|---|
| 1286 | cls.GetResourcesAsArcGISTable(url, resourcesTable, relatedInfoTable, contactsTable, schemasTable, dataElementsTable, timeout, maxRetryTime, overwriteExisting)
|
|---|
| 1287 |
|
|---|
| 1288 | @classmethod
|
|---|
| 1289 | def SearchOBISSEAMAPAndCreateArcGISPoints(cls, pointFeatureClass, filterExpression=None, resources=None, dataElementNames=None, maxRecords=None, url=u'http://seamap.env.duke.edu/digir/DiGIR.php', timeout=120, maxRetryTime=300, overwriteExisting=False):
|
|---|
| 1290 | cls.__doc__.Obj.ValidateMethodInvocation()
|
|---|
| 1291 | cls.SearchAndCreateArcGISPoints(url, pointFeatureClass, filterExpression, resources, dataElementNames, xCoordinateDataElementName=u'Longitude', yCoordinateDataElementName=u'Latitude', maxRecords=None, timeout=timeout, maxRetryTime=maxRetryTime, overwriteExisting=False)
|
|---|
| 1292 |
|
|---|
| 1293 |
|
|---|
| 1294 | ###############################################################################
|
|---|
| 1295 | # Metadata: module
|
|---|
| 1296 | ###############################################################################
|
|---|
| 1297 |
|
|---|
| 1298 | from GeoEco.ArcGIS import ArcGISDependency
|
|---|
| 1299 | from GeoEco.Dependencies import PythonAggregatedModuleDependency
|
|---|
| 1300 | from GeoEco.Metadata import *
|
|---|
| 1301 | from GeoEco.Types import *
|
|---|
| 1302 |
|
|---|
| 1303 | AddModuleMetadata(shortDescription=_(u'Provides methods for retrieving biogeographic data using the Distributed Generic Information Retrieval (DiGIR) protocol.'))
|
|---|
| 1304 |
|
|---|
| 1305 | ###############################################################################
|
|---|
| 1306 | # Metadata: DiGIR class
|
|---|
| 1307 | ###############################################################################
|
|---|
| 1308 |
|
|---|
| 1309 | AddClassMetadata(DiGIR,
|
|---|
| 1310 | shortDescription=_(u'Provides methods for retrieving biogeographic data using the DiGIR protocol.'),
|
|---|
| 1311 | isExposedAsCOMServer=True,
|
|---|
| 1312 | comIID=u'{8422D2EB-61EC-43E6-93C6-AD5419BACC86}',
|
|---|
| 1313 | comCLSID=u'{8C47BA04-6E9F-44F3-86FC-BCC9B2B379ED}')
|
|---|
| 1314 |
|
|---|
| 1315 | # Public method: DiGIR.SendRequestToServer
|
|---|
| 1316 |
|
|---|
| 1317 | AddMethodMetadata(DiGIR.SendRequestToServer,
|
|---|
| 1318 | shortDescription=_(u'Sends a request to a Distributed Generic Information Retrieval (DiGIR) server and returns the XML response.'),
|
|---|
| 1319 | isExposedToPythonCallers=True,
|
|---|
| 1320 | dependencies=[PythonAggregatedModuleDependency('lxml')])
|
|---|
| 1321 |
|
|---|
| 1322 | AddArgumentMetadata(DiGIR.SendRequestToServer, u'cls',
|
|---|
| 1323 | typeMetadata=ClassOrClassInstanceTypeMetadata(cls=DiGIR),
|
|---|
| 1324 | description=_(u'%s class or an instance of it.') % DiGIR.__name__)
|
|---|
| 1325 |
|
|---|
| 1326 | AddArgumentMetadata(DiGIR.SendRequestToServer, u'url',
|
|---|
| 1327 | typeMetadata=UnicodeStringTypeMetadata(),
|
|---|
| 1328 | description=_(u'URL of the DiGIR server to query.'),
|
|---|
| 1329 | arcGISDisplayName=_(u'DiGIR server URL'))
|
|---|
| 1330 |
|
|---|
| 1331 | AddArgumentMetadata(DiGIR.SendRequestToServer, u'requestXML',
|
|---|
| 1332 | typeMetadata=AnyObjectTypeMetadata(canBeNone=True),
|
|---|
| 1333 | description=_(
|
|---|
| 1334 | u"""XML representing the request to send to the server.
|
|---|
| 1335 |
|
|---|
| 1336 | To issue a DiGIR metadata request, omit this parameter. To send an
|
|---|
| 1337 | inventory or search request, provide a string or Unicode string
|
|---|
| 1338 | containing XML or an instance of lxml.etree._Element. The root element
|
|---|
| 1339 | of the XML must be <inventory> or <search> from the
|
|---|
| 1340 | http://digir.net/schema/protocol/2003/1.0 XML namespace, as defined in
|
|---|
| 1341 | the DiGIR protocol schema, (available at
|
|---|
| 1342 | http://www.digir.net/schema/protocol/2003/1.0/digir.xsd at the time of
|
|---|
| 1343 | this writing)."""))
|
|---|
| 1344 |
|
|---|
| 1345 | AddArgumentMetadata(DiGIR.SendRequestToServer, u'resource',
|
|---|
| 1346 | typeMetadata=UnicodeStringTypeMetadata(canBeNone=True),
|
|---|
| 1347 | description=_(
|
|---|
| 1348 | u"""Name of the resource to query. For metadata requests, a resource
|
|---|
| 1349 | does not need to be provided. For inventory and search requests, most
|
|---|
| 1350 | DiGIR server implementations require the client to scope the request
|
|---|
| 1351 | to a specific resource. If you omit this parameter and from an
|
|---|
| 1352 | inventory or search request and receive an error, try providing
|
|---|
| 1353 | it."""))
|
|---|
| 1354 |
|
|---|
| 1355 | AddArgumentMetadata(DiGIR.SendRequestToServer, u'httpobj',
|
|---|
| 1356 | typeMetadata=ClassInstanceTypeMetadata(Http, canBeNone=True),
|
|---|
| 1357 | description=_(
|
|---|
| 1358 | u"""Instance of GeoEco.httplib2.Http to be used to place HTTP
|
|---|
| 1359 | requests. If not provided, a new one will be allocated.
|
|---|
| 1360 |
|
|---|
| 1361 | Whether provided or newly allocated, the instance is returned. You can
|
|---|
| 1362 | provide it back to subsequent calls of this function to cause the HTTP
|
|---|
| 1363 | connection to the server to be reused."""))
|
|---|
| 1364 |
|
|---|
| 1365 | AddArgumentMetadata(DiGIR.SendRequestToServer, u'timeout',
|
|---|
| 1366 | typeMetadata=IntegerTypeMetadata(minValue=1, canBeNone=True),
|
|---|
| 1367 | description=_(
|
|---|
| 1368 | u"""Number of seconds to wait for the DiGIR server to respond before
|
|---|
| 1369 | failing with a timeout error.
|
|---|
| 1370 |
|
|---|
| 1371 | If you also provide a Maximum Retry Time and it is larger than the
|
|---|
| 1372 | timeout value, the failed request will be retried automatically (with
|
|---|
| 1373 | the same timout value) until it succeeds or the Maximum Retry Time has
|
|---|
| 1374 | elapsed.
|
|---|
| 1375 |
|
|---|
| 1376 | If you receive a timeout error you should investigate the server to
|
|---|
| 1377 | determine if it is malfunctioning or just slow. Check the server's
|
|---|
| 1378 | website to see if the operator has posted a notice about the problem,
|
|---|
| 1379 | or contact the operator directly. If the server just slow, increase
|
|---|
| 1380 | the timeout value to a larger number, to give the server more time to
|
|---|
| 1381 | respond."""),
|
|---|
| 1382 | arcGISDisplayName=_(u'Timeout value'),
|
|---|
| 1383 | arcGISCategory=_(u'Network options'))
|
|---|
| 1384 |
|
|---|
| 1385 | AddArgumentMetadata(DiGIR.SendRequestToServer, u'maxRetryTime',
|
|---|
| 1386 | typeMetadata=IntegerTypeMetadata(minValue=1, canBeNone=True),
|
|---|
| 1387 | description=_(
|
|---|
| 1388 | u"""Number of seconds to retry requests to the DiGIR server before
|
|---|
| 1389 | giving up.
|
|---|
| 1390 |
|
|---|
| 1391 | Use this parameter to cope with a server that experiences transient
|
|---|
| 1392 | failures. For example, some servers are rebooted as part of nightly
|
|---|
| 1393 | maintenance cycles. If you start a long running operation and want it
|
|---|
| 1394 | to run overnight without failing, set the maximum retry time to a
|
|---|
| 1395 | duration that is longer than the time that the server is offline
|
|---|
| 1396 | during the maintenance cycle.
|
|---|
| 1397 |
|
|---|
| 1398 | To maximize performance while minimizing load during failure
|
|---|
| 1399 | situations, retries are scheduled with progressive delays:
|
|---|
| 1400 |
|
|---|
| 1401 | * The first retry is issued immediately.
|
|---|
| 1402 |
|
|---|
| 1403 | * Then, so long as fewer than 10 seconds have elapsed since the
|
|---|
| 1404 | original request was issued, retries are issued every second.
|
|---|
| 1405 |
|
|---|
| 1406 | * After that, retries are issued every 30 seconds until the maximum
|
|---|
| 1407 | retry time is reached or the request succeeds.
|
|---|
| 1408 | """),
|
|---|
| 1409 | arcGISDisplayName=_(u'Maximum retry time'),
|
|---|
| 1410 | arcGISCategory=_(u'Network options'))
|
|---|
| 1411 |
|
|---|
| 1412 | AddResultMetadata(DiGIR.SendRequestToServer, u'response',
|
|---|
| 1413 | typeMetadata=ClassInstanceTypeMetadata(str),
|
|---|
| 1414 | description=_(u'The XML response returned by the server.'))
|
|---|
| 1415 |
|
|---|
| 1416 | AddResultMetadata(DiGIR.SendRequestToServer, u'httpobj',
|
|---|
| 1417 | typeMetadata=ClassInstanceTypeMetadata(Http),
|
|---|
| 1418 | description=_(u'Instance of GeoEco.httplib2.Http that can be provided back to this method to reuse the HTTP connection.'))
|
|---|
| 1419 |
|
|---|
| 1420 | # Public method: DiGIR.GetResourcesAsArcGISTable
|
|---|
| 1421 |
|
|---|
| 1422 | AddMethodMetadata(DiGIR.GetResourcesAsArcGISTable,
|
|---|
| 1423 | shortDescription=_(u'Gets the list of Distributed Generic Information Retrieval (DiGIR) resources available from a DiGIR server and writes it to an ArcGIS table.'),
|
|---|
| 1424 | isExposedToPythonCallers=True,
|
|---|
| 1425 | isExposedByCOM=True,
|
|---|
| 1426 | isExposedAsArcGISTool=True,
|
|---|
| 1427 | arcGISDisplayName=_(u'Get DiGIR Resources as Table'),
|
|---|
| 1428 | arcGISToolCategory=_(u'Conversion\\To Points\\From DiGIR Server'),
|
|---|
| 1429 | dependencies=[ArcGISDependency(9, 1, requiresCOMInstantiation=True), PythonAggregatedModuleDependency('lxml')])
|
|---|
| 1430 |
|
|---|
| 1431 | CopyArgumentMetadata(DiGIR.SendRequestToServer, u'cls', DiGIR.GetResourcesAsArcGISTable, u'cls')
|
|---|
| 1432 | CopyArgumentMetadata(DiGIR.SendRequestToServer, u'url', DiGIR.GetResourcesAsArcGISTable, u'url')
|
|---|
| 1433 |
|
|---|
| 1434 | AddArgumentMetadata(DiGIR.GetResourcesAsArcGISTable, u'resourcesTable',
|
|---|
| 1435 | typeMetadata=ArcGISTableTypeMetadata(deleteIfParameterIsTrue=u'overwriteExisting', createParentDirectories=True),
|
|---|
| 1436 | description=_(
|
|---|
| 1437 | u"""Table to create representing the DiGIR resources available from
|
|---|
| 1438 | the server.
|
|---|
| 1439 |
|
|---|
| 1440 | The table will contain the following fields. Each field represents an
|
|---|
| 1441 | XML element from within the <resource> element of the DiGIR protocol
|
|---|
| 1442 | schema (available at
|
|---|
| 1443 | http://www.digir.net/schema/protocol/2003/1.0/digir.xsd at the time of
|
|---|
| 1444 | this writing). Please see the schema for description of these
|
|---|
| 1445 | fields::
|
|---|
| 1446 |
|
|---|
| 1447 | Field Schema Element ArcGIS Type
|
|---|
| 1448 | ========== ================ ===========
|
|---|
| 1449 | Code code TEXT
|
|---|
| 1450 | NumRecords numberOfRecords LONG
|
|---|
| 1451 | LastUpdate dateLastUpdated DATE
|
|---|
| 1452 | Name name TEXT
|
|---|
| 1453 | Abstract abstract TEXT
|
|---|
| 1454 | Keywords keywords TEXT
|
|---|
| 1455 | Citation citation TEXT
|
|---|
| 1456 | UseRestr useRestrictions TEXT
|
|---|
| 1457 | RecordID recordIdentifier TEXT
|
|---|
| 1458 | RecBasis recordBasis TEXT
|
|---|
| 1459 |
|
|---|
| 1460 | Code should be considered the primary key of this table, and may be
|
|---|
| 1461 | used to join this table to the other tables."""),
|
|---|
| 1462 | direction=u'Output',
|
|---|
| 1463 | arcGISDisplayName=_(u'Output resources table'))
|
|---|
| 1464 |
|
|---|
| 1465 | AddArgumentMetadata(DiGIR.GetResourcesAsArcGISTable, u'relatedInfoTable',
|
|---|
| 1466 | typeMetadata=ArcGISTableTypeMetadata(mustBeDifferentThanArguments=[u'resourcesTable'], canBeNone=True, deleteIfParameterIsTrue=u'overwriteExisting', createParentDirectories=True),
|
|---|
| 1467 | description=_(
|
|---|
| 1468 | u"""Table to create representing the related information entries for
|
|---|
| 1469 | each DiGIR resource.
|
|---|
| 1470 |
|
|---|
| 1471 | The table will contain the following fields. Each field represents an
|
|---|
| 1472 | XML element from within the <resource> element of the DiGIR protocol
|
|---|
| 1473 | schema (available at
|
|---|
| 1474 | http://www.digir.net/schema/protocol/2003/1.0/digir.xsd at the time of
|
|---|
| 1475 | this writing)::
|
|---|
| 1476 |
|
|---|
| 1477 | Field Schema Element ArcGIS Type
|
|---|
| 1478 | ======= ================ ===========
|
|---|
| 1479 | Code code TEXT
|
|---|
| 1480 | RelInfo relatedInformation TEXT
|
|---|
| 1481 |
|
|---|
| 1482 | This table can be joined to the other tables using the Code field.
|
|---|
| 1483 | Each resource can have zero or more RelInfo entries. Each RelInfo is
|
|---|
| 1484 | typically a URL."""),
|
|---|
| 1485 | direction=u'Output',
|
|---|
| 1486 | arcGISDisplayName=_(u'Output related information table'))
|
|---|
| 1487 |
|
|---|
| 1488 | AddArgumentMetadata(DiGIR.GetResourcesAsArcGISTable, u'contactsTable',
|
|---|
| 1489 | typeMetadata=ArcGISTableTypeMetadata(mustBeDifferentThanArguments=[u'resourcesTable', u'relatedInfoTable'], canBeNone=True, deleteIfParameterIsTrue=u'overwriteExisting', createParentDirectories=True),
|
|---|
| 1490 | description=_(
|
|---|
| 1491 | u"""Table to create representing the contacts for each DiGIR resource.
|
|---|
| 1492 |
|
|---|
| 1493 | The table will contain the following fields. Each field represents an
|
|---|
| 1494 | XML element from within the <resource> element of the DiGIR protocol
|
|---|
| 1495 | schema (available at
|
|---|
| 1496 | http://www.digir.net/schema/protocol/2003/1.0/digir.xsd at the time of
|
|---|
| 1497 | this writing)::
|
|---|
| 1498 |
|
|---|
| 1499 | Field Schema Element ArcGIS Type
|
|---|
| 1500 | ===== ============== ===========
|
|---|
| 1501 | Code code TEXT
|
|---|
| 1502 | Name name TEXT
|
|---|
| 1503 | Title title TEXT
|
|---|
| 1504 | Email emailAddress TEXT
|
|---|
| 1505 | Phone phone TEXT
|
|---|
| 1506 | Type type TEXT
|
|---|
| 1507 |
|
|---|
| 1508 | This table can be joined to the other tables using the Code field.
|
|---|
| 1509 | Each resource can have zero or more contacts."""),
|
|---|
| 1510 | direction=u'Output',
|
|---|
| 1511 | arcGISDisplayName=_(u'Output contacts table'))
|
|---|
| 1512 |
|
|---|
| 1513 | AddArgumentMetadata(DiGIR.GetResourcesAsArcGISTable, u'schemasTable',
|
|---|
| 1514 | typeMetadata=ArcGISTableTypeMetadata(mustBeDifferentThanArguments=[u'resourcesTable', u'relatedInfoTable', u'contactsTable'], canBeNone=True, deleteIfParameterIsTrue=u'overwriteExisting', createParentDirectories=True),
|
|---|
| 1515 | description=_(
|
|---|
| 1516 | u"""Table to create listing the conceptual schemas used by each
|
|---|
| 1517 | resource.
|
|---|
| 1518 |
|
|---|
| 1519 | The table will contain the following fields. Each field represents an
|
|---|
| 1520 | XML element from within the <resource> element of the DiGIR protocol
|
|---|
| 1521 | schema (available at
|
|---|
| 1522 | http://www.digir.net/schema/protocol/2003/1.0/digir.xsd at the time of
|
|---|
| 1523 | this writing)::
|
|---|
| 1524 |
|
|---|
| 1525 | Field Schema Element ArcGIS Type
|
|---|
| 1526 | ========= ========================== ===========
|
|---|
| 1527 | Code code TEXT
|
|---|
| 1528 | Namespace conceptualSchema TEXT
|
|---|
| 1529 | Location schemaLocation (attribute) TEXT
|
|---|
| 1530 |
|
|---|
| 1531 | This table can be joined to the other tables using the Code field.
|
|---|
| 1532 | Each resource can have one or more conceptual schemas. At the time of
|
|---|
| 1533 | this writing, the most commonly used conceptual schema was called
|
|---|
| 1534 | Darwin2, which had the namespace
|
|---|
| 1535 | http://digir.net/schema/conceptual/darwin/2003/1.0 and the location
|
|---|
| 1536 | http://digir.sourceforge.net/schema/conceptual/darwin/2003/1.0/darwin2.xsd."""),
|
|---|
| 1537 | direction=u'Output',
|
|---|
| 1538 | arcGISDisplayName=_(u'Output conceptual schemas table'))
|
|---|
| 1539 |
|
|---|
| 1540 | AddArgumentMetadata(DiGIR.GetResourcesAsArcGISTable, u'dataElementsTable',
|
|---|
| 1541 | typeMetadata=ArcGISTableTypeMetadata(mustBeDifferentThanArguments=[u'resourcesTable', u'relatedInfoTable', u'contactsTable', u'schemasTable'], canBeNone=True, deleteIfParameterIsTrue=u'overwriteExisting', createParentDirectories=True),
|
|---|
| 1542 | description=_(
|
|---|
| 1543 | u"""Table to create representing the data elements contained by the
|
|---|
| 1544 | conceptual schemas used by the resources.
|
|---|
| 1545 |
|
|---|
| 1546 | The table will contain the following fields:
|
|---|
| 1547 |
|
|---|
| 1548 | * Name (TEXT) - Name of the data element, as it appears in XML schema
|
|---|
| 1549 | (XSD file) for the conceptual schema.
|
|---|
| 1550 |
|
|---|
| 1551 | * XSDType (TEXT) - XML schema data type of the data element.
|
|---|
| 1552 |
|
|---|
| 1553 | * ArcGISType (TEXT) - ArcGIS data type of the data element.
|
|---|
| 1554 |
|
|---|
| 1555 | * Searchable (SHORT) - 1 indicates that the data element can be used
|
|---|
| 1556 | in a filter expression when searching the DiGIR server. 0 indicates
|
|---|
| 1557 | it may not be used in a filter expresison.
|
|---|
| 1558 |
|
|---|
| 1559 | * Returnable (SHORT) - 1 indicates that the data element can be
|
|---|
| 1560 | included in records returned by the DiGIR server in response to a
|
|---|
| 1561 | search request. 0 indicates it cannot be included in records
|
|---|
| 1562 | returned by the server.
|
|---|
| 1563 |
|
|---|
| 1564 | * Namespace (TEXT) - XML namespace of the conceptual schema that
|
|---|
| 1565 | defines this data element. At the time of this writing, the most
|
|---|
| 1566 | commonly used conceptual schema was called Darwin2, which had the
|
|---|
| 1567 | namespace http://digir.net/schema/conceptual/darwin/2003/1.0
|
|---|
| 1568 |
|
|---|
| 1569 | * Location (TEXT) - Location of the XML schema (XSD) for the
|
|---|
| 1570 | conceptual schema that defines this data element. This information
|
|---|
| 1571 | is redundant; it also appears in the conceptual schemas table, but
|
|---|
| 1572 | is repeated here for convenience.
|
|---|
| 1573 |
|
|---|
| 1574 | * Descr (TEXT) - Description of this data element, taken from the XML
|
|---|
| 1575 | <annotation> element in the XML schema.
|
|---|
| 1576 | """),
|
|---|
| 1577 | direction=u'Output',
|
|---|
| 1578 | arcGISDisplayName=_(u'Output data elements table'))
|
|---|
| 1579 |
|
|---|
| 1580 | CopyArgumentMetadata(DiGIR.SendRequestToServer, u'timeout', DiGIR.GetResourcesAsArcGISTable, u'timeout')
|
|---|
| 1581 | CopyArgumentMetadata(DiGIR.SendRequestToServer, u'maxRetryTime', DiGIR.GetResourcesAsArcGISTable, u'maxRetryTime')
|
|---|
| 1582 |
|
|---|
| 1583 | AddArgumentMetadata(DiGIR.GetResourcesAsArcGISTable, u'overwriteExisting',
|
|---|
| 1584 | typeMetadata=BooleanTypeMetadata(),
|
|---|
| 1585 | description=_(
|
|---|
| 1586 | u"""If True, the output tables will be overwritten, if they exist. If
|
|---|
| 1587 | False, a ValueError will be raised if any of the output tables
|
|---|
| 1588 | exist."""))
|
|---|
| 1589 |
|
|---|
| 1590 | # Public method: DiGIR.SearchAndCreateArcGISPoints
|
|---|
| 1591 |
|
|---|
| 1592 | AddMethodMetadata(DiGIR.SearchAndCreateArcGISPoints,
|
|---|
| 1593 | shortDescription=_(u'Searches a Distributed Generic Information Retrieval (DiGIR) server for georeferenced records and creates an ArcGIS point feature class from them.'),
|
|---|
| 1594 | isExposedToPythonCallers=True,
|
|---|
| 1595 | isExposedByCOM=True,
|
|---|
| 1596 | isExposedAsArcGISTool=True,
|
|---|
| 1597 | arcGISDisplayName=_(u'Search DiGIR Records and Create Points'),
|
|---|
| 1598 | arcGISToolCategory=_(u'Conversion\\To Points\\From DiGIR Server'),
|
|---|
| 1599 | dependencies=[ArcGISDependency(9, 1, requiresCOMInstantiation=True), PythonAggregatedModuleDependency('lxml'), PythonAggregatedModuleDependency('pyparsing')])
|
|---|
| 1600 |
|
|---|
| 1601 | CopyArgumentMetadata(DiGIR.SendRequestToServer, u'cls', DiGIR.SearchAndCreateArcGISPoints, u'cls')
|
|---|
| 1602 | CopyArgumentMetadata(DiGIR.SendRequestToServer, u'url', DiGIR.SearchAndCreateArcGISPoints, u'url')
|
|---|
| 1603 |
|
|---|
| 1604 | AddArgumentMetadata(DiGIR.SearchAndCreateArcGISPoints, u'pointFeatureClass',
|
|---|
| 1605 | typeMetadata=ArcGISFeatureClassTypeMetadata(deleteIfParameterIsTrue=u'overwriteExisting', createParentDirectories=True),
|
|---|
| 1606 | description=_(
|
|---|
| 1607 | u"""Output point feature class to create.
|
|---|
| 1608 |
|
|---|
| 1609 | To create points from the DiGIR records, the tool must decide which
|
|---|
| 1610 | data elements represent the X and Y coordinates. If you specify data
|
|---|
| 1611 | element names (in the parameters below), the tool will use those
|
|---|
| 1612 | elements. If you don't, the tool will look for data elements named
|
|---|
| 1613 | Longitude and Latitude. If those are not found, it will try
|
|---|
| 1614 | DecimalLongitude and DecimalLatitude. If those are not found it will
|
|---|
| 1615 | report an error, indicating that you must specify the names.
|
|---|
| 1616 |
|
|---|
| 1617 | The points will have an attribute named ResourceCode, which specifies
|
|---|
| 1618 | the DiGIR resource that provided the point. This column may be joined
|
|---|
| 1619 | to the Code column of the table output by the Get DiGIR Resources
|
|---|
| 1620 | tool.
|
|---|
| 1621 |
|
|---|
| 1622 | You may also specify (in a parameter below) a list of data elements to
|
|---|
| 1623 | retrieve from the server and store as additional attributes of the
|
|---|
| 1624 | points. If you omit this list, the tool will retrieve all of the data
|
|---|
| 1625 | elements defined in the conceptual schemas.
|
|---|
| 1626 |
|
|---|
| 1627 | Data elements that are not available for a given record will be set to
|
|---|
| 1628 | NULL. If the output feature class is a shapefile, the value -999 will
|
|---|
| 1629 | be stored in integer or floating point columns instead of NULL
|
|---|
| 1630 | (because shapefiles do not support NULL)."""),
|
|---|
| 1631 | direction = u'Output',
|
|---|
| 1632 | arcGISDisplayName=_(u'Output point feature class'))
|
|---|
| 1633 |
|
|---|
| 1634 | AddArgumentMetadata(DiGIR.SearchAndCreateArcGISPoints, u'filterExpression',
|
|---|
| 1635 | typeMetadata=UnicodeStringTypeMetadata(canBeNone=True),
|
|---|
| 1636 | description=_(
|
|---|
| 1637 | u"""Expression that describes the records to request from the server.
|
|---|
| 1638 |
|
|---|
| 1639 | If you do not specify an expression, all of the records will be
|
|---|
| 1640 | requested. On some servers, this may be millions of records, so we
|
|---|
| 1641 | recommend you provide an expression. You can also limit the records
|
|---|
| 1642 | returned by specifying (in following parameters) a list of resources
|
|---|
| 1643 | to query and a maximum number of records to retrieve.
|
|---|
| 1644 |
|
|---|
| 1645 | The expression syntax is a subset of the SQL "where clause" syntax,
|
|---|
| 1646 | also called "SQL expression" syntax in ArcGIS. For example, to specify
|
|---|
| 1647 | a bounding box::
|
|---|
| 1648 |
|
|---|
| 1649 | Latitude >= 0 and Latitude <= 30 and Longitude >= -60 and Longitude <= -40
|
|---|
| 1650 |
|
|---|
| 1651 | The expression may reference DiGIR data elements that are flagged as
|
|---|
| 1652 | "searchable" in the conceptual schemas used on the server. For a list
|
|---|
| 1653 | of available data elements, use the Get DiGIR Resources tool. Data
|
|---|
| 1654 | element names are case sensitive.
|
|---|
| 1655 |
|
|---|
| 1656 | The following comparison operators are supported::
|
|---|
| 1657 |
|
|---|
| 1658 | Operator Meaning
|
|---|
| 1659 | ======== ========================
|
|---|
| 1660 | = equal to
|
|---|
| 1661 | != not equal to
|
|---|
| 1662 | <> not equal to
|
|---|
| 1663 | < less than
|
|---|
| 1664 | <= less than or equal to
|
|---|
| 1665 | > greater than
|
|---|
| 1666 | >= greater than or equal to
|
|---|
| 1667 | LIKE matches simple wildcard
|
|---|
| 1668 | IN matches any one of a set
|
|---|
| 1669 |
|
|---|
| 1670 | The LIKE operator may only be used with data elements that are
|
|---|
| 1671 | represented in ArcGIS with the TEXT data type. Use the % symbol to
|
|---|
| 1672 | represent a wildcard, for example::
|
|---|
| 1673 |
|
|---|
| 1674 | ScientificName LIKE 'Stenella%'
|
|---|
| 1675 |
|
|---|
| 1676 | The IN operator takes a list of one or more values in parentheses::
|
|---|
| 1677 |
|
|---|
| 1678 | MonthIdentified IN (1, 2, 3, 10, 11, 12)
|
|---|
| 1679 |
|
|---|
| 1680 | Two logical operators are supported: AND and OR. No other operators
|
|---|
| 1681 | are supported (e.g. the NOT operator). AND has higher precedence than
|
|---|
| 1682 | OR. Comparison operations have higher precedence than AND.
|
|---|
| 1683 |
|
|---|
| 1684 | Logical expressions may be grouped with parentheses, for example::
|
|---|
| 1685 |
|
|---|
| 1686 | Latitude > 45 AND (ScientificName = 'Foo' OR ScientificName = 'Bar')
|
|---|
| 1687 |
|
|---|
| 1688 | For a string literal, enclose the characters in single or double
|
|---|
| 1689 | quotation marks::
|
|---|
| 1690 |
|
|---|
| 1691 | ScientificName = 'Stenella frontalis'
|
|---|
| 1692 | ScientificName = "Stenella frontalis"
|
|---|
| 1693 |
|
|---|
| 1694 | Date literals be enclosed in # characters and take one of the
|
|---|
| 1695 | following two formats::
|
|---|
| 1696 |
|
|---|
| 1697 | #YYYY-MM-DD HH:MM:SS#
|
|---|
| 1698 | #YYYY-MM-DD#
|
|---|
| 1699 |
|
|---|
| 1700 | If the second format is used, it will be assumed that HH:MM:SS is
|
|---|
| 1701 | 00:00:00. All dates are assumed to be in the UTC time zone.
|
|---|
| 1702 | """),
|
|---|
| 1703 | arcGISDisplayName=_(u'Filter expression'))
|
|---|
| 1704 |
|
|---|
| 1705 | AddArgumentMetadata(DiGIR.SearchAndCreateArcGISPoints, u'resources',
|
|---|
| 1706 | typeMetadata=ListTypeMetadata(elementType=UnicodeStringTypeMetadata(), minLength=1, canBeNone=True),
|
|---|
| 1707 | description=_(
|
|---|
| 1708 | u"""List of DiGIR resources to request records from.
|
|---|
| 1709 |
|
|---|
| 1710 | If you omit this parameter, records will be requested from all DiGIR
|
|---|
| 1711 | resources available on the server. In this parameter, resources are
|
|---|
| 1712 | referred to using their "code". To obtain a list of codes for a
|
|---|
| 1713 | server, use the Get DiGIR Resources tool."""),
|
|---|
| 1714 | arcGISDisplayName=_(u'DiGIR resources to query'),
|
|---|
| 1715 | arcGISCategory=_(u'Additional search options'))
|
|---|
| 1716 |
|
|---|
| 1717 | AddArgumentMetadata(DiGIR.SearchAndCreateArcGISPoints, u'dataElementNames',
|
|---|
| 1718 | typeMetadata=ListTypeMetadata(elementType=UnicodeStringTypeMetadata(), canBeNone=True),
|
|---|
| 1719 | description=_(
|
|---|
| 1720 | u"""List of DiGIR data elements to retrieve and store as attributes of
|
|---|
| 1721 | the points.
|
|---|
| 1722 |
|
|---|
| 1723 | If you omit this parameter, the tool will request all of the data
|
|---|
| 1724 | elements defined in the conceptual schemas used by the resources
|
|---|
| 1725 | available from the server. To optimize the performance of this tool,
|
|---|
| 1726 | and minimize network traffic and disk space utilization, specify only
|
|---|
| 1727 | the data elements that you actually need."""),
|
|---|
| 1728 | arcGISDisplayName=_(u'DiGIR data elements to retrieve'),
|
|---|
| 1729 | arcGISCategory=_(u'Additional search options'))
|
|---|
| 1730 |
|
|---|
| 1731 | AddArgumentMetadata(DiGIR.SearchAndCreateArcGISPoints, u'xCoordinateDataElementName',
|
|---|
| 1732 | typeMetadata=UnicodeStringTypeMetadata(canBeNone=True),
|
|---|
| 1733 | description=_(
|
|---|
| 1734 | u"""Name of the data element to use for the X coordinates of the points.
|
|---|
| 1735 |
|
|---|
| 1736 | If you omit this parameter, the tool will search the conceptual
|
|---|
| 1737 | schemas for an element called Longitude. If it is not found, the tool
|
|---|
| 1738 | will try DecimalLongitude. If that is not found, the tool will report
|
|---|
| 1739 | an error."""),
|
|---|
| 1740 | arcGISDisplayName=_(u'DiGIR data element representing the X coordinate'),
|
|---|
| 1741 | arcGISCategory=_(u'Additional search options'))
|
|---|
| 1742 |
|
|---|
| 1743 | AddArgumentMetadata(DiGIR.SearchAndCreateArcGISPoints, u'yCoordinateDataElementName',
|
|---|
| 1744 | typeMetadata=UnicodeStringTypeMetadata(canBeNone=True),
|
|---|
| 1745 | description=_(
|
|---|
| 1746 | u"""Name of the data element to use for the Y coordinates of the points.
|
|---|
| 1747 |
|
|---|
| 1748 | If you omit this parameter, the tool will search the conceptual
|
|---|
| 1749 | schemas for an element called Latitude. If it is not found, the tool
|
|---|
| 1750 | will try DecimalLatitude. If that is not found, the tool will report
|
|---|
| 1751 | an error."""),
|
|---|
| 1752 | arcGISDisplayName=_(u'DiGIR data element representing the Y coordinate'),
|
|---|
| 1753 | arcGISCategory=_(u'Additional search options'))
|
|---|
| 1754 |
|
|---|
| 1755 | AddArgumentMetadata(DiGIR.SearchAndCreateArcGISPoints, u'maxRecords',
|
|---|
| 1756 | typeMetadata=IntegerTypeMetadata(canBeNone=True, minValue=1),
|
|---|
| 1757 | description=_(
|
|---|
| 1758 | u"""Maximum number of records to retrieve from the DiGIR server.
|
|---|
| 1759 |
|
|---|
| 1760 | If you omit this parameter, the tool will retrieve all of the records
|
|---|
| 1761 | available on the server (that match your filter expression and list of
|
|---|
| 1762 | resources, if you provided them)."""),
|
|---|
| 1763 | arcGISDisplayName=_(u'Maximum number of records to retrieve'),
|
|---|
| 1764 | arcGISCategory=_(u'Additional search options'))
|
|---|
| 1765 |
|
|---|
| 1766 | CopyArgumentMetadata(DiGIR.SendRequestToServer, u'timeout', DiGIR.SearchAndCreateArcGISPoints, u'timeout')
|
|---|
| 1767 | CopyArgumentMetadata(DiGIR.SendRequestToServer, u'maxRetryTime', DiGIR.SearchAndCreateArcGISPoints, u'maxRetryTime')
|
|---|
| 1768 |
|
|---|
| 1769 | AddArgumentMetadata(DiGIR.SearchAndCreateArcGISPoints, u'overwriteExisting',
|
|---|
| 1770 | typeMetadata=BooleanTypeMetadata(),
|
|---|
| 1771 | description=_(
|
|---|
| 1772 | u"""If True, the output feature class will be overwritten, if it
|
|---|
| 1773 | exists. If False, a ValueError will be raised if it exists."""))
|
|---|
| 1774 |
|
|---|
| 1775 | # Public method: DiGIR.GetOBISResourcesAsArcGISTable
|
|---|
| 1776 |
|
|---|
| 1777 | AddMethodMetadata(DiGIR.GetOBISResourcesAsArcGISTable,
|
|---|
| 1778 | shortDescription=_(u'Gets the list of Distributed Generic Information Retrieval (DiGIR) resources available from OBIS and writes it to an ArcGIS table.'),
|
|---|
| 1779 | isExposedToPythonCallers=True,
|
|---|
| 1780 | isExposedByCOM=True,
|
|---|
| 1781 | isExposedAsArcGISTool=True,
|
|---|
| 1782 | arcGISDisplayName=_(u'Get OBIS DiGIR Resources as Table'),
|
|---|
| 1783 | arcGISToolCategory=_(u'Data Products\\Ocean Biogeographic Information System'),
|
|---|
| 1784 | dependencies=[ArcGISDependency(9, 1, requiresCOMInstantiation=True), PythonAggregatedModuleDependency('lxml')])
|
|---|
| 1785 |
|
|---|
| 1786 | CopyArgumentMetadata(DiGIR.GetResourcesAsArcGISTable, u'cls', DiGIR.GetOBISResourcesAsArcGISTable, u'cls')
|
|---|
| 1787 | CopyArgumentMetadata(DiGIR.GetResourcesAsArcGISTable, u'resourcesTable', DiGIR.GetOBISResourcesAsArcGISTable, u'resourcesTable')
|
|---|
| 1788 | CopyArgumentMetadata(DiGIR.GetResourcesAsArcGISTable, u'relatedInfoTable', DiGIR.GetOBISResourcesAsArcGISTable, u'relatedInfoTable')
|
|---|
| 1789 | CopyArgumentMetadata(DiGIR.GetResourcesAsArcGISTable, u'contactsTable', DiGIR.GetOBISResourcesAsArcGISTable, u'contactsTable')
|
|---|
| 1790 | CopyArgumentMetadata(DiGIR.GetResourcesAsArcGISTable, u'schemasTable', DiGIR.GetOBISResourcesAsArcGISTable, u'schemasTable')
|
|---|
| 1791 | CopyArgumentMetadata(DiGIR.GetResourcesAsArcGISTable, u'dataElementsTable', DiGIR.GetOBISResourcesAsArcGISTable, u'dataElementsTable')
|
|---|
| 1792 |
|
|---|
| 1793 | AddArgumentMetadata(DiGIR.GetOBISResourcesAsArcGISTable, u'url',
|
|---|
| 1794 | typeMetadata=UnicodeStringTypeMetadata(),
|
|---|
| 1795 | description=_(
|
|---|
| 1796 | u"""URL of the OBIS DiGIR server to query. You should use the default
|
|---|
| 1797 | URL unless otherwise instructed by OBIS personnel."""),
|
|---|
| 1798 | arcGISDisplayName=_(u'OBIS DiGIR server URL'))
|
|---|
| 1799 |
|
|---|
| 1800 | CopyArgumentMetadata(DiGIR.GetResourcesAsArcGISTable, u'timeout', DiGIR.GetOBISResourcesAsArcGISTable, u'timeout')
|
|---|
| 1801 | CopyArgumentMetadata(DiGIR.GetResourcesAsArcGISTable, u'maxRetryTime', DiGIR.GetOBISResourcesAsArcGISTable, u'maxRetryTime')
|
|---|
| 1802 |
|
|---|
| 1803 | CopyArgumentMetadata(DiGIR.GetResourcesAsArcGISTable, u'overwriteExisting', DiGIR.GetOBISResourcesAsArcGISTable, u'overwriteExisting')
|
|---|
| 1804 |
|
|---|
| 1805 | # Public method: DiGIR.SearchOBISAndCreateArcGISPoints
|
|---|
| 1806 |
|
|---|
| 1807 | AddMethodMetadata(DiGIR.SearchOBISAndCreateArcGISPoints,
|
|---|
| 1808 | shortDescription=_(u'Searches OBIS for georeferenced records using the Distributed Generic Information Retrieval (DiGIR) protocol and creates an ArcGIS point feature class from them.'),
|
|---|
| 1809 | isExposedToPythonCallers=True,
|
|---|
| 1810 | isExposedByCOM=True,
|
|---|
| 1811 | isExposedAsArcGISTool=True,
|
|---|
| 1812 | arcGISDisplayName=_(u'Search OBIS DiGIR Records and Create Points'),
|
|---|
| 1813 | arcGISToolCategory=_(u'Data Products\\Ocean Biogeographic Information System'),
|
|---|
| 1814 | dependencies=[ArcGISDependency(9, 1, requiresCOMInstantiation=True), PythonAggregatedModuleDependency('lxml'), PythonAggregatedModuleDependency('pyparsing')])
|
|---|
| 1815 |
|
|---|
| 1816 | CopyArgumentMetadata(DiGIR.SearchAndCreateArcGISPoints, u'cls', DiGIR.SearchOBISAndCreateArcGISPoints, u'cls')
|
|---|
| 1817 |
|
|---|
| 1818 | AddArgumentMetadata(DiGIR.SearchOBISAndCreateArcGISPoints, u'pointFeatureClass',
|
|---|
| 1819 | typeMetadata=ArcGISFeatureClassTypeMetadata(deleteIfParameterIsTrue=u'overwriteExisting', createParentDirectories=True),
|
|---|
| 1820 | description=_(
|
|---|
| 1821 | u"""Output point feature class to create.
|
|---|
| 1822 |
|
|---|
| 1823 | The points will have an attribute named ResourceCode, which specifies
|
|---|
| 1824 | the DiGIR resource that provided the point. This column may be joined
|
|---|
| 1825 | to the Code column of the table output by the Get OBIS DiGIR Resources
|
|---|
| 1826 | as Table tool.
|
|---|
| 1827 |
|
|---|
| 1828 | You may also specify (in a parameter below) a list of data elements to
|
|---|
| 1829 | retrieve from the server and store as additional attributes of the
|
|---|
| 1830 | points. If you omit this list, the tool will retrieve all of the data
|
|---|
| 1831 | elements defined in the conceptual schemas.
|
|---|
| 1832 |
|
|---|
| 1833 | Data elements that are not available for a given record will be set to
|
|---|
| 1834 | NULL. If the output feature class is a shapefile, the value -999 will
|
|---|
| 1835 | be stored in integer or floating point columns instead of NULL
|
|---|
| 1836 | (because shapefiles do not support NULL)."""),
|
|---|
| 1837 | direction = u'Output',
|
|---|
| 1838 | arcGISDisplayName=_(u'Output point feature class'))
|
|---|
| 1839 |
|
|---|
| 1840 | CopyArgumentMetadata(DiGIR.SearchAndCreateArcGISPoints, u'filterExpression', DiGIR.SearchOBISAndCreateArcGISPoints, u'filterExpression')
|
|---|
| 1841 | CopyArgumentMetadata(DiGIR.SearchAndCreateArcGISPoints, u'resources', DiGIR.SearchOBISAndCreateArcGISPoints, u'resources')
|
|---|
| 1842 | CopyArgumentMetadata(DiGIR.SearchAndCreateArcGISPoints, u'dataElementNames', DiGIR.SearchOBISAndCreateArcGISPoints, u'dataElementNames')
|
|---|
| 1843 | CopyArgumentMetadata(DiGIR.SearchAndCreateArcGISPoints, u'maxRecords', DiGIR.SearchOBISAndCreateArcGISPoints, u'maxRecords')
|
|---|
| 1844 | CopyArgumentMetadata(DiGIR.GetOBISResourcesAsArcGISTable, u'url', DiGIR.SearchOBISAndCreateArcGISPoints, u'url')
|
|---|
| 1845 | CopyArgumentMetadata(DiGIR.SearchAndCreateArcGISPoints, u'timeout', DiGIR.SearchOBISAndCreateArcGISPoints, u'timeout')
|
|---|
| 1846 | CopyArgumentMetadata(DiGIR.SearchAndCreateArcGISPoints, u'maxRetryTime', DiGIR.SearchOBISAndCreateArcGISPoints, u'maxRetryTime')
|
|---|
| 1847 | CopyArgumentMetadata(DiGIR.SearchAndCreateArcGISPoints, u'overwriteExisting', DiGIR.SearchOBISAndCreateArcGISPoints, u'overwriteExisting')
|
|---|
| 1848 |
|
|---|
| 1849 | # Public method: DiGIR.GetOBISSEAMAPResourcesAsArcGISTable
|
|---|
| 1850 |
|
|---|
| 1851 | AddMethodMetadata(DiGIR.GetOBISSEAMAPResourcesAsArcGISTable,
|
|---|
| 1852 | shortDescription=_(u'Gets the list of Distributed Generic Information Retrieval (DiGIR) resources available from OBIS-SEAMAP and writes it to an ArcGIS table.'),
|
|---|
| 1853 | isExposedToPythonCallers=True,
|
|---|
| 1854 | isExposedByCOM=True,
|
|---|
| 1855 | isExposedAsArcGISTool=True,
|
|---|
| 1856 | arcGISDisplayName=_(u'Get OBIS-SEAMAP DiGIR Resources as Table'),
|
|---|
| 1857 | arcGISToolCategory=_(u'Data Products\\Duke University Marine Geospatial Ecology Laboratory'),
|
|---|
| 1858 | dependencies=[ArcGISDependency(9, 1, requiresCOMInstantiation=True), PythonAggregatedModuleDependency('lxml')])
|
|---|
| 1859 |
|
|---|
| 1860 | CopyArgumentMetadata(DiGIR.GetResourcesAsArcGISTable, u'cls', DiGIR.GetOBISSEAMAPResourcesAsArcGISTable, u'cls')
|
|---|
| 1861 | CopyArgumentMetadata(DiGIR.GetResourcesAsArcGISTable, u'resourcesTable', DiGIR.GetOBISSEAMAPResourcesAsArcGISTable, u'resourcesTable')
|
|---|
| 1862 | CopyArgumentMetadata(DiGIR.GetResourcesAsArcGISTable, u'relatedInfoTable', DiGIR.GetOBISSEAMAPResourcesAsArcGISTable, u'relatedInfoTable')
|
|---|
| 1863 | CopyArgumentMetadata(DiGIR.GetResourcesAsArcGISTable, u'contactsTable', DiGIR.GetOBISSEAMAPResourcesAsArcGISTable, u'contactsTable')
|
|---|
| 1864 | CopyArgumentMetadata(DiGIR.GetResourcesAsArcGISTable, u'schemasTable', DiGIR.GetOBISSEAMAPResourcesAsArcGISTable, u'schemasTable')
|
|---|
| 1865 | CopyArgumentMetadata(DiGIR.GetResourcesAsArcGISTable, u'dataElementsTable', DiGIR.GetOBISSEAMAPResourcesAsArcGISTable, u'dataElementsTable')
|
|---|
| 1866 |
|
|---|
| 1867 | AddArgumentMetadata(DiGIR.GetOBISSEAMAPResourcesAsArcGISTable, u'url',
|
|---|
| 1868 | typeMetadata=UnicodeStringTypeMetadata(),
|
|---|
| 1869 | description=_(
|
|---|
| 1870 | u"""URL of the OBIS-SEAMAP DiGIR server to query. You should use the
|
|---|
| 1871 | default URL unless otherwise instructed by OBIS-SEAMAP personnel."""),
|
|---|
| 1872 | arcGISDisplayName=_(u'OBIS-SEAMAP DiGIR server URL'))
|
|---|
| 1873 |
|
|---|
| 1874 | CopyArgumentMetadata(DiGIR.GetResourcesAsArcGISTable, u'timeout', DiGIR.GetOBISSEAMAPResourcesAsArcGISTable, u'timeout')
|
|---|
| 1875 | CopyArgumentMetadata(DiGIR.GetResourcesAsArcGISTable, u'maxRetryTime', DiGIR.GetOBISSEAMAPResourcesAsArcGISTable, u'maxRetryTime')
|
|---|
| 1876 |
|
|---|
| 1877 | CopyArgumentMetadata(DiGIR.GetResourcesAsArcGISTable, u'overwriteExisting', DiGIR.GetOBISSEAMAPResourcesAsArcGISTable, u'overwriteExisting')
|
|---|
| 1878 |
|
|---|
| 1879 | # Public method: DiGIR.SearchOBISSEAMAPAndCreateArcGISPoints
|
|---|
| 1880 |
|
|---|
| 1881 | AddMethodMetadata(DiGIR.SearchOBISSEAMAPAndCreateArcGISPoints,
|
|---|
| 1882 | shortDescription=_(u'Searches OBIS-SEAMAP for georeferenced records using the Distributed Generic Information Retrieval (DiGIR) protocol and creates an ArcGIS point feature class from them.'),
|
|---|
| 1883 | isExposedToPythonCallers=True,
|
|---|
| 1884 | isExposedByCOM=True,
|
|---|
| 1885 | isExposedAsArcGISTool=True,
|
|---|
| 1886 | arcGISDisplayName=_(u'Search OBIS-SEAMAP DiGIR Records and Create Points'),
|
|---|
| 1887 | arcGISToolCategory=_(u'Data Products\\Duke University Marine Geospatial Ecology Laboratory'),
|
|---|
| 1888 | dependencies=[ArcGISDependency(9, 1, requiresCOMInstantiation=True), PythonAggregatedModuleDependency('lxml'), PythonAggregatedModuleDependency('pyparsing')])
|
|---|
| 1889 |
|
|---|
| 1890 | CopyArgumentMetadata(DiGIR.SearchAndCreateArcGISPoints, u'cls', DiGIR.SearchOBISSEAMAPAndCreateArcGISPoints, u'cls')
|
|---|
| 1891 |
|
|---|
| 1892 | AddArgumentMetadata(DiGIR.SearchOBISSEAMAPAndCreateArcGISPoints, u'pointFeatureClass',
|
|---|
| 1893 | typeMetadata=ArcGISFeatureClassTypeMetadata(deleteIfParameterIsTrue=u'overwriteExisting', createParentDirectories=True),
|
|---|
| 1894 | description=_(
|
|---|
| 1895 | u"""Output point feature class to create.
|
|---|
| 1896 |
|
|---|
| 1897 | The points will have an attribute named ResourceCode, which specifies
|
|---|
| 1898 | the DiGIR resource that provided the point. This column may be joined
|
|---|
| 1899 | to the Code column of the table output by the Get OBIS-SEAMAP DiGIR
|
|---|
| 1900 | Resources as Table tool.
|
|---|
| 1901 |
|
|---|
| 1902 | You may also specify (in a parameter below) a list of data elements to
|
|---|
| 1903 | retrieve from the server and store as additional attributes of the
|
|---|
| 1904 | points. If you omit this list, the tool will retrieve all of the data
|
|---|
| 1905 | elements defined in the conceptual schemas.
|
|---|
| 1906 |
|
|---|
| 1907 | Data elements that are not available for a given record will be set to
|
|---|
| 1908 | NULL. If the output feature class is a shapefile, the value -999 will
|
|---|
| 1909 | be stored in integer or floating point columns instead of NULL
|
|---|
| 1910 | (because shapefiles do not support NULL)."""),
|
|---|
| 1911 | direction = u'Output',
|
|---|
| 1912 | arcGISDisplayName=_(u'Output point feature class'))
|
|---|
| 1913 |
|
|---|
| 1914 | CopyArgumentMetadata(DiGIR.SearchAndCreateArcGISPoints, u'filterExpression', DiGIR.SearchOBISSEAMAPAndCreateArcGISPoints, u'filterExpression')
|
|---|
| 1915 | CopyArgumentMetadata(DiGIR.SearchAndCreateArcGISPoints, u'resources', DiGIR.SearchOBISSEAMAPAndCreateArcGISPoints, u'resources')
|
|---|
| 1916 | CopyArgumentMetadata(DiGIR.SearchAndCreateArcGISPoints, u'dataElementNames', DiGIR.SearchOBISSEAMAPAndCreateArcGISPoints, u'dataElementNames')
|
|---|
| 1917 | CopyArgumentMetadata(DiGIR.SearchAndCreateArcGISPoints, u'maxRecords', DiGIR.SearchOBISSEAMAPAndCreateArcGISPoints, u'maxRecords')
|
|---|
| 1918 | CopyArgumentMetadata(DiGIR.GetOBISSEAMAPResourcesAsArcGISTable, u'url', DiGIR.SearchOBISSEAMAPAndCreateArcGISPoints, u'url')
|
|---|
| 1919 | CopyArgumentMetadata(DiGIR.SearchAndCreateArcGISPoints, u'timeout', DiGIR.SearchOBISSEAMAPAndCreateArcGISPoints, u'timeout')
|
|---|
| 1920 | CopyArgumentMetadata(DiGIR.SearchAndCreateArcGISPoints, u'maxRetryTime', DiGIR.SearchOBISSEAMAPAndCreateArcGISPoints, u'maxRetryTime')
|
|---|
| 1921 | CopyArgumentMetadata(DiGIR.SearchAndCreateArcGISPoints, u'overwriteExisting', DiGIR.SearchOBISSEAMAPAndCreateArcGISPoints, u'overwriteExisting')
|
|---|
| 1922 |
|
|---|
| 1923 | ###############################################################################
|
|---|
| 1924 | # Names exported by this module
|
|---|
| 1925 | ###############################################################################
|
|---|
| 1926 |
|
|---|
| 1927 | __all__ = ['DiGIR']
|
|---|
