this is the download code I now use for most of my downloads
Python
pythonVer = 2
if sys.version_info.major == 3:
pythonVer = 3
if pythonVer == 3:
from urllib.request import urlopen, Request
from urllib.error import URLError
else:
from urllib2 import urlopen, Request, URLError
def checkGZIP(url):
response = None
request = Request(url, headers=hdr)
try:
response = urlopen(request)
if response.info().get('Content-Encoding') == 'gzip':
buffer = StringIO(response.read())
deflatedContent = gzip.GzipFile(fileobj=buffer)
if pythonVer == 3:
return deflatedContent.read().decode('utf-8')
else:
return deflatedContent.read()
else:
if pythonVer == 3:
return response.read().decode('utf-8')
else:
return response.read()
except:
return None
