From 340ba8815036f617ecec516e5756457ad5c84ff0 Mon Sep 17 00:00:00 2001 From: CorpNewt Date: Fri, 9 Nov 2018 09:30:39 -0600 Subject: [PATCH] Fixes for py2 on Sierra --- Scripts/downloader.py | 128 ++++++++++++++++++++++++------------------ 1 file changed, 72 insertions(+), 56 deletions(-) diff --git a/Scripts/downloader.py b/Scripts/downloader.py index 80a25dd..ecf3284 100644 --- a/Scripts/downloader.py +++ b/Scripts/downloader.py @@ -1,8 +1,10 @@ -import sys, os, time +import sys, os, time, ssl # Python-aware urllib stuff if sys.version_info >= (3, 0): from urllib.request import urlopen else: + # Import urllib2 to catch errors + import urllib2 from urllib2 import urlopen class Downloader: @@ -10,6 +12,23 @@ class Downloader: def __init__(self): return + def open_url(self, url): + # Wrap up the try/except block so we don't have to do this for each function + try: + response = urlopen(url) + except Exception as e: + if sys.version_info >= (3, 0) or not (isinstance(e, urllib2.URLError) and "CERTIFICATE_VERIFY_FAILED" in str(e)): + # Either py3, or not the right error for this "fix" + return None + # Py2 and a Cert verify error - let's set the unverified context + context = ssl._create_unverified_context() + try: + response = urlopen(url, context=context) + except: + # No fixing this - bail + return None + return response + def get_size(self, size, suff=None): if size == -1: return "Unknown" @@ -45,37 +64,58 @@ class Downloader: sys.stdout.write("Downloaded {}\r".format(b_s)) def get_string(self, url, progress = True): - try: - response = urlopen(url) - CHUNK = 16 * 1024 - bytes_so_far = 0 - try: - total_size = int(response.headers['Content-Length']) - except: - total_size = -1 - chunk_so_far = "".encode("utf-8") - while True: - chunk = response.read(CHUNK) - bytes_so_far += len(chunk) - if progress: - self._progress_hook(response, bytes_so_far, total_size) - if not chunk: - break - chunk_so_far += chunk - return chunk_so_far.decode("utf-8") - except: + response = self.open_url(url) + if not response: return None + CHUNK = 16 * 1024 + bytes_so_far = 0 + try: + total_size = int(response.headers['Content-Length']) + except: + total_size = -1 + chunk_so_far = "".encode("utf-8") + while True: + chunk = response.read(CHUNK) + bytes_so_far += len(chunk) + if progress: + self._progress_hook(response, bytes_so_far, total_size) + if not chunk: + break + chunk_so_far += chunk + return chunk_so_far.decode("utf-8") def get_bytes(self, url, progress = True): + response = self.open_url(url) + if not response: + return None + CHUNK = 16 * 1024 + bytes_so_far = 0 try: - response = urlopen(url) - CHUNK = 16 * 1024 - bytes_so_far = 0 - try: - total_size = int(response.headers['Content-Length']) - except: - total_size = -1 - chunk_so_far = "".encode("utf-8") + total_size = int(response.headers['Content-Length']) + except: + total_size = -1 + chunk_so_far = "".encode("utf-8") + while True: + chunk = response.read(CHUNK) + bytes_so_far += len(chunk) + if progress: + self._progress_hook(response, bytes_so_far, total_size) + if not chunk: + break + chunk_so_far += chunk + return chunk_so_far + + def stream_to_file(self, url, file, progress = True): + response = self.open_url(url) + if not response: + return None + CHUNK = 16 * 1024 + bytes_so_far = 0 + try: + total_size = int(response.headers['Content-Length']) + except: + total_size = -1 + with open(file, 'wb') as f: while True: chunk = response.read(CHUNK) bytes_so_far += len(chunk) @@ -83,32 +123,8 @@ class Downloader: self._progress_hook(response, bytes_so_far, total_size) if not chunk: break - chunk_so_far += chunk - return chunk_so_far - except: - return None - - def stream_to_file(self, url, file, progress = True): - try: - response = urlopen(url) - CHUNK = 16 * 1024 - bytes_so_far = 0 - try: - total_size = int(response.headers['Content-Length']) - except: - total_size = -1 - with open(file, 'wb') as f: - while True: - chunk = response.read(CHUNK) - bytes_so_far += len(chunk) - if progress: - self._progress_hook(response, bytes_so_far, total_size) - if not chunk: - break - f.write(chunk) - if os.path.exists(file): - return file - else: - return None - except: + f.write(chunk) + if os.path.exists(file): + return file + else: return None