From fc7da4b8e5870497f4ca26ec39fc5e2a9239b95a Mon Sep 17 00:00:00 2001 From: corpnewt Date: Thu, 27 Dec 2018 18:11:18 -0600 Subject: [PATCH] Update downloader.py from pymodules --- Scripts/downloader.py | 260 +++++++++++++++++++++--------------------- 1 file changed, 130 insertions(+), 130 deletions(-) mode change 100644 => 100755 Scripts/downloader.py diff --git a/Scripts/downloader.py b/Scripts/downloader.py old mode 100644 new mode 100755 index ecf3284..38c3f9d --- a/Scripts/downloader.py +++ b/Scripts/downloader.py @@ -1,130 +1,130 @@ -import sys, os, time, ssl -# Python-aware urllib stuff -if sys.version_info >= (3, 0): - from urllib.request import urlopen -else: - # Import urllib2 to catch errors - import urllib2 - from urllib2 import urlopen - -class Downloader: - - def __init__(self): - return - - def open_url(self, url): - # Wrap up the try/except block so we don't have to do this for each function - try: - response = urlopen(url) - except Exception as e: - if sys.version_info >= (3, 0) or not (isinstance(e, urllib2.URLError) and "CERTIFICATE_VERIFY_FAILED" in str(e)): - # Either py3, or not the right error for this "fix" - return None - # Py2 and a Cert verify error - let's set the unverified context - context = ssl._create_unverified_context() - try: - response = urlopen(url, context=context) - except: - # No fixing this - bail - return None - return response - - def get_size(self, size, suff=None): - if size == -1: - return "Unknown" - ext = ["B","KB","MB","GB","PB"] - s = float(size) - s_dict = {} - # Iterate the ext list, and divide by 1000 each time - for e in ext: - s_dict[e] = s - s /= 1000 - if suff and suff.upper() in s_dict: - # We supplied the suffix - use it \o/ - bval = round(s_dict[suff.upper()], 2) - biggest = suff.upper() - else: - # Get the maximum >= 1 type - biggest = next((x for x in ext[::-1] if s_dict[x] >= 1), "B") - # Round to 2 decimal places - bval = round(s_dict[biggest], 2) - return "{:,.2f} {}".format(bval, biggest) - - def _progress_hook(self, response, bytes_so_far, total_size): - if total_size > 0: - percent = float(bytes_so_far) / total_size - percent = round(percent*100, 2) - t_s = self.get_size(total_size) - try: - b_s = self.get_size(bytes_so_far, t_s.split(" ")[1]) - except: - b_s = self.get_size(bytes_so_far) - sys.stdout.write("Downloaded {} of {} ({:.2f}%)\r".format(b_s, t_s, percent)) - else: - sys.stdout.write("Downloaded {}\r".format(b_s)) - - def get_string(self, url, progress = True): - response = self.open_url(url) - if not response: - return None - CHUNK = 16 * 1024 - bytes_so_far = 0 - try: - total_size = int(response.headers['Content-Length']) - except: - total_size = -1 - chunk_so_far = "".encode("utf-8") - while True: - chunk = response.read(CHUNK) - bytes_so_far += len(chunk) - if progress: - self._progress_hook(response, bytes_so_far, total_size) - if not chunk: - break - chunk_so_far += chunk - return chunk_so_far.decode("utf-8") - - def get_bytes(self, url, progress = True): - response = self.open_url(url) - if not response: - return None - CHUNK = 16 * 1024 - bytes_so_far = 0 - try: - total_size = int(response.headers['Content-Length']) - except: - total_size = -1 - chunk_so_far = "".encode("utf-8") - while True: - chunk = response.read(CHUNK) - bytes_so_far += len(chunk) - if progress: - self._progress_hook(response, bytes_so_far, total_size) - if not chunk: - break - chunk_so_far += chunk - return chunk_so_far - - def stream_to_file(self, url, file, progress = True): - response = self.open_url(url) - if not response: - return None - CHUNK = 16 * 1024 - bytes_so_far = 0 - try: - total_size = int(response.headers['Content-Length']) - except: - total_size = -1 - with open(file, 'wb') as f: - while True: - chunk = response.read(CHUNK) - bytes_so_far += len(chunk) - if progress: - self._progress_hook(response, bytes_so_far, total_size) - if not chunk: - break - f.write(chunk) - if os.path.exists(file): - return file - else: - return None +import sys, os, time, ssl +# Python-aware urllib stuff +if sys.version_info >= (3, 0): + from urllib.request import urlopen +else: + # Import urllib2 to catch errors + import urllib2 + from urllib2 import urlopen + +class Downloader: + + def __init__(self): + return + + def open_url(self, url): + # Wrap up the try/except block so we don't have to do this for each function + try: + response = urlopen(url) + except Exception as e: + if sys.version_info >= (3, 0) or not (isinstance(e, urllib2.URLError) and "CERTIFICATE_VERIFY_FAILED" in str(e)): + # Either py3, or not the right error for this "fix" + return None + # Py2 and a Cert verify error - let's set the unverified context + context = ssl._create_unverified_context() + try: + response = urlopen(url, context=context) + except: + # No fixing this - bail + return None + return response + + def get_size(self, size, suff=None): + if size == -1: + return "Unknown" + ext = ["B","KB","MB","GB","PB"] + s = float(size) + s_dict = {} + # Iterate the ext list, and divide by 1000 each time + for e in ext: + s_dict[e] = s + s /= 1000 + if suff and suff.upper() in s_dict: + # We supplied the suffix - use it \o/ + bval = round(s_dict[suff.upper()], 2) + biggest = suff.upper() + else: + # Get the maximum >= 1 type + biggest = next((x for x in ext[::-1] if s_dict[x] >= 1), "B") + # Round to 2 decimal places + bval = round(s_dict[biggest], 2) + return "{:,.2f} {}".format(bval, biggest) + + def _progress_hook(self, response, bytes_so_far, total_size): + if total_size > 0: + percent = float(bytes_so_far) / total_size + percent = round(percent*100, 2) + t_s = self.get_size(total_size) + try: + b_s = self.get_size(bytes_so_far, t_s.split(" ")[1]) + except: + b_s = self.get_size(bytes_so_far) + sys.stdout.write("Downloaded {} of {} ({:.2f}%)\r".format(b_s, t_s, percent)) + else: + sys.stdout.write("Downloaded {:,}\r".format(bytes_so_far)) + + def get_string(self, url, progress = True): + response = self.open_url(url) + if not response: + return None + CHUNK = 16 * 1024 + bytes_so_far = 0 + try: + total_size = int(response.headers['Content-Length']) + except: + total_size = -1 + chunk_so_far = "".encode("utf-8") + while True: + chunk = response.read(CHUNK) + bytes_so_far += len(chunk) + if progress: + self._progress_hook(response, bytes_so_far, total_size) + if not chunk: + break + chunk_so_far += chunk + return chunk_so_far.decode("utf-8") + + def get_bytes(self, url, progress = True): + response = self.open_url(url) + if not response: + return None + CHUNK = 16 * 1024 + bytes_so_far = 0 + try: + total_size = int(response.headers['Content-Length']) + except: + total_size = -1 + chunk_so_far = "".encode("utf-8") + while True: + chunk = response.read(CHUNK) + bytes_so_far += len(chunk) + if progress: + self._progress_hook(response, bytes_so_far, total_size) + if not chunk: + break + chunk_so_far += chunk + return chunk_so_far + + def stream_to_file(self, url, file, progress = True): + response = self.open_url(url) + if not response: + return None + CHUNK = 16 * 1024 + bytes_so_far = 0 + try: + total_size = int(response.headers['Content-Length']) + except: + total_size = -1 + with open(file, 'wb') as f: + while True: + chunk = response.read(CHUNK) + bytes_so_far += len(chunk) + if progress: + self._progress_hook(response, bytes_so_far, total_size) + if not chunk: + break + f.write(chunk) + if os.path.exists(file): + return file + else: + return None