From 472f59e2524f371f4f49aa82d4b3148f056a1f60 Mon Sep 17 00:00:00 2001 From: Kazuki Suzuki Przyborowski Date: Wed, 11 Oct 2023 20:06:35 -0500 Subject: [PATCH] Add files via upload --- pywwwget.py | 44 +++++++++++++++++++++++++++++++++++++++++++- pywwwgetold.py | 44 +++++++++++++------------------------------- 2 files changed, 56 insertions(+), 32 deletions(-) diff --git a/pywwwget.py b/pywwwget.py index 0c779e1..556a91e 100644 --- a/pywwwget.py +++ b/pywwwget.py @@ -1344,7 +1344,7 @@ def download_from_url(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpheadersentout = dict(make_http_headers_from_pycurl_to_dict("\r\n".join(httpheadersentout))); httpheadersentout = fix_header_names(httpheadersentout); log.info("Downloading URL "+httpurl); - if(httplibuse=="urllib" or httplibuse=="request" or httplibuse=="request3" or httplibuse=="aiohttp" or httplibuse=="httplib" or httplibuse=="httplib2" or httplibuse=="urllib3" or httplibuse=="mechanize" or httplibuse=="httpx" or httplibuse=="httpx2" or httplibuse=="httpcore" or httplibuse=="httpcore2"): + if(httplibuse=="urllib" or httplibuse=="request" or httplibuse=="request3" or httplibuse=="aiohttp" or httplibuse=="httplib" or httplibuse=="httplib2" or httplibuse=="urllib3" or httplibuse=="mechanize"): downloadsize = httpheaderout.get('Content-Length'); if(downloadsize is not None): downloadsize = int(downloadsize); @@ -1388,6 +1388,48 @@ def download_from_url(httpurl, httpheaders=geturls_headers, httpuseragent=None, returnval_content = zstandard.decompress(returnval_content); except zstandard.error: pass; + elif(httplibuse=="httpx" or httplibuse=="httpx2" or httplibuse=="httpcore" or httplibuse=="httpcore2"): + downloadsize = httpheaderout.get('Content-Length'); + if(downloadsize is not None): + downloadsize = int(downloadsize); + if downloadsize is None: downloadsize = 0; + fulldatasize = 0; + prevdownsize = 0; + log.info("Downloading URL "+httpurl); + with BytesIO() as strbuf + for databytes in geturls_text.iter_content(chunk_size=buffersize): + datasize = len(databytes); + fulldatasize = datasize + fulldatasize; + percentage = ""; + if(downloadsize>0): + percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%"; + downloaddiff = fulldatasize - prevdownsize; + log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']); + prevdownsize = fulldatasize; + strbuf.write(databytes); + strbuf.seek(0); + returnval_content = strbuf.read(); + geturls_text.close(); + if(httpheaderout.get("Content-Encoding")=="gzip"): + try: + returnval_content = zlib.decompress(returnval_content, 16+zlib.MAX_WBITS); + except zlib.error: + pass; + elif(httpheaderout.get("Content-Encoding")=="deflate"): + try: + returnval_content = zlib.decompress(returnval_content); + except zlib.error: + pass; + elif(httpheaderout.get("Content-Encoding")=="br" and havebrotli): + try: + returnval_content = brotli.decompress(returnval_content); + except brotli.error: + pass; + elif(httpheaderout.get("Content-Encoding")=="zstd" and havezstd): + try: + returnval_content = zstandard.decompress(returnval_content); + except zstandard.error: + pass; elif(httplibuse=="requests"): log.info("Downloading URL "+httpurl); downloadsize = httpheaderout.get('Content-Length'); diff --git a/pywwwgetold.py b/pywwwgetold.py index b0de409..f467dc2 100755 --- a/pywwwgetold.py +++ b/pywwwgetold.py @@ -129,7 +129,7 @@ __version_info__ = (2, 0, 2, "RC 1", 1); __version_date_info__ = (2023, 10, 5, "RC 1", 1); __version_date__ = str(__version_date_info__[0])+"."+str(__version_date_info__[1]).zfill(2)+"."+str(__version_date_info__[2]).zfill(2); __revision__ = __version_info__[3]; -__revision_id__ = "$Id: 346873973981d4271d1530c99011ffb8047a7d18 $"; +__revision_id__ = "$Id$"; if(__version_info__[4] is not None): __version_date_plusrc__ = __version_date__+"-"+str(__version_date_info__[4]); if(__version_info__[4] is None): @@ -975,7 +975,6 @@ def download_from_url_with_urllib(httpurl, httpheaders=geturls_headers, httpuser if(isinstance(httpheadersentout, list)): httpheadersentout = dict(make_http_headers_from_list_to_dict(httpheadersentout)); httpheadersentout = fix_header_names(httpheadersentout); - log.info("Downloading URL "+httpurl); downloadsize = httpheaderout.get('Content-Length'); if(downloadsize is not None): downloadsize = int(downloadsize); @@ -1212,7 +1211,6 @@ def download_from_url_with_httplib(httpurl, httpheaders=geturls_headers, httpuse if(isinstance(httpheadersentout, list)): httpheadersentout = dict(make_http_headers_from_list_to_dict(httpheadersentout)); httpheadersentout = fix_header_names(httpheadersentout); - log.info("Downloading URL "+httpurl); downloadsize = httpheaderout.get('Content-Length'); if(downloadsize is not None): downloadsize = int(downloadsize); @@ -1450,7 +1448,6 @@ if(havehttplib2): if(isinstance(httpheadersentout, list)): httpheadersentout = dict(make_http_headers_from_list_to_dict(httpheadersentout)); httpheadersentout = fix_header_names(httpheadersentout); - log.info("Downloading URL "+httpurl); downloadsize = httpheaderout.get('Content-Length'); if(downloadsize is not None): downloadsize = int(downloadsize); @@ -1709,7 +1706,6 @@ if(haverequests): if(isinstance(httpheadersentout, list)): httpheadersentout = dict(make_http_headers_from_list_to_dict(httpheadersentout)); httpheadersentout = fix_header_names(httpheadersentout); - log.info("Downloading URL "+httpurl); downloadsize = httpheaderout.get('Content-Length'); if(downloadsize is not None): downloadsize = int(downloadsize); @@ -1953,7 +1949,6 @@ if(haveaiohttp): if(isinstance(httpheadersentout, list)): httpheadersentout = dict(make_http_headers_from_list_to_dict(httpheadersentout)); httpheadersentout = fix_header_names(httpheadersentout); - log.info("Downloading URL "+httpurl); downloadsize = httpheaderout.get('Content-Length'); if(downloadsize is not None): downloadsize = int(downloadsize); @@ -2202,7 +2197,6 @@ if(havehttpx): if(isinstance(httpheadersentout, list)): httpheadersentout = dict(make_http_headers_from_list_to_dict(httpheadersentout)); httpheadersentout = fix_header_names(httpheadersentout); - log.info("Downloading URL "+httpurl); downloadsize = httpheaderout.get('Content-Length'); if(downloadsize is not None): downloadsize = int(downloadsize); @@ -2210,10 +2204,8 @@ if(havehttpx): fulldatasize = 0; prevdownsize = 0; log.info("Downloading URL "+httpurl); - with BytesIO() as strbuf: - while True: - databytes = geturls_text.read(buffersize); - if not databytes: break; + with BytesIO() as strbuf + for databytes in geturls_text.iter_content(chunk_size=buffersize): datasize = len(databytes); fulldatasize = datasize + fulldatasize; percentage = ""; @@ -2225,6 +2217,7 @@ if(havehttpx): strbuf.write(databytes); strbuf.seek(0); returnval_content = strbuf.read(); + geturls_text.close(); if(httpheaderout.get("Content-Encoding")=="gzip"): try: returnval_content = zlib.decompress(returnval_content, 16+zlib.MAX_WBITS); @@ -2451,7 +2444,6 @@ if(havehttpx): if(isinstance(httpheadersentout, list)): httpheadersentout = dict(make_http_headers_from_list_to_dict(httpheadersentout)); httpheadersentout = fix_header_names(httpheadersentout); - log.info("Downloading URL "+httpurl); downloadsize = httpheaderout.get('Content-Length'); if(downloadsize is not None): downloadsize = int(downloadsize); @@ -2459,10 +2451,8 @@ if(havehttpx): fulldatasize = 0; prevdownsize = 0; log.info("Downloading URL "+httpurl); - with BytesIO() as strbuf: - while True: - databytes = geturls_text.read(buffersize); - if not databytes: break; + with BytesIO() as strbuf + for databytes in geturls_text.iter_content(chunk_size=buffersize): datasize = len(databytes); fulldatasize = datasize + fulldatasize; percentage = ""; @@ -2474,6 +2464,7 @@ if(havehttpx): strbuf.write(databytes); strbuf.seek(0); returnval_content = strbuf.read(); + geturls_text.close(); if(httpheaderout.get("Content-Encoding")=="gzip"): try: returnval_content = zlib.decompress(returnval_content, 16+zlib.MAX_WBITS); @@ -2697,7 +2688,6 @@ if(havehttpcore): if(isinstance(httpheadersentout, list)): httpheadersentout = dict(make_http_headers_from_list_to_dict(httpheadersentout)); httpheadersentout = fix_header_names(httpheadersentout); - log.info("Downloading URL "+httpurl); downloadsize = httpheaderout.get('Content-Length'); if(downloadsize is not None): downloadsize = int(downloadsize); @@ -2705,10 +2695,8 @@ if(havehttpcore): fulldatasize = 0; prevdownsize = 0; log.info("Downloading URL "+httpurl); - with BytesIO() as strbuf: - while True: - databytes = geturls_text.read(buffersize); - if not databytes: break; + with BytesIO() as strbuf + for databytes in geturls_text.iter_content(chunk_size=buffersize): datasize = len(databytes); fulldatasize = datasize + fulldatasize; percentage = ""; @@ -2720,6 +2708,7 @@ if(havehttpcore): strbuf.write(databytes); strbuf.seek(0); returnval_content = strbuf.read(); + geturls_text.close(); if(httpheaderout.get("Content-Encoding")=="gzip"): try: returnval_content = zlib.decompress(returnval_content, 16+zlib.MAX_WBITS); @@ -2943,7 +2932,6 @@ if(havehttpcore): if(isinstance(httpheadersentout, list)): httpheadersentout = dict(make_http_headers_from_list_to_dict(httpheadersentout)); httpheadersentout = fix_header_names(httpheadersentout); - log.info("Downloading URL "+httpurl); downloadsize = httpheaderout.get('Content-Length'); if(downloadsize is not None): downloadsize = int(downloadsize); @@ -2951,10 +2939,8 @@ if(havehttpcore): fulldatasize = 0; prevdownsize = 0; log.info("Downloading URL "+httpurl); - with BytesIO() as strbuf: - while True: - databytes = geturls_text.read(buffersize); - if not databytes: break; + with BytesIO() as strbuf + for databytes in geturls_text.iter_content(chunk_size=buffersize): datasize = len(databytes); fulldatasize = datasize + fulldatasize; percentage = ""; @@ -2966,6 +2952,7 @@ if(havehttpcore): strbuf.write(databytes); strbuf.seek(0); returnval_content = strbuf.read(); + geturls_text.close(); if(httpheaderout.get("Content-Encoding")=="gzip"): try: returnval_content = zlib.decompress(returnval_content, 16+zlib.MAX_WBITS); @@ -3227,7 +3214,6 @@ if(haveurllib3): if(isinstance(httpheadersentout, list)): httpheadersentout = dict(make_http_headers_from_list_to_dict(httpheadersentout)); httpheadersentout = fix_header_names(httpheadersentout); - log.info("Downloading URL "+httpurl); downloadsize = httpheaderout.get('Content-Length'); if(downloadsize is not None): downloadsize = int(downloadsize); @@ -3477,7 +3463,6 @@ if(havemechanize): if(isinstance(httpheadersentout, list)): httpheadersentout = dict(make_http_headers_from_list_to_dict(httpheadersentout)); httpheadersentout = fix_header_names(httpheadersentout); - log.info("Downloading URL "+httpurl); downloadsize = httpheaderout.get('Content-Length'); if(downloadsize is not None): downloadsize = int(downloadsize); @@ -3757,7 +3742,6 @@ if(havepycurl): if(isinstance(httpheadersentout, list)): httpheadersentout = dict(make_http_headers_from_pycurl_to_dict("\r\n".join(httpheadersentout))); httpheadersentout = fix_header_names(httpheadersentout); - log.info("Downloading URL "+httpurl); downloadsize = httpheaderout.get('Content-Length'); if(downloadsize is not None): downloadsize = int(downloadsize); @@ -4040,7 +4024,6 @@ if(havepycurl and hasattr(pycurl, "CURL_HTTP_VERSION_2_0")): if(isinstance(httpheadersentout, list)): httpheadersentout = dict(make_http_headers_from_pycurl_to_dict("\r\n".join(httpheadersentout))); httpheadersentout = fix_header_names(httpheadersentout); - log.info("Downloading URL "+httpurl); downloadsize = httpheaderout.get('Content-Length'); if(downloadsize is not None): downloadsize = int(downloadsize); @@ -4338,7 +4321,6 @@ if(havepycurl and hasattr(pycurl, "CURL_HTTP_VERSION_3_0")): if(isinstance(httpheadersentout, list)): httpheadersentout = dict(make_http_headers_from_pycurl_to_dict("\r\n".join(httpheadersentout))); httpheadersentout = fix_header_names(httpheadersentout); - log.info("Downloading URL "+httpurl); downloadsize = httpheaderout.get('Content-Length'); if(downloadsize is not None): downloadsize = int(downloadsize); -- 2.11.4.GIT