Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Make URL path modification more robust #54

Merged
merged 3 commits into from
Mar 7, 2022
Merged

Make URL path modification more robust #54

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
90598fb
Merge pull request #44 from Kalanyr/master
Kalanyr Jun 25, 2021
0c9638c
Merge pull request #52 from Kalanyr/master
Kalanyr Feb 3, 2022
c28b231
Fix URL path modification
IdamkinI Feb 19, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 7 additions & 4 deletions gogrepoc.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,14 +41,14 @@
# python 2
from Queue import Queue
import cookielib as cookiejar
from urlparse import urlparse,unquote
from urlparse import urlparse,unquote,urlunparse
from itertools import izip_longest as zip_longest
from StringIO import StringIO
except ImportError:
# python 3
from queue import Queue
import http.cookiejar as cookiejar
from urllib.parse import urlparse, unquote
from urllib.parse import urlparse, unquote, urlunparse
from itertools import zip_longest
from io import StringIO

Expand Down Expand Up @@ -638,7 +638,7 @@ def fetch_chunk_tree(response, session):
file_ext = os.path.splitext(urlparse(response.url).path)[1].lower()
if file_ext not in SKIP_MD5_FILE_EXT:
try:
chunk_url = response.url.replace('?', '.xml?')
chunk_url = append_xml_extension_to_url_path(response.url)
chunk_response = request(session,chunk_url)
shelf_etree = xml.etree.ElementTree.fromstring(chunk_response.content)
return shelf_etree
Expand Down Expand Up @@ -668,7 +668,7 @@ def fetch_file_info(d, fetch_md5,updateSession):
file_ext = os.path.splitext(urlparse(response.url).path)[1].lower()
if file_ext not in SKIP_MD5_FILE_EXT:
try:
tmp_md5_url = response.url.replace('?', '.xml?')
tmp_md5_url = append_xml_extension_to_url_path(response.url)
md5_response = request(updateSession,tmp_md5_url)
shelf_etree = xml.etree.ElementTree.fromstring(md5_response.content)
d.md5 = shelf_etree.attrib['md5']
Expand Down Expand Up @@ -839,6 +839,9 @@ def is_numeric_id(s):
except ValueError:
return False

def append_xml_extension_to_url_path(url):
parsed = urlparse(url)
return urlunparse(parsed._replace(path = parsed.path + ".xml"))

def process_argv(argv):
p1 = argparse.ArgumentParser(description='%s (%s)' % (__appname__, __url__), add_help=False)
Expand Down