From ac77935005ba75c0b95f6dcb5b5308163794147e Mon Sep 17 00:00:00 2001
From: Brooke McKim <brooke.mckim@gmail.com>
Date: Mon, 30 Sep 2024 09:28:53 -0400
Subject: [PATCH] fix: Optionally unzip.

---
 requirements.txt | 2 +-
 scrape.py        | 9 ++++++++-
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index 624d45c3..59f5ecdc 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -6,4 +6,4 @@ lxml
 requests
 six
 boto3
-pyyaml
+pyyaml
\ No newline at end of file
diff --git a/scrape.py b/scrape.py
index 5a8c9889..7fd8bb84 100755
--- a/scrape.py
+++ b/scrape.py
@@ -3,6 +3,7 @@
 import re
 import json
 import locale
+import gzip
 import ec2
 import os
 import requests
@@ -252,7 +253,13 @@ def add_pricing_info(instances):
 
 
 def fetch_data(url):
-    content = urllib2.urlopen(url).read().decode()
+    response = urllib2.urlopen(url).read()
+
+    try:
+        content = response.decode()
+    except UnicodeDecodeError:
+        content = gzip.decompress(response).decode()
+
     try:
         pricing = json.loads(content)
     except ValueError: