Hacks to make reviews endpoint work for books.

dhruvg · Jan 24, 2023 · bce6ebf · bce6ebf
1 parent fc01357
commit bce6ebf
Show file tree

Hide file tree

Showing 2 changed files with 29 additions and 6 deletions.
diff --git a/google_play_scraper/constants/request.py b/google_play_scraper/constants/request.py
@@ -42,6 +42,9 @@ def build(self, lang: str, country: str) -> str:
         PAYLOAD_FORMAT_FOR_FIRST_PAGE = "f.req=%5B%5B%5B%22UsvDTd%22%2C%22%5Bnull%2Cnull%2C%5B2%2C{sort}%2C%5B{count}%2Cnull%2Cnull%5D%2Cnull%2C%5Bnull%2C{score}%5D%5D%2C%5B%5C%22{app_id}%5C%22%2C{store_id}%5D%5D%22%2Cnull%2C%22generic%22%5D%5D%5D"
         PAYLOAD_FORMAT_FOR_PAGINATED_PAGE = "f.req=%5B%5B%5B%22UsvDTd%22%2C%22%5Bnull%2Cnull%2C%5B2%2C{sort}%2C%5B{count}%2Cnull%2C%5C%22{pagination_token}%5C%22%5D%2Cnull%2C%5Bnull%2C{score}%5D%5D%2C%5B%5C%22{app_id}%5C%22%2C{store_id}%5D%5D%22%2Cnull%2C%22generic%22%5D%5D%5D"
 
+        PAYLOAD_FORMAT_FOR_BOOKS_FIRST_PAGE = "f.req=%5B%5B%5B%22oCPfdb%22%2C%22%5Bnull%2C%5B2%2C{sort}%2C%5B{count}%2Cnull%2Cnull%5D%2Cnull%2C%5Bnull%2C{score}%5D%5D%2C%5B%5C%22{app_id}%5C%22%2C{store_id}%5D%5D%22%2Cnull%2C%22generic%22%5D%5D%5D"
+        PAYLOAD_FORMAT_FOR_BOOKS_PAGINATED_PAGE = "f.req=%5B%5B%5B%22oCPfdb%22%2C%22%5Bnull%2C%5B2%2C{sort}%2C%5B{count}%2Cnull%2C%5C%22{pagination_token}%5C%22%5D%2Cnull%2C%5Bnull%2C{score}%5D%5D%2C%5B%5C%22{app_id}%5C%22%2C{store_id}%5D%5D%22%2Cnull%2C%22generic%22%5D%5D%5D"
+
         def build_body(
             self,
             store_id: int,
@@ -51,8 +54,15 @@ def build_body(
             filter_score_with: int,
             pagination_token: str,
         ) -> bytes:
+            if store_id == 9:
+                paginated_format = self.PAYLOAD_FORMAT_FOR_BOOKS_PAGINATED_PAGE
+                first_page_format = self.PAYLOAD_FORMAT_FOR_BOOKS_FIRST_PAGE
+            else:
+                paginated_format = self.PAYLOAD_FORMAT_FOR_PAGINATED_PAGE
+                first_page_format = self.PAYLOAD_FORMAT_FOR_FIRST_PAGE
+
             if pagination_token is not None:
-                result = self.PAYLOAD_FORMAT_FOR_PAGINATED_PAGE.format(
+                result = paginated_format.format(
                     store_id=store_id,
                     app_id=app_id,
                     sort=sort,
@@ -61,7 +71,7 @@ def build_body(
                     pagination_token=pagination_token,
                 )
             else:
-                result = self.PAYLOAD_FORMAT_FOR_FIRST_PAGE.format(
+                result = first_page_format.format(
                     store_id=store_id, app_id=app_id, sort=sort, score=filter_score_with, count=count
                 )
 

diff --git a/google_play_scraper/features/reviews.py b/google_play_scraper/features/reviews.py
@@ -11,6 +11,7 @@
 
 
 MAX_COUNT_EACH_FETCH = 199
+MAX_COUNT_EACH_FETCH_BOOKS = 99
 
 
 class _ContinuationToken:
@@ -49,7 +50,15 @@ def _fetch_review_items(
 
     match = json.loads(Regex.REVIEWS.findall(dom)[0])
 
-    return json.loads(match[0][2])[0], json.loads(match[0][2])[-1][-1]
+    if store_id == 9:
+        try:
+            token = json.loads(match[0][2])[-2][-1]
+        except:
+            token = []
+    else:
+        token = json.loads(match[0][2])[-1][-1]
+
+    return json.loads(match[0][2])[0], token
 
 
 def reviews(
@@ -84,13 +93,15 @@ def reviews(
     _fetch_count = count
 
     result = []
+    max_count_per_fetch = MAX_COUNT_EACH_FETCH_BOOKS \
+        if store_id == 9 else MAX_COUNT_EACH_FETCH
 
     while True:
         if _fetch_count == 0:
             break
 
-        if _fetch_count > MAX_COUNT_EACH_FETCH:
-            _fetch_count = MAX_COUNT_EACH_FETCH
+        if _fetch_count > max_count_per_fetch:
+            _fetch_count = max_count_per_fetch
 
         try:
             review_items, token = _fetch_review_items(
@@ -127,12 +138,14 @@ def reviews_all(store_id: int, app_id: str, sleep_milliseconds: int = 0, **kwarg
     continuation_token = None
 
     result = []
+    max_count_per_fetch = MAX_COUNT_EACH_FETCH_BOOKS \
+        if store_id == 9 else MAX_COUNT_EACH_FETCH
 
     while True:
         _result, continuation_token = reviews(
             store_id,
             app_id,
-            count=MAX_COUNT_EACH_FETCH,
+            count=max_count_per_fetch,
             continuation_token=continuation_token,
             **kwargs
         )