From fc0135752f167f45c1f019d3c1f2f4724beb8fcd Mon Sep 17 00:00:00 2001 From: Dhruv Garg Date: Mon, 23 Jan 2023 17:46:38 +0000 Subject: [PATCH] Introduce store_id variable. --- README.md | 4 ++++ google_play_scraper/constants/request.py | 14 ++++++++------ google_play_scraper/features/permissions.py | 4 ++-- google_play_scraper/features/reviews.py | 8 ++++++-- tests/e2e_tests/test_permissions.py | 4 ++-- tests/e2e_tests/test_reviews.py | 17 ++++++++++++----- tests/e2e_tests/test_reviews_all.py | 9 +++++---- 7 files changed, 39 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 9e83299..f14d1b0 100644 --- a/README.md +++ b/README.md @@ -146,6 +146,7 @@ Result of `print(result)`: from google_play_scraper import Sort, reviews result, continuation_token = reviews( + 7, 'com.fantome.penguinisle', lang='en', # defaults to 'en' country='us', # defaults to 'us' @@ -158,6 +159,7 @@ result, continuation_token = reviews( # it will crawl the items after 3 review items. result, _ = reviews( + 7, 'com.fantome.penguinisle', continuation_token=continuation_token # defaults to None(load from the beginning) ) @@ -215,6 +217,7 @@ Result of `print(result)`: from google_play_scraper import Sort, reviews_all result = reviews_all( + 7, 'com.fantome.penguinisle', sleep_milliseconds=0, # defaults to 0 lang='en', # defaults to 'en' @@ -232,6 +235,7 @@ result = reviews_all( from google_play_scraper import permissions result = permissions( + 7, 'com.spotify.music', lang='en', # defaults to 'en' country='us', # defaults to 'us' diff --git a/google_play_scraper/constants/request.py b/google_play_scraper/constants/request.py index 9cb061b..cf99aa7 100644 --- a/google_play_scraper/constants/request.py +++ b/google_play_scraper/constants/request.py @@ -39,11 +39,12 @@ class _Reviews(Format): def build(self, lang: str, country: str) -> str: return self.URL_FORMAT.format(lang=lang, country=country) - PAYLOAD_FORMAT_FOR_FIRST_PAGE = "f.req=%5B%5B%5B%22UsvDTd%22%2C%22%5Bnull%2Cnull%2C%5B2%2C{sort}%2C%5B{count}%2Cnull%2Cnull%5D%2Cnull%2C%5Bnull%2C{score}%5D%5D%2C%5B%5C%22{app_id}%5C%22%2C7%5D%5D%22%2Cnull%2C%22generic%22%5D%5D%5D" - PAYLOAD_FORMAT_FOR_PAGINATED_PAGE = "f.req=%5B%5B%5B%22UsvDTd%22%2C%22%5Bnull%2Cnull%2C%5B2%2C{sort}%2C%5B{count}%2Cnull%2C%5C%22{pagination_token}%5C%22%5D%2Cnull%2C%5Bnull%2C{score}%5D%5D%2C%5B%5C%22{app_id}%5C%22%2C7%5D%5D%22%2Cnull%2C%22generic%22%5D%5D%5D" + PAYLOAD_FORMAT_FOR_FIRST_PAGE = "f.req=%5B%5B%5B%22UsvDTd%22%2C%22%5Bnull%2Cnull%2C%5B2%2C{sort}%2C%5B{count}%2Cnull%2Cnull%5D%2Cnull%2C%5Bnull%2C{score}%5D%5D%2C%5B%5C%22{app_id}%5C%22%2C{store_id}%5D%5D%22%2Cnull%2C%22generic%22%5D%5D%5D" + PAYLOAD_FORMAT_FOR_PAGINATED_PAGE = "f.req=%5B%5B%5B%22UsvDTd%22%2C%22%5Bnull%2Cnull%2C%5B2%2C{sort}%2C%5B{count}%2Cnull%2C%5C%22{pagination_token}%5C%22%5D%2Cnull%2C%5Bnull%2C{score}%5D%5D%2C%5B%5C%22{app_id}%5C%22%2C{store_id}%5D%5D%22%2Cnull%2C%22generic%22%5D%5D%5D" def build_body( self, + store_id: int, app_id: str, sort: int, count: int, @@ -52,6 +53,7 @@ def build_body( ) -> bytes: if pagination_token is not None: result = self.PAYLOAD_FORMAT_FOR_PAGINATED_PAGE.format( + store_id=store_id, app_id=app_id, sort=sort, count=count, @@ -60,7 +62,7 @@ def build_body( ) else: result = self.PAYLOAD_FORMAT_FOR_FIRST_PAGE.format( - app_id=app_id, sort=sort, score=filter_score_with, count=count + store_id=store_id, app_id=app_id, sort=sort, score=filter_score_with, count=count ) return result.encode() @@ -73,10 +75,10 @@ class _Permissions(Format): def build(self, lang: str, country: str) -> str: return self.URL_FORMAT.format(lang=lang, country=country) - PAYLOAD_FORMAT_FOR_PERMISSION = "f.req=%5B%5B%5B%22xdSrCf%22%2C%22%5B%5Bnull%2C%5B%5C%22{app_id}%5C%22%2C7%5D%2C%5B%5D%5D%5D%22%2Cnull%2C%221%22%5D%5D%5D" + PAYLOAD_FORMAT_FOR_PERMISSION = "f.req=%5B%5B%5B%22xdSrCf%22%2C%22%5B%5Bnull%2C%5B%5C%22{app_id}%5C%22%2C{store_id}%5D%2C%5B%5D%5D%5D%22%2Cnull%2C%221%22%5D%5D%5D" - def build_body(self, app_id: str) -> bytes: - result = self.PAYLOAD_FORMAT_FOR_PERMISSION.format(app_id=app_id) + def build_body(self, store_id: int, app_id: str) -> bytes: + result = self.PAYLOAD_FORMAT_FOR_PERMISSION.format(store_id=store_id, app_id=app_id) return result.encode() diff --git a/google_play_scraper/features/permissions.py b/google_play_scraper/features/permissions.py index 2b14a0a..acafbae 100644 --- a/google_play_scraper/features/permissions.py +++ b/google_play_scraper/features/permissions.py @@ -7,10 +7,10 @@ from google_play_scraper.constants.element import ElementSpecs -def permissions(app_id: str, lang: str = "en", country: str = "us") -> Dict[str, list]: +def permissions(store_id: int, app_id: str, lang: str = "en", country: str = "us") -> Dict[str, list]: dom = post( Formats.Permissions.build(lang=lang, country=country), - Formats.Permissions.build_body(app_id), + Formats.Permissions.build_body(store_id, app_id), {"content-type": "application/x-www-form-urlencoded"}, ) diff --git a/google_play_scraper/features/reviews.py b/google_play_scraper/features/reviews.py index e2e87ee..2a164d7 100644 --- a/google_play_scraper/features/reviews.py +++ b/google_play_scraper/features/reviews.py @@ -27,6 +27,7 @@ def __init__(self, token, lang, country, sort, count, filter_score_with): def _fetch_review_items( url: str, + store_id: int, app_id: str, sort: int, count: int, @@ -36,6 +37,7 @@ def _fetch_review_items( dom = post( url, Formats.Reviews.build_body( + store_id, app_id, sort, count, @@ -51,6 +53,7 @@ def _fetch_review_items( def reviews( + store_id: int, app_id: str, lang: str = "en", country: str = "us", @@ -91,7 +94,7 @@ def reviews( try: review_items, token = _fetch_review_items( - url, app_id, sort, _fetch_count, filter_score_with, token + url, store_id, app_id, sort, _fetch_count, filter_score_with, token ) except (TypeError, IndexError): token = None @@ -117,7 +120,7 @@ def reviews( ) -def reviews_all(app_id: str, sleep_milliseconds: int = 0, **kwargs) -> list: +def reviews_all(store_id: int, app_id: str, sleep_milliseconds: int = 0, **kwargs) -> list: kwargs.pop("count", None) kwargs.pop("continuation_token", None) @@ -127,6 +130,7 @@ def reviews_all(app_id: str, sleep_milliseconds: int = 0, **kwargs) -> list: while True: _result, continuation_token = reviews( + store_id, app_id, count=MAX_COUNT_EACH_FETCH, continuation_token=continuation_token, diff --git a/tests/e2e_tests/test_permissions.py b/tests/e2e_tests/test_permissions.py index ad885e2..5cdb284 100644 --- a/tests/e2e_tests/test_permissions.py +++ b/tests/e2e_tests/test_permissions.py @@ -5,7 +5,7 @@ class TestPermission(TestCase): def test_reply_data_all_types(self): - result = permissions("com.spotify.music", lang="en", country="us") + result = permissions(7, "com.spotify.music", lang="en", country="us") self.assertDictEqual( { @@ -46,7 +46,7 @@ def test_reply_data_all_types(self): ) def test_reply_data_only_other_type(self): - result = permissions("example.matharithmetics", lang="en", country="us") + result = permissions(7, "example.matharithmetics", lang="en", country="us") self.assertDictEqual( { diff --git a/tests/e2e_tests/test_reviews.py b/tests/e2e_tests/test_reviews.py index e4f0b66..da95010 100644 --- a/tests/e2e_tests/test_reviews.py +++ b/tests/e2e_tests/test_reviews.py @@ -14,6 +14,7 @@ class TestReviews(TestCase): def test_sort_by_newest(self): result, pagination_token = reviews( + 7, "com.mojang.minecraftpe", sort=Sort.NEWEST, count=500 ) @@ -57,7 +58,7 @@ def test_sort_by_newest(self): self.assertTrue(well_sorted_review_count > 490) def test_sort_by_most_relevant(self): - result, _ = reviews("com.mojang.minecraftpe", sort=Sort.MOST_RELEVANT, count=30) + result, _ = reviews(7, "com.mojang.minecraftpe", sort=Sort.MOST_RELEVANT, count=30) self.assertEqual(30, len(result)) @@ -72,6 +73,7 @@ def test_sort_by_most_relevant(self): def test_score_filter(self): for score in {1, 2, 3, 4, 5}: result, _ = reviews( + 7, "com.mojang.minecraftpe", sort=Sort.NEWEST, count=300, @@ -86,6 +88,7 @@ def test_reply_data(self): """ result, _ = reviews( + 7, "com.ekkorr.endlessfrontier", lang="ko", country="kr", @@ -122,7 +125,7 @@ def test_review_count_is_under_count_of_first_request(self): tests length of results of first request is lower than specified count argument """ - result, ct = reviews("com.ekkorr.endlessfrontier") + result, ct = reviews(7, "com.ekkorr.endlessfrontier") self.assertTrue(len(result) < 100) @@ -133,7 +136,7 @@ def test_continuation_token(self): tests continuation_token parameter """ - result, continuation_token = reviews("com.mojang.minecraftpe") + result, continuation_token = reviews(7, "com.mojang.minecraftpe") self.assertEqual(100, len(result)) self.assertIsNotNone(continuation_token) @@ -147,6 +150,7 @@ def test_continuation_token(self): last_review_at = result[0]["at"] result, _ = reviews( + 7, "com.mojang.minecraftpe", continuation_token=continuation_token ) @@ -164,6 +168,7 @@ def test_continuation_token_preserves_argument_info(self): """ result, continuation_token = reviews( + 7, "com.mojang.minecraftpe", lang="ko", country="kr", @@ -185,7 +190,7 @@ def test_continuation_token_preserves_argument_info(self): "google_play_scraper.features.reviews._fetch_review_items", wraps=_fetch_review_items, ) as m: - _ = reviews("com.mojang.minecraftpe", continuation_token=continuation_token) + _ = reviews(7, "com.mojang.minecraftpe", continuation_token=continuation_token) self.assertEqual("hl=ko&gl=kr", urlparse(m.call_args[0][0]).query) self.assertEqual(Sort.MOST_RELEVANT, m.call_args[0][2]) @@ -204,6 +209,7 @@ def test_priority_between_preserved_argument_of_continuation_token_and_specified wraps=_fetch_review_items, ) as m: _ = reviews( + 7, "com.mojang.minecraftpe", continuation_token=_ContinuationToken( "", "ko", "kr", Sort.MOST_RELEVANT, 10, 5 @@ -222,6 +228,7 @@ def test_priority_between_preserved_argument_of_continuation_token_and_specified def test_invalid_continuation_token(self): result, ct = reviews( + 7, "com.mojang.minecraftpe", continuation_token=_ContinuationToken( "foo", "ko", "kr", Sort.MOST_RELEVANT, 10, 5 @@ -238,7 +245,7 @@ def test_invalid_continuation_token(self): self.assertEqual(5, ct.filter_score_with) def test_no_reviews(self): - result, ct = reviews("product.dp.io.ab180blog", lang="sw", country="it") + result, ct = reviews(7, "product.dp.io.ab180blog", lang="sw", country="it") self.assertListEqual([], result) diff --git a/tests/e2e_tests/test_reviews_all.py b/tests/e2e_tests/test_reviews_all.py index f028c59..c7e747f 100644 --- a/tests/e2e_tests/test_reviews_all.py +++ b/tests/e2e_tests/test_reviews_all.py @@ -9,10 +9,10 @@ def test_request_once(self): with patch( "google_play_scraper.features.reviews.reviews", wraps=reviews ) as mock_reviews: - result = reviews_all("co.kr.uaram.userdeliver_") + result = reviews_all(7, "co.kr.uaram.userdeliver_") self.assertEqual(1, mock_reviews.call_count) - result_of_reviews, _ = reviews("co.kr.uaram.userdeliver_", count=10000) + result_of_reviews, _ = reviews(7, "co.kr.uaram.userdeliver_", count=10000) self.assertTrue(0 < len(result) < 10) self.assertEqual(len(result), len(result_of_reviews)) @@ -21,10 +21,11 @@ def test_request_multiple_times(self): with patch( "google_play_scraper.features.reviews.reviews", wraps=reviews ) as mock_reviews: - result = reviews_all("co.kr.uaram.userdeliver_", lang="ko", country="kr") + result = reviews_all(7, "co.kr.uaram.userdeliver_", lang="ko", country="kr") self.assertEqual(3, mock_reviews.call_count) result_of_reviews, _ = reviews( + 7, "co.kr.uaram.userdeliver_", lang="ko", country="kr", count=10000 ) @@ -32,6 +33,6 @@ def test_request_multiple_times(self): self.assertEqual(len(result), len(result_of_reviews)) def test_no_reviews(self): - result = reviews_all("product.dp.io.ab180blog", lang="sw", country="it") + result = reviews_all(7, "product.dp.io.ab180blog", lang="sw", country="it") self.assertListEqual([], result)