Skip to content

Commit

Permalink
Introduce store_id variable.
Browse files Browse the repository at this point in the history
  • Loading branch information
dhruvg committed Jan 23, 2023
1 parent d27568c commit fc01357
Show file tree
Hide file tree
Showing 7 changed files with 39 additions and 21 deletions.
4 changes: 4 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,7 @@ Result of `print(result)`:
from google_play_scraper import Sort, reviews

result, continuation_token = reviews(
7,
'com.fantome.penguinisle',
lang='en', # defaults to 'en'
country='us', # defaults to 'us'
Expand All @@ -158,6 +159,7 @@ result, continuation_token = reviews(
# it will crawl the items after 3 review items.

result, _ = reviews(
7,
'com.fantome.penguinisle',
continuation_token=continuation_token # defaults to None(load from the beginning)
)
Expand Down Expand Up @@ -215,6 +217,7 @@ Result of `print(result)`:
from google_play_scraper import Sort, reviews_all

result = reviews_all(
7,
'com.fantome.penguinisle',
sleep_milliseconds=0, # defaults to 0
lang='en', # defaults to 'en'
Expand All @@ -232,6 +235,7 @@ result = reviews_all(
from google_play_scraper import permissions

result = permissions(
7,
'com.spotify.music',
lang='en', # defaults to 'en'
country='us', # defaults to 'us'
Expand Down
14 changes: 8 additions & 6 deletions google_play_scraper/constants/request.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,11 +39,12 @@ class _Reviews(Format):
def build(self, lang: str, country: str) -> str:
return self.URL_FORMAT.format(lang=lang, country=country)

PAYLOAD_FORMAT_FOR_FIRST_PAGE = "f.req=%5B%5B%5B%22UsvDTd%22%2C%22%5Bnull%2Cnull%2C%5B2%2C{sort}%2C%5B{count}%2Cnull%2Cnull%5D%2Cnull%2C%5Bnull%2C{score}%5D%5D%2C%5B%5C%22{app_id}%5C%22%2C7%5D%5D%22%2Cnull%2C%22generic%22%5D%5D%5D"
PAYLOAD_FORMAT_FOR_PAGINATED_PAGE = "f.req=%5B%5B%5B%22UsvDTd%22%2C%22%5Bnull%2Cnull%2C%5B2%2C{sort}%2C%5B{count}%2Cnull%2C%5C%22{pagination_token}%5C%22%5D%2Cnull%2C%5Bnull%2C{score}%5D%5D%2C%5B%5C%22{app_id}%5C%22%2C7%5D%5D%22%2Cnull%2C%22generic%22%5D%5D%5D"
PAYLOAD_FORMAT_FOR_FIRST_PAGE = "f.req=%5B%5B%5B%22UsvDTd%22%2C%22%5Bnull%2Cnull%2C%5B2%2C{sort}%2C%5B{count}%2Cnull%2Cnull%5D%2Cnull%2C%5Bnull%2C{score}%5D%5D%2C%5B%5C%22{app_id}%5C%22%2C{store_id}%5D%5D%22%2Cnull%2C%22generic%22%5D%5D%5D"
PAYLOAD_FORMAT_FOR_PAGINATED_PAGE = "f.req=%5B%5B%5B%22UsvDTd%22%2C%22%5Bnull%2Cnull%2C%5B2%2C{sort}%2C%5B{count}%2Cnull%2C%5C%22{pagination_token}%5C%22%5D%2Cnull%2C%5Bnull%2C{score}%5D%5D%2C%5B%5C%22{app_id}%5C%22%2C{store_id}%5D%5D%22%2Cnull%2C%22generic%22%5D%5D%5D"

def build_body(
self,
store_id: int,
app_id: str,
sort: int,
count: int,
Expand All @@ -52,6 +53,7 @@ def build_body(
) -> bytes:
if pagination_token is not None:
result = self.PAYLOAD_FORMAT_FOR_PAGINATED_PAGE.format(
store_id=store_id,
app_id=app_id,
sort=sort,
count=count,
Expand All @@ -60,7 +62,7 @@ def build_body(
)
else:
result = self.PAYLOAD_FORMAT_FOR_FIRST_PAGE.format(
app_id=app_id, sort=sort, score=filter_score_with, count=count
store_id=store_id, app_id=app_id, sort=sort, score=filter_score_with, count=count
)

return result.encode()
Expand All @@ -73,10 +75,10 @@ class _Permissions(Format):
def build(self, lang: str, country: str) -> str:
return self.URL_FORMAT.format(lang=lang, country=country)

PAYLOAD_FORMAT_FOR_PERMISSION = "f.req=%5B%5B%5B%22xdSrCf%22%2C%22%5B%5Bnull%2C%5B%5C%22{app_id}%5C%22%2C7%5D%2C%5B%5D%5D%5D%22%2Cnull%2C%221%22%5D%5D%5D"
PAYLOAD_FORMAT_FOR_PERMISSION = "f.req=%5B%5B%5B%22xdSrCf%22%2C%22%5B%5Bnull%2C%5B%5C%22{app_id}%5C%22%2C{store_id}%5D%2C%5B%5D%5D%5D%22%2Cnull%2C%221%22%5D%5D%5D"

def build_body(self, app_id: str) -> bytes:
result = self.PAYLOAD_FORMAT_FOR_PERMISSION.format(app_id=app_id)
def build_body(self, store_id: int, app_id: str) -> bytes:
result = self.PAYLOAD_FORMAT_FOR_PERMISSION.format(store_id=store_id, app_id=app_id)

return result.encode()

Expand Down
4 changes: 2 additions & 2 deletions google_play_scraper/features/permissions.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,10 @@
from google_play_scraper.constants.element import ElementSpecs


def permissions(app_id: str, lang: str = "en", country: str = "us") -> Dict[str, list]:
def permissions(store_id: int, app_id: str, lang: str = "en", country: str = "us") -> Dict[str, list]:
dom = post(
Formats.Permissions.build(lang=lang, country=country),
Formats.Permissions.build_body(app_id),
Formats.Permissions.build_body(store_id, app_id),
{"content-type": "application/x-www-form-urlencoded"},
)

Expand Down
8 changes: 6 additions & 2 deletions google_play_scraper/features/reviews.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ def __init__(self, token, lang, country, sort, count, filter_score_with):

def _fetch_review_items(
url: str,
store_id: int,
app_id: str,
sort: int,
count: int,
Expand All @@ -36,6 +37,7 @@ def _fetch_review_items(
dom = post(
url,
Formats.Reviews.build_body(
store_id,
app_id,
sort,
count,
Expand All @@ -51,6 +53,7 @@ def _fetch_review_items(


def reviews(
store_id: int,
app_id: str,
lang: str = "en",
country: str = "us",
Expand Down Expand Up @@ -91,7 +94,7 @@ def reviews(

try:
review_items, token = _fetch_review_items(
url, app_id, sort, _fetch_count, filter_score_with, token
url, store_id, app_id, sort, _fetch_count, filter_score_with, token
)
except (TypeError, IndexError):
token = None
Expand All @@ -117,7 +120,7 @@ def reviews(
)


def reviews_all(app_id: str, sleep_milliseconds: int = 0, **kwargs) -> list:
def reviews_all(store_id: int, app_id: str, sleep_milliseconds: int = 0, **kwargs) -> list:
kwargs.pop("count", None)
kwargs.pop("continuation_token", None)

Expand All @@ -127,6 +130,7 @@ def reviews_all(app_id: str, sleep_milliseconds: int = 0, **kwargs) -> list:

while True:
_result, continuation_token = reviews(
store_id,
app_id,
count=MAX_COUNT_EACH_FETCH,
continuation_token=continuation_token,
Expand Down
4 changes: 2 additions & 2 deletions tests/e2e_tests/test_permissions.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

class TestPermission(TestCase):
def test_reply_data_all_types(self):
result = permissions("com.spotify.music", lang="en", country="us")
result = permissions(7, "com.spotify.music", lang="en", country="us")

self.assertDictEqual(
{
Expand Down Expand Up @@ -46,7 +46,7 @@ def test_reply_data_all_types(self):
)

def test_reply_data_only_other_type(self):
result = permissions("example.matharithmetics", lang="en", country="us")
result = permissions(7, "example.matharithmetics", lang="en", country="us")

self.assertDictEqual(
{
Expand Down
17 changes: 12 additions & 5 deletions tests/e2e_tests/test_reviews.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
class TestReviews(TestCase):
def test_sort_by_newest(self):
result, pagination_token = reviews(
7,
"com.mojang.minecraftpe", sort=Sort.NEWEST, count=500
)

Expand Down Expand Up @@ -57,7 +58,7 @@ def test_sort_by_newest(self):
self.assertTrue(well_sorted_review_count > 490)

def test_sort_by_most_relevant(self):
result, _ = reviews("com.mojang.minecraftpe", sort=Sort.MOST_RELEVANT, count=30)
result, _ = reviews(7, "com.mojang.minecraftpe", sort=Sort.MOST_RELEVANT, count=30)

self.assertEqual(30, len(result))

Expand All @@ -72,6 +73,7 @@ def test_sort_by_most_relevant(self):
def test_score_filter(self):
for score in {1, 2, 3, 4, 5}:
result, _ = reviews(
7,
"com.mojang.minecraftpe",
sort=Sort.NEWEST,
count=300,
Expand All @@ -86,6 +88,7 @@ def test_reply_data(self):
"""

result, _ = reviews(
7,
"com.ekkorr.endlessfrontier",
lang="ko",
country="kr",
Expand Down Expand Up @@ -122,7 +125,7 @@ def test_review_count_is_under_count_of_first_request(self):
tests length of results of first request is lower than specified count argument
"""

result, ct = reviews("com.ekkorr.endlessfrontier")
result, ct = reviews(7, "com.ekkorr.endlessfrontier")

self.assertTrue(len(result) < 100)

Expand All @@ -133,7 +136,7 @@ def test_continuation_token(self):
tests continuation_token parameter
"""

result, continuation_token = reviews("com.mojang.minecraftpe")
result, continuation_token = reviews(7, "com.mojang.minecraftpe")

self.assertEqual(100, len(result))
self.assertIsNotNone(continuation_token)
Expand All @@ -147,6 +150,7 @@ def test_continuation_token(self):
last_review_at = result[0]["at"]

result, _ = reviews(
7,
"com.mojang.minecraftpe", continuation_token=continuation_token
)

Expand All @@ -164,6 +168,7 @@ def test_continuation_token_preserves_argument_info(self):
"""

result, continuation_token = reviews(
7,
"com.mojang.minecraftpe",
lang="ko",
country="kr",
Expand All @@ -185,7 +190,7 @@ def test_continuation_token_preserves_argument_info(self):
"google_play_scraper.features.reviews._fetch_review_items",
wraps=_fetch_review_items,
) as m:
_ = reviews("com.mojang.minecraftpe", continuation_token=continuation_token)
_ = reviews(7, "com.mojang.minecraftpe", continuation_token=continuation_token)

self.assertEqual("hl=ko&gl=kr", urlparse(m.call_args[0][0]).query)
self.assertEqual(Sort.MOST_RELEVANT, m.call_args[0][2])
Expand All @@ -204,6 +209,7 @@ def test_priority_between_preserved_argument_of_continuation_token_and_specified
wraps=_fetch_review_items,
) as m:
_ = reviews(
7,
"com.mojang.minecraftpe",
continuation_token=_ContinuationToken(
"", "ko", "kr", Sort.MOST_RELEVANT, 10, 5
Expand All @@ -222,6 +228,7 @@ def test_priority_between_preserved_argument_of_continuation_token_and_specified

def test_invalid_continuation_token(self):
result, ct = reviews(
7,
"com.mojang.minecraftpe",
continuation_token=_ContinuationToken(
"foo", "ko", "kr", Sort.MOST_RELEVANT, 10, 5
Expand All @@ -238,7 +245,7 @@ def test_invalid_continuation_token(self):
self.assertEqual(5, ct.filter_score_with)

def test_no_reviews(self):
result, ct = reviews("product.dp.io.ab180blog", lang="sw", country="it")
result, ct = reviews(7, "product.dp.io.ab180blog", lang="sw", country="it")

self.assertListEqual([], result)

Expand Down
9 changes: 5 additions & 4 deletions tests/e2e_tests/test_reviews_all.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,10 @@ def test_request_once(self):
with patch(
"google_play_scraper.features.reviews.reviews", wraps=reviews
) as mock_reviews:
result = reviews_all("co.kr.uaram.userdeliver_")
result = reviews_all(7, "co.kr.uaram.userdeliver_")
self.assertEqual(1, mock_reviews.call_count)

result_of_reviews, _ = reviews("co.kr.uaram.userdeliver_", count=10000)
result_of_reviews, _ = reviews(7, "co.kr.uaram.userdeliver_", count=10000)

self.assertTrue(0 < len(result) < 10)
self.assertEqual(len(result), len(result_of_reviews))
Expand All @@ -21,17 +21,18 @@ def test_request_multiple_times(self):
with patch(
"google_play_scraper.features.reviews.reviews", wraps=reviews
) as mock_reviews:
result = reviews_all("co.kr.uaram.userdeliver_", lang="ko", country="kr")
result = reviews_all(7, "co.kr.uaram.userdeliver_", lang="ko", country="kr")
self.assertEqual(3, mock_reviews.call_count)

result_of_reviews, _ = reviews(
7,
"co.kr.uaram.userdeliver_", lang="ko", country="kr", count=10000
)

self.assertTrue(300 < len(result) < 500)
self.assertEqual(len(result), len(result_of_reviews))

def test_no_reviews(self):
result = reviews_all("product.dp.io.ab180blog", lang="sw", country="it")
result = reviews_all(7, "product.dp.io.ab180blog", lang="sw", country="it")

self.assertListEqual([], result)

0 comments on commit fc01357

Please sign in to comment.