From 4a8e4eb0664748051723c911a543c330483114a5 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Sat, 10 Aug 2024 04:04:10 -0400 Subject: [PATCH 01/37] Clarify README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5521f0a3..1489d9d8 100644 --- a/README.md +++ b/README.md @@ -89,8 +89,8 @@ `(ex: http://user:pass@host:port)` - `-cv/--chromeversion` to use a specific version of chrome `(ex: 118)` -- `-da/--disable-apprise` to disable Apprise notification, overriding [config.yaml](config.yaml). Useful when running - manually as opposed to on a schedule. +- `-da/--disable-apprise` disables Apprise notifications for the session, overriding [config.yaml](config.yaml). + Useful when running manually as opposed to on a schedule. - `-t/--searchtype` to only do `desktop` or `mobile` searches, `(ex: --searchtype=mobile)` ## Features From 39814ad1d558eddf0a1184cf92b96af988e704af Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Sat, 10 Aug 2024 04:04:52 -0400 Subject: [PATCH 02/37] Remove from .gitignore since already committed --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index 2c4f2fa5..3a797e99 100644 --- a/.gitignore +++ b/.gitignore @@ -179,7 +179,6 @@ pyrightconfig.json # Custom rules (everything added below won't be overridden by 'Generate .gitignore File' if you use 'Update' option) accounts.json -config.yaml sessions logs runbot.bat From 4c3549f84916e678a4511f8ccaadd7264f7161f1 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Sat, 10 Aug 2024 04:13:05 -0400 Subject: [PATCH 03/37] Add private config and move --- .gitignore | 1 + .template-config-private.yaml | 5 +++++ config.yaml | 3 --- src/utils.py | 13 +++++++++---- 4 files changed, 15 insertions(+), 7 deletions(-) create mode 100644 .template-config-private.yaml diff --git a/.gitignore b/.gitignore index 3a797e99..7302770d 100644 --- a/.gitignore +++ b/.gitignore @@ -186,3 +186,4 @@ runbot.bat /google_trends.dat /google_trends.dir /google_trends.bak +/config-private.yaml diff --git a/.template-config-private.yaml b/.template-config-private.yaml new file mode 100644 index 00000000..60b62b9d --- /dev/null +++ b/.template-config-private.yaml @@ -0,0 +1,5 @@ +# config-private.yaml +# Copy this file to config-private.yaml to use +apprise: + urls: + - 'discord://WebhookID/WebhookToken' # Replace with your actual Apprise service URLs diff --git a/config.yaml b/config.yaml index bbc9a93f..7e45c418 100644 --- a/config.yaml +++ b/config.yaml @@ -1,9 +1,6 @@ # config.yaml apprise: summary: ON_ERROR - urls: - - 'discord://WebhookID/WebhookToken' # Replace with your actual Apprise service URLs -attempts: retries: base_delay_in_seconds: 14.0625 # base_delay_in_seconds * 2^max = 14.0625 * 2^6 = 900 = 15 minutes max: 8 diff --git a/src/utils.py b/src/utils.py index deab1289..dd99a49f 100644 --- a/src/utils.py +++ b/src/utils.py @@ -42,16 +42,21 @@ def getProjectRoot() -> Path: return Path(__file__).parent.parent @staticmethod - def loadConfig(config_file=getProjectRoot() / "config.yaml") -> dict: - with open(config_file, "r") as file: - return yaml.safe_load(file) + def loadConfig(configFilename="config.yaml") -> dict: + configFile = Utils.getProjectRoot() / configFilename + try: + with open(configFile, "r") as file: + return yaml.safe_load(file) + except OSError: + logging.warning(f"{configFilename} doesn't exist") + return {} @staticmethod def sendNotification(title, body) -> None: if Utils.args.disable_apprise: return apprise = Apprise() - urls: list[str] = Utils.loadConfig().get("apprise", {}).get("urls", []) + urls: list[str] = Utils.loadConfig("config-private.yaml").get("apprise", {}).get("urls", []) for url in urls: apprise.add(url) apprise.notify(body=body, title=title) From f6ae9eec9bae2a18a0aea6fc4654e1c3b69b7bab Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Sat, 10 Aug 2024 04:13:41 -0400 Subject: [PATCH 04/37] Send batch notification for all failed promos --- src/morePromotions.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/morePromotions.py b/src/morePromotions.py index 3fe19d29..10022b2e 100644 --- a/src/morePromotions.py +++ b/src/morePromotions.py @@ -25,6 +25,7 @@ def completeMorePromotions(self): "morePromotions" ] self.browser.utils.goToRewards() + incompletePromotions: list[tuple[str, str]] = [] for promotion in morePromotions: try: promotionTitle = promotion["title"].replace("\u200b", "").replace("\xa0", " ") @@ -97,14 +98,9 @@ def completeMorePromotions(self): self.browser.webdriver.execute_script("window.scrollTo(0, 1080)") time.sleep(random.randint(5, 10)) - # todo Bundle this into one notification pointsAfter = self.browser.utils.getAccountPoints() - if pointsBefore == pointsAfter: - Utils.sendNotification( - "Incomplete promotion", - f"title={promotionTitle} type={promotion['promotionType']}", - ) - + if pointsBefore >= pointsAfter: + incompletePromotions.append((promotionTitle, promotion["promotionType"])) self.browser.utils.resetTabs() time.sleep(2) except Exception: # pylint: disable=broad-except @@ -112,4 +108,6 @@ def completeMorePromotions(self): # Reset tabs in case of an exception self.browser.utils.resetTabs() continue + if incompletePromotions: + Utils.sendNotification("Incomplete promotions(s)", incompletePromotions) logging.info("[MORE PROMOS] Exiting") From fe24e5874d8528d8d6a1a0dd07eefabf0b6cee4f Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Sat, 10 Aug 2024 04:13:55 -0400 Subject: [PATCH 05/37] Fix promo --- src/morePromotions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/morePromotions.py b/src/morePromotions.py index 10022b2e..0986c438 100644 --- a/src/morePromotions.py +++ b/src/morePromotions.py @@ -70,7 +70,7 @@ def completeMorePromotions(self): searchbar.send_keys("directions to new york") searchbar.submit() elif "Too tired to cook tonight?" in promotionTitle: - searchbar.send_keys("pizza delivery near me") + searchbar.send_keys("mcdonalds") searchbar.submit() elif "Quickly convert your money" in promotionTitle: searchbar.send_keys("convert 374 usd to yen") From 92dcfb5b76930fa222167cadf5f70d1549b74921 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Sat, 10 Aug 2024 04:14:24 -0400 Subject: [PATCH 06/37] Use default session, add some todos --- src/searches.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/searches.py b/src/searches.py index 3259751c..a294fe6d 100644 --- a/src/searches.py +++ b/src/searches.py @@ -97,7 +97,7 @@ def getGoogleTrends(self, wordsCount: int) -> list[str]: f"https://trends.google.com/trends/api/dailytrends?hl={self.browser.localeLang}" f'&ed={(date.today() - timedelta(days=i)).strftime("%Y%m%d")}&geo={self.browser.localeGeo}&ns=15' ) - assert r.status_code == requests.codes.ok + assert r.status_code == requests.codes.ok # todo Add guidance if assertion fails trends = json.loads(r.text[6:]) for topic in trends["default"]["trendingSearchesDays"][0][ "trendingSearches" @@ -113,10 +113,10 @@ def getGoogleTrends(self, wordsCount: int) -> list[str]: def getRelatedTerms(self, term: str) -> list[str]: # Function to retrieve related terms from Bing API - relatedTerms: list[str] = requests.get( + relatedTerms: list[str] = Utils.makeRequestsSession().get( f"https://api.bing.com/osjson.aspx?query={term}", headers={"User-agent": self.browser.userAgent}, - ).json()[1] + ).json()[1] # todo Wrap if failed, or assert response? if not relatedTerms: return [term] return relatedTerms From 6b9a51a591be07b4c4455ad675fda65ea6c124fb Mon Sep 17 00:00:00 2001 From: Marvin <18004362+GCMarvin@users.noreply.github.com> Date: Tue, 13 Aug 2024 12:55:48 +0200 Subject: [PATCH 07/37] Implement support for two factor logins Added reading TOTP token from accounts.json. Automatic generation of OTP from token is handled by PyOTP. For clarification, changed all previous references to "2FA" to "passwordless". --- accounts.json.sample | 2 ++ requirements.txt | 1 + src/account.py | 1 + src/browser.py | 1 + src/login.py | 45 +++++++++++++++++++++++++++++++++----------- 5 files changed, 39 insertions(+), 11 deletions(-) diff --git a/accounts.json.sample b/accounts.json.sample index b341c57f..9aceda60 100644 --- a/accounts.json.sample +++ b/accounts.json.sample @@ -2,11 +2,13 @@ { "username": "Your Email 1", "password": "Your Password 1", + "totp": "0123 4567 89ab cdef", "proxy": "http://user:pass@host1:port" }, { "username": "Your Email 2", "password": "Your Password 2", + "totp": "0123 4567 89ab cdef", "proxy": "http://user:pass@host2:port" } ] diff --git a/requirements.txt b/requirements.txt index 074533cf..f22b270a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,3 +12,4 @@ pyyaml~=6.0.2 urllib3>=2.2.2 # not directly required, pinned by Snyk to avoid a vulnerability requests-oauthlib zipp>=3.19.1 # not directly required, pinned by Snyk to avoid a vulnerability +pyotp diff --git a/src/account.py b/src/account.py index 35773dd6..3ec30515 100644 --- a/src/account.py +++ b/src/account.py @@ -5,4 +5,5 @@ class Account: username: str password: str + totp: str | None = None proxy: str | None = None diff --git a/src/browser.py b/src/browser.py index ad95b509..9cee35a9 100644 --- a/src/browser.py +++ b/src/browser.py @@ -40,6 +40,7 @@ def __init__( self.headless = not args.visible self.username = account.username self.password = account.password + self.totp = account.totp self.localeLang, self.localeGeo = self.getCCodeLang(args.lang, args.geo) self.proxy = None if args.proxy: diff --git a/src/login.py b/src/login.py index 93426c65..756d853c 100644 --- a/src/login.py +++ b/src/login.py @@ -3,6 +3,7 @@ import logging from argparse import Namespace +from pyotp import TOTP from selenium.common import TimeoutException from selenium.webdriver.common.by import By from undetected_chromedriver import Chrome @@ -42,13 +43,13 @@ def executeLogin(self) -> None: self.utils.waitUntilClickable(By.ID, "idSIButton9").click() # noinspection PyUnusedLocal - isTwoFactorEnabled: bool = False + isPasswordlessEnabled: bool = False with contextlib.suppress(TimeoutException): self.utils.waitUntilVisible(By.ID, "pushNotificationsTitle") - isTwoFactorEnabled = True - logging.debug(f"isTwoFactorEnabled = {isTwoFactorEnabled}") + isPasswordlessEnabled = True + logging.debug(f"isPasswordlessEnabled = {isPasswordlessEnabled}") - if isTwoFactorEnabled: + if isPasswordlessEnabled: # todo - Handle 2FA when running headless assert ( self.args.visible @@ -58,13 +59,6 @@ def executeLogin(self) -> None: ) input() - with contextlib.suppress( - TimeoutException - ): # In case user clicked stay signed in - self.utils.waitUntilVisible( - By.NAME, "kmsiForm" - ) # kmsi = keep me signed form - self.utils.waitUntilClickable(By.ID, "acceptButton").click() else: passwordField = self.utils.waitUntilClickable(By.NAME, "passwd") logging.info("[LOGIN] Entering password...") @@ -73,6 +67,35 @@ def executeLogin(self) -> None: assert passwordField.get_attribute("value") == self.browser.password self.utils.waitUntilClickable(By.ID, "idSIButton9").click() + # noinspection PyUnusedLocal + isTwoFactorEnabled: bool = False + with contextlib.suppress(TimeoutException): + self.utils.waitUntilVisible(By.ID, "idTxtBx_SAOTCC_OTC") + isTwoFactorEnabled = True + logging.debug(f"isTwoFactorEnabled = {isTwoFactorEnabled}") + + if isTwoFactorEnabled: + if self.browser.totp is not None: + # TOTP token provided + logging.info("[LOGIN] Entering OTP...") + otp = TOTP(self.browser.totp.replace(" ", "")).now() + otpField = self.utils.waitUntilClickable(By.ID, "idTxtBx_SAOTCC_OTC") + otpField.send_keys(otp) + assert otpField.get_attribute("value") == otp + self.utils.waitUntilClickable(By.ID, "idSubmit_SAOTCC_Continue").click() + else: + # No TOTP token provided, manual intervention required + assert ( + self.args.visible + ), "2FA detected, provide token in accounts.json or run in visible mode to handle login" + print( + "2FA detected, handle prompts and press enter when on keep me signed in page" + ) + input() + + with contextlib.suppress( + TimeoutException + ): # In case user clicked stay signed in self.utils.waitUntilVisible( By.NAME, "kmsiForm" ) # kmsi = keep me signed form From 2069c2771eda4619fa7128577f4467fc2785cd31 Mon Sep 17 00:00:00 2001 From: Marvin <18004362+GCMarvin@users.noreply.github.com> Date: Tue, 13 Aug 2024 13:16:43 +0200 Subject: [PATCH 08/37] Update documentation & add changelog entry --- CHANGELOG.md | 6 ++++++ README.md | 25 +++++++++++++++---------- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 539ba758..e55395bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Added + +- Support for automatic handling of time-based one-time password authentication + ## [0.2.1] - 2024-08-13 ### Fixed diff --git a/README.md b/README.md index 1489d9d8..d48f509e 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,9 @@ 4. Edit the `accounts.json.sample` with your accounts credentials and rename it by removing `.sample` at the end. + The "totp" field is not mandatory, only enter your TOTP key if you use it for 2FA (if ommitting, don't keep + it as an empty string, remove the line completely). + The "proxy" field is not mandatory, you can omit it if you don't want to use proxy (don't keep it as an empty string, remove the line completely). @@ -54,16 +57,18 @@ ```json [ - { - "username": "Your Email 1", - "password": "Your Password 1", - "proxy": "http://user:pass@host1:port" - }, - { - "username": "Your Email 2", - "password": "Your Password 2", - "proxy": "http://user:pass@host2:port" - } + { + "username": "Your Email 1", + "password": "Your Password 1", + "totp": "0123 4567 89ab cdef", + "proxy": "http://user:pass@host1:port" + }, + { + "username": "Your Email 2", + "password": "Your Password 2", + "totp": "0123 4567 89ab cdef", + "proxy": "http://user:pass@host2:port" + } ] ``` From d2dd7e62fcb6e0a3b5f6a571a5807df46de1b045 Mon Sep 17 00:00:00 2001 From: Marvin <18004362+GCMarvin@users.noreply.github.com> Date: Thu, 15 Aug 2024 16:34:32 +0200 Subject: [PATCH 09/37] Implement support for headless passwordless authentication Signed-off-by: Marvin <18004362+GCMarvin@users.noreply.github.com> --- src/login.py | 52 +++++++++++++++++++++++++--------------------------- 1 file changed, 25 insertions(+), 27 deletions(-) diff --git a/src/login.py b/src/login.py index 756d853c..cf1ee6eb 100644 --- a/src/login.py +++ b/src/login.py @@ -33,33 +33,34 @@ def login(self) -> None: assert self.utils.isLoggedIn() def executeLogin(self) -> None: - self.utils.waitUntilVisible(By.ID, "i0116") - - emailField = self.utils.waitUntilClickable(By.NAME, "loginfmt") + # Email field + emailField = self.utils.waitUntilVisible(By.ID, "i0116") logging.info("[LOGIN] Entering email...") emailField.click() emailField.send_keys(self.browser.username) assert emailField.get_attribute("value") == self.browser.username self.utils.waitUntilClickable(By.ID, "idSIButton9").click() - # noinspection PyUnusedLocal - isPasswordlessEnabled: bool = False + # Passwordless check + isPasswordless = False with contextlib.suppress(TimeoutException): - self.utils.waitUntilVisible(By.ID, "pushNotificationsTitle") - isPasswordlessEnabled = True - logging.debug(f"isPasswordlessEnabled = {isPasswordlessEnabled}") - - if isPasswordlessEnabled: - # todo - Handle 2FA when running headless - assert ( - self.args.visible - ), "2FA detected, run in visible mode to handle login" - print( - "2FA detected, handle prompts and press enter when on keep me signed in page" + self.utils.waitUntilVisible(By.ID, "displaySign", 5) + isPasswordless = True + logging.debug("isPasswordless = %s", isPasswordless) + + if isPasswordless: + # Passworless login, have user confirm code on phone + codeField = self.utils.waitUntilVisible(By.ID, "displaySign") + logging.warning( + "[LOGIN] Confirm your login with code %s on your phone (you have" + " one minute)!\a", + codeField.text, ) - input() + self.utils.waitUntilVisible(By.NAME, "kmsiForm", 60) + logging.info("[LOGIN] Successfully verified!") else: + # Password-based login, enter password from accounts.json passwordField = self.utils.waitUntilClickable(By.NAME, "passwd") logging.info("[LOGIN] Entering password...") passwordField.click() @@ -68,7 +69,7 @@ def executeLogin(self) -> None: self.utils.waitUntilClickable(By.ID, "idSIButton9").click() # noinspection PyUnusedLocal - isTwoFactorEnabled: bool = False + isTwoFactorEnabled = False with contextlib.suppress(TimeoutException): self.utils.waitUntilVisible(By.ID, "idTxtBx_SAOTCC_OTC") isTwoFactorEnabled = True @@ -93,18 +94,15 @@ def executeLogin(self) -> None: ) input() - with contextlib.suppress( - TimeoutException - ): # In case user clicked stay signed in - self.utils.waitUntilVisible( - By.NAME, "kmsiForm" - ) # kmsi = keep me signed form - self.utils.waitUntilClickable(By.ID, "acceptButton").click() + self.utils.waitUntilVisible(By.NAME, "kmsiForm") + self.utils.waitUntilClickable(By.ID, "acceptButton").click() + # TODO: This should probably instead be checked with an element's id, + # as the hardcoded text might be different in other languages isAskingToProtect = self.utils.checkIfTextPresentAfterDelay( - "protect your account" + "protect your account", 5 ) - logging.debug(f"isAskingToProtect = {isAskingToProtect}") + logging.debug(f"isAskingToProtect = %s", isAskingToProtect) if isAskingToProtect: assert ( From 3306f9c089fc2af0783b1423fdcfa23aa51d2b26 Mon Sep 17 00:00:00 2001 From: Marvin <18004362+GCMarvin@users.noreply.github.com> Date: Thu, 15 Aug 2024 18:04:52 +0200 Subject: [PATCH 10/37] Implement handling of second factor after password Signed-off-by: Marvin <18004362+GCMarvin@users.noreply.github.com> --- src/login.py | 57 ++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 44 insertions(+), 13 deletions(-) diff --git a/src/login.py b/src/login.py index cf1ee6eb..d42a84f5 100644 --- a/src/login.py +++ b/src/login.py @@ -44,7 +44,7 @@ def executeLogin(self) -> None: # Passwordless check isPasswordless = False with contextlib.suppress(TimeoutException): - self.utils.waitUntilVisible(By.ID, "displaySign", 5) + self.utils.waitUntilVisible(By.ID, "displaySign") isPasswordless = True logging.debug("isPasswordless = %s", isPasswordless) @@ -68,14 +68,42 @@ def executeLogin(self) -> None: assert passwordField.get_attribute("value") == self.browser.password self.utils.waitUntilClickable(By.ID, "idSIButton9").click() - # noinspection PyUnusedLocal - isTwoFactorEnabled = False + # Check if 2FA is enabled, both device auth and TOTP are supported + isDeviceAuthEnabled = False with contextlib.suppress(TimeoutException): - self.utils.waitUntilVisible(By.ID, "idTxtBx_SAOTCC_OTC") - isTwoFactorEnabled = True - logging.debug(f"isTwoFactorEnabled = {isTwoFactorEnabled}") + self.utils.waitUntilVisible(By.ID, "idSpan_SAOTCAS_DescSessionID") + isDeviceAuthEnabled = True + logging.debug("isDeviceAuthEnabled = %s", isDeviceAuthEnabled) - if isTwoFactorEnabled: + isTOTPEnabled = False + with contextlib.suppress(TimeoutException): + self.utils.waitUntilVisible(By.ID, "idTxtBx_SAOTCC_OTC", 1) + isTOTPEnabled = True + logging.debug("isTOTPEnabled = %s", isTOTPEnabled) + + if isDeviceAuthEnabled: + # For some reason, undetected chromedriver doesn't receive the confirmation + # after the user has confirmed the login on their phone. + raise Exception( + "Unfortunatly, device auth is not supported yet. Turn on" + " passwordless login in your account settings, use TOTPs or remove" + " 2FA altogether." + ) + + # Device auth, have user confirm code on phone + codeField = self.utils.waitUntilVisible( + By.ID, "idSpan_SAOTCAS_DescSessionID" + ) + logging.warning( + "[LOGIN] Confirm your login with code %s on your phone (you have" + " one minute)!\a", + codeField.text, + ) + self.utils.waitUntilVisible(By.NAME, "kmsiForm", 60) + logging.info("[LOGIN] Successfully verified!") + + elif isTOTPEnabled: + # One-time password required if self.browser.totp is not None: # TOTP token provided logging.info("[LOGIN] Entering OTP...") @@ -84,13 +112,16 @@ def executeLogin(self) -> None: otpField.send_keys(otp) assert otpField.get_attribute("value") == otp self.utils.waitUntilClickable(By.ID, "idSubmit_SAOTCC_Continue").click() + else: - # No TOTP token provided, manual intervention required - assert ( - self.args.visible - ), "2FA detected, provide token in accounts.json or run in visible mode to handle login" + # TOTP token not provided, manual intervention required + assert self.args.visible, ( + "[LOGIN] 2FA detected, provide token in accounts.json or run in" + " visible mode to handle login." + ) print( - "2FA detected, handle prompts and press enter when on keep me signed in page" + "[LOGIN] 2FA detected, handle prompts and press enter when on" + " keep me signed in page." ) input() @@ -102,7 +133,7 @@ def executeLogin(self) -> None: isAskingToProtect = self.utils.checkIfTextPresentAfterDelay( "protect your account", 5 ) - logging.debug(f"isAskingToProtect = %s", isAskingToProtect) + logging.debug("isAskingToProtect = %s", isAskingToProtect) if isAskingToProtect: assert ( From b12afdb81d2a34e3ed02810c64031d056f14a5f5 Mon Sep 17 00:00:00 2001 From: Marvin <18004362+GCMarvin@users.noreply.github.com> Date: Fri, 16 Aug 2024 11:44:17 +0200 Subject: [PATCH 11/37] Update changelog Signed-off-by: Marvin <18004362+GCMarvin@users.noreply.github.com> --- CHANGELOG.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e55395bb..205bacee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- Support for automatic handling of time-based one-time password authentication +- Support for automatic handling of logins with 2FA and for passwordless setups: + - Passwordless login is supported in both visible and headless mode by displaying the code that the user has to select on their phone in the terminal window + - 2FA login with TOTPs is supported in both visible and headless mode by allowing the user to provide their TOTP key in `accounts.json` which automatically generates the one time password + - 2FA login with device-based authentication is supported in theory, BUT doesn't currently work as the undetected chromedriver for some reason does not receive the confirmation signal after the user approves the login ## [0.2.1] - 2024-08-13 From fc433440ac5ce380fdfd0d0af0fc128970f6218d Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Sat, 17 Aug 2024 10:10:06 -0400 Subject: [PATCH 12/37] Add support for quizzes that were started previously but not finished --- CHANGELOG.md | 1 + src/activities.py | 21 +++++++++++---------- src/dailySet.py | 1 - src/morePromotions.py | 1 - 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 205bacee..7ab04d0c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Passwordless login is supported in both visible and headless mode by displaying the code that the user has to select on their phone in the terminal window - 2FA login with TOTPs is supported in both visible and headless mode by allowing the user to provide their TOTP key in `accounts.json` which automatically generates the one time password - 2FA login with device-based authentication is supported in theory, BUT doesn't currently work as the undetected chromedriver for some reason does not receive the confirmation signal after the user approves the login +- Completing quizzes started but not completed in previous runs ## [0.2.1] - 2024-08-13 diff --git a/src/activities.py b/src/activities.py index b1519e6b..923265e2 100644 --- a/src/activities.py +++ b/src/activities.py @@ -1,6 +1,8 @@ +import contextlib import random import time +from selenium.common import TimeoutException from selenium.webdriver.common.by import By from src.browser import Browser @@ -39,19 +41,22 @@ def completeSurvey(self): def completeQuiz(self): # Simulate completing a quiz activity - startQuiz = self.browser.utils.waitUntilQuizLoads() - startQuiz.click() + with contextlib.suppress(TimeoutException): + startQuiz = self.browser.utils.waitUntilQuizLoads() + startQuiz.click() self.browser.utils.waitUntilVisible( - By.XPATH, '//*[@id="currentQuestionContainer"]/div/div[1]', 5 + By.ID, "overlayPanel", 5 ) - time.sleep(random.randint(10, 15)) - numberOfQuestions = self.webdriver.execute_script( + currentQuestionNumber: int = self.webdriver.execute_script( + "return _w.rewardsQuizRenderInfo.currentQuestionNumber" + ) + maxQuestions = self.webdriver.execute_script( "return _w.rewardsQuizRenderInfo.maxQuestions" ) numberOfOptions = self.webdriver.execute_script( "return _w.rewardsQuizRenderInfo.numberOfOptions" ) - for question in range(numberOfQuestions): + for _ in range(currentQuestionNumber, maxQuestions + 1): if numberOfOptions == 8: answers = [] for i in range(numberOfOptions): @@ -76,13 +81,9 @@ def completeQuiz(self): == correctOption ): self.webdriver.find_element(By.ID, f"rqAnswerOption{i}").click() - time.sleep(random.randint(10, 15)) self.browser.utils.waitUntilQuestionRefresh() break - if question + 1 != numberOfQuestions: - time.sleep(random.randint(10, 15)) - time.sleep(random.randint(10, 15)) self.browser.utils.closeCurrentTab() def completeABC(self): diff --git a/src/dailySet.py b/src/dailySet.py index 3148f9f6..e16b6146 100644 --- a/src/dailySet.py +++ b/src/dailySet.py @@ -41,7 +41,6 @@ def completeDailySet(self): self.activities.completeThisOrThat() elif ( activity["pointProgressMax"] in [40, 30] - and activity["pointProgress"] == 0 ): logging.info(f"[DAILY SET] Completing quiz of card {cardId}") # Complete quiz for specific point progress max diff --git a/src/morePromotions.py b/src/morePromotions.py index 0986c438..486da739 100644 --- a/src/morePromotions.py +++ b/src/morePromotions.py @@ -83,7 +83,6 @@ def completeMorePromotions(self): self.activities.completeSearch() elif ( promotion["promotionType"] == "quiz" - and promotion["pointProgress"] == 0 ): # Complete different types of quizzes based on point progress max if promotion["pointProgressMax"] == 10: From e5b46efc04c3e2159e65b03cd897b08516271289 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Sat, 17 Aug 2024 10:18:21 -0400 Subject: [PATCH 13/37] Change how incomplete promotions determined and update CHANGELOG.md --- CHANGELOG.md | 6 ++++++ src/morePromotions.py | 4 +--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ab04d0c..3cda4a6a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - 2FA login with device-based authentication is supported in theory, BUT doesn't currently work as the undetected chromedriver for some reason does not receive the confirmation signal after the user approves the login - Completing quizzes started but not completed in previous runs +### Changed + +- Incomplete promotions Apprise notifications + - How incomplete promotions are determined + - Batched into single versus multiple notifications + ## [0.2.1] - 2024-08-13 ### Fixed diff --git a/src/morePromotions.py b/src/morePromotions.py index 486da739..df99b409 100644 --- a/src/morePromotions.py +++ b/src/morePromotions.py @@ -37,7 +37,6 @@ def completeMorePromotions(self): ): logging.debug("Already done, continuing") continue - pointsBefore = self.browser.utils.getAccountPoints() self.activities.openMorePromotionsActivity( morePromotions.index(promotion) ) @@ -97,8 +96,7 @@ def completeMorePromotions(self): self.browser.webdriver.execute_script("window.scrollTo(0, 1080)") time.sleep(random.randint(5, 10)) - pointsAfter = self.browser.utils.getAccountPoints() - if pointsBefore >= pointsAfter: + if promotion["pointProgress"] < promotion["pointProgressMax"]: incompletePromotions.append((promotionTitle, promotion["promotionType"])) self.browser.utils.resetTabs() time.sleep(2) From 6d146116146ce9286ccf8570b7abc3c4c320af7f Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Sat, 17 Aug 2024 10:20:41 -0400 Subject: [PATCH 14/37] Support find places to stay --- CHANGELOG.md | 7 +++++++ src/morePromotions.py | 5 ++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3cda4a6a..c8e0a17e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - 2FA login with TOTPs is supported in both visible and headless mode by allowing the user to provide their TOTP key in `accounts.json` which automatically generates the one time password - 2FA login with device-based authentication is supported in theory, BUT doesn't currently work as the undetected chromedriver for some reason does not receive the confirmation signal after the user approves the login - Completing quizzes started but not completed in previous runs +- Promotions/More activities + - Find places to stay ### Changed @@ -21,6 +23,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - How incomplete promotions are determined - Batched into single versus multiple notifications +### Fixed + +- Promotions/More activities + - Too tired to cook tonight? + ## [0.2.1] - 2024-08-13 ### Fixed diff --git a/src/morePromotions.py b/src/morePromotions.py index df99b409..8715ca9f 100644 --- a/src/morePromotions.py +++ b/src/morePromotions.py @@ -69,7 +69,7 @@ def completeMorePromotions(self): searchbar.send_keys("directions to new york") searchbar.submit() elif "Too tired to cook tonight?" in promotionTitle: - searchbar.send_keys("mcdonalds") + searchbar.send_keys("Pizza Hut near me") searchbar.submit() elif "Quickly convert your money" in promotionTitle: searchbar.send_keys("convert 374 usd to yen") @@ -77,6 +77,9 @@ def completeMorePromotions(self): elif "Learn to cook a new recipe" in promotionTitle: searchbar.send_keys("how cook pierogi") searchbar.submit() + elif "Find places to stay" in promotionTitle: + searchbar.send_keys("hotels rome italy") + searchbar.submit() elif promotion["promotionType"] == "urlreward": # Complete search for URL reward self.activities.completeSearch() From f72814246f69f93be850093392b0001ffe3e4077 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Sat, 17 Aug 2024 10:21:14 -0400 Subject: [PATCH 15/37] Change verbiage to better reflect what's happening --- src/searches.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/searches.py b/src/searches.py index a294fe6d..ef682bf1 100644 --- a/src/searches.py +++ b/src/searches.py @@ -161,7 +161,7 @@ def bingSearch(self) -> None: else: raise AssertionError logging.debug( - f"[BING] Search attempt failed {i}/{Searches.maxRetries}, sleeping {sleepTime}" + f"[BING] Search attempt not counted {i}/{Searches.maxRetries}, sleeping {sleepTime}" f" seconds..." ) time.sleep(sleepTime) From 8718434c842fa4130a50921602a9597bd1407d2f Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Sat, 17 Aug 2024 10:21:44 -0400 Subject: [PATCH 16/37] Remove hard-coded waits --- src/utils.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/utils.py b/src/utils.py index dd99a49f..23c7b23e 100644 --- a/src/utils.py +++ b/src/utils.py @@ -206,10 +206,7 @@ def tryDismissBingCookieBanner(self) -> None: self.webdriver.find_element(By.ID, "bnp_btn_accept").click() def switchToNewTab(self, timeToWait: float = 0) -> None: - time.sleep(0.5) self.webdriver.switch_to.window(window_name=self.webdriver.window_handles[1]) - if timeToWait > 0: - time.sleep(timeToWait) def closeCurrentTab(self) -> None: self.webdriver.close() From 07f4487c55d7e3f321996d524fa03ab815879f91 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Sat, 17 Aug 2024 10:25:03 -0400 Subject: [PATCH 17/37] Make backoff_factor more generous --- src/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils.py b/src/utils.py index 23c7b23e..e8b89f1a 100644 --- a/src/utils.py +++ b/src/utils.py @@ -143,7 +143,7 @@ def getBingInfo(self) -> Any: @staticmethod def makeRequestsSession(session: Session = requests.session()) -> Session: retry = Retry( - total=5, backoff_factor=0.1, status_forcelist=[500, 502, 503, 504] + total=5, backoff_factor=1, status_forcelist=[500, 502, 503, 504] ) session.mount( "https://", HTTPAdapter(max_retries=retry) From 8c0dc2ea933469c8e41a0c2109ab6e18e6da90da Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Sat, 17 Aug 2024 10:53:49 -0400 Subject: [PATCH 18/37] Remove sleep --- src/activities.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/activities.py b/src/activities.py index 923265e2..2194f5c2 100644 --- a/src/activities.py +++ b/src/activities.py @@ -67,7 +67,6 @@ def completeQuiz(self): answers.append(f"rqAnswerOption{i}") for answer in answers: self.webdriver.find_element(By.ID, answer).click() - time.sleep(random.randint(10, 15)) self.browser.utils.waitUntilQuestionRefresh() elif numberOfOptions in [2, 3, 4]: correctOption = self.webdriver.execute_script( From fc51b099239570695565790a75bd2f9ec340d534 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Sat, 17 Aug 2024 10:57:06 -0400 Subject: [PATCH 19/37] Remove title stuff (under assumption people want to report better bugs to get better support) --- .github/ISSUE_TEMPLATE/bug_report.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 752b1870..16db6461 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -1,6 +1,5 @@ name: Bug report description: Make sure you check if you are purposefully causing an error! (bad installation, etc.) -title: "Title" labels: [ "bug" ] body: @@ -16,14 +15,6 @@ body: - label: | I've cleared the sessions folder. required: true - - type: checkboxes - id: title - attributes: - label: Title - options: - - label: | - The title is no longer "Title" and I edited it with the right error name. - required: true - type: dropdown id: branch attributes: From 8ecec9b0fc2a30fc2ba7e5fd2a38b02c7efbf564 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Sun, 18 Aug 2024 10:57:09 -0400 Subject: [PATCH 20/37] Loop until remaining searches is zero --- src/__init__.py | 1 + src/browser.py | 12 ++---------- src/remainingSearches.py | 9 +++++++++ src/searches.py | 37 +++++++++++++++---------------------- 4 files changed, 27 insertions(+), 32 deletions(-) create mode 100644 src/remainingSearches.py diff --git a/src/__init__.py b/src/__init__.py index 976323bf..dbad1477 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -5,4 +5,5 @@ from .morePromotions import MorePromotions from .punchCards import PunchCards from .readToEarn import ReadToEarn +from .remainingSearches import RemainingSearches from .searches import Searches diff --git a/src/browser.py b/src/browser.py index 9cee35a9..93215280 100644 --- a/src/browser.py +++ b/src/browser.py @@ -3,7 +3,7 @@ import random from pathlib import Path from types import TracebackType -from typing import Any, Type, NamedTuple +from typing import Any, Type import ipapi import seleniumwire.undetected_chromedriver as webdriver @@ -12,19 +12,11 @@ from selenium.webdriver import ChromeOptions from selenium.webdriver.chrome.webdriver import WebDriver -from src import Account +from src import Account, RemainingSearches from src.userAgentGenerator import GenerateUserAgent from src.utils import Utils -class RemainingSearches(NamedTuple): - desktop: int - mobile: int - - def getTotal(self) -> int: - return self.desktop + self.mobile - - class Browser: """WebDriver wrapper class.""" diff --git a/src/remainingSearches.py b/src/remainingSearches.py new file mode 100644 index 00000000..8a6b9250 --- /dev/null +++ b/src/remainingSearches.py @@ -0,0 +1,9 @@ +from typing import NamedTuple + + +class RemainingSearches(NamedTuple): + desktop: int + mobile: int + + def getTotal(self) -> int: + return self.desktop + self.mobile diff --git a/src/searches.py b/src/searches.py index ef682bf1..ca01c766 100644 --- a/src/searches.py +++ b/src/searches.py @@ -60,24 +60,6 @@ def __init__(self, browser: Browser): dumbDbm = dbm.dumb.open((Utils.getProjectRoot() / "google_trends").__str__()) self.googleTrendsShelf: shelve.Shelf = shelve.Shelf(dumbDbm) - logging.debug(f"googleTrendsShelf.__dict__ = {self.googleTrendsShelf.__dict__}") - logging.debug(f"google_trends = {list(self.googleTrendsShelf.items())}") - loadDate: date | None = None - if LOAD_DATE_KEY in self.googleTrendsShelf: - loadDate = self.googleTrendsShelf[LOAD_DATE_KEY] - - if loadDate is None or loadDate < date.today(): - self.googleTrendsShelf.clear() - trends = self.getGoogleTrends( - browser.getRemainingSearches(desktopAndMobile=True).getTotal() - ) - random.shuffle(trends) - for trend in trends: - self.googleTrendsShelf[trend] = None - self.googleTrendsShelf[LOAD_DATE_KEY] = date.today() - logging.debug( - f"google_trends after load = {list(self.googleTrendsShelf.items())}" - ) def __enter__(self): return self @@ -129,10 +111,21 @@ def bingSearches(self) -> None: self.browser.utils.goToSearch() - remainingSearches = self.browser.getRemainingSearches() - for searchCount in range(1, remainingSearches + 1): - # todo Disable cooldown for first 3 searches (Earning starts with your third search) - logging.info(f"[BING] {searchCount}/{remainingSearches}") + while (remainingSearches := self.browser.getRemainingSearches()) > 0: + logging.info(f"[BING] Remaining searches={remainingSearches}") + desktopAndMobileRemaining = self.browser.getRemainingSearches(desktopAndMobile=True) + if desktopAndMobileRemaining.getTotal() > len(self.googleTrendsShelf): + # self.googleTrendsShelf.clear() # Maybe needed? + logging.debug( + f"google_trends before load = {list(self.googleTrendsShelf.items())}" + ) + trends = self.getGoogleTrends(desktopAndMobileRemaining.getTotal()) + random.shuffle(trends) + for trend in trends: + self.googleTrendsShelf[trend] = None + logging.debug( + f"google_trends after load = {list(self.googleTrendsShelf.items())}" + ) self.bingSearch() time.sleep(random.randint(10, 15)) From 301e6958fdd03631f40eb6c08323abc078ca4967 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Sun, 18 Aug 2024 10:57:29 -0400 Subject: [PATCH 21/37] Add compatible tags --- requirements.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index f22b270a..859f29d1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ -requests +requests~=2.32.3 selenium>=4.15.2 # not directly required, pinned by Snyk to avoid a vulnerability ipapi~=1.0.4 undetected-chromedriver==3.5.5 -selenium-wire +selenium-wire~=5.1.0 numpy>=1.22.2 # not directly required, pinned by Snyk to avoid a vulnerability setuptools psutil @@ -10,6 +10,6 @@ blinker==1.7.0 # prevents issues on newer versions apprise~=1.8.1 pyyaml~=6.0.2 urllib3>=2.2.2 # not directly required, pinned by Snyk to avoid a vulnerability -requests-oauthlib +requests-oauthlib~=2.0.0 zipp>=3.19.1 # not directly required, pinned by Snyk to avoid a vulnerability -pyotp +pyotp~=2.9.0 From 5295c9f00ac1c5e091980e98a85faa4210381ca9 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Sun, 18 Aug 2024 10:57:50 -0400 Subject: [PATCH 22/37] Print stack trace --- main.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/main.py b/main.py index 907726b5..813c0382 100644 --- a/main.py +++ b/main.py @@ -7,6 +7,7 @@ import random import re import sys +import traceback from datetime import datetime from enum import Enum, auto @@ -41,7 +42,7 @@ def main(): logging.error("", exc_info=True) Utils.sendNotification( f"⚠️ Error executing {currentAccount.username}, please check the log", - f"{e1}\n{e1.__traceback__}", + traceback.format_exc(), ) continue previous_points = previous_points_data.get(currentAccount.username, 0) @@ -238,7 +239,7 @@ def executeBot(currentAccount: Account, args: argparse.Namespace): logging.info( f"[POINTS] You have {utils.formatNumber(startingPoints)} points on your account" ) - # todo Send notification if these fail to Apprise versus just logging + # todo Combine these classes so main loop isn't duplicated DailySet(desktopBrowser).completeDailySet() PunchCards(desktopBrowser).completePunchCards() MorePromotions(desktopBrowser).completeMorePromotions() @@ -356,5 +357,5 @@ def save_previous_points_data(data): except Exception as e: logging.exception("") Utils.sendNotification( - "⚠️ Error occurred, please check the log", f"{e}\n{e.__traceback__}" + "⚠️ Error occurred, please check the log", traceback.format_exc() ) From 278414143c7b78a7e19f2ce41cbf03762e1624ca Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Sun, 18 Aug 2024 11:03:28 -0400 Subject: [PATCH 23/37] Update CHANGELOG.md --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c8e0a17e..d0a0dcd1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Removed + +- `apprise.urls` from [config.yaml](config.yaml) + - This now lives in `config-private.yaml`, see [.template-config-private.yaml](.template-config-private.yaml) on how + - This prevents accidentally leaking sensitive information since `config-private.yaml` is .gitignore'd + ### Added - Support for automatic handling of logins with 2FA and for passwordless setups: @@ -27,6 +33,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Promotions/More activities - Too tired to cook tonight? +- Last searches always timing out (#172) ## [0.2.1] - 2024-08-13 From f3885564fce692d3e1661a388bb4cd9b65281468 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Mon, 19 Aug 2024 09:48:46 -0400 Subject: [PATCH 24/37] Fix import order --- src/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/__init__.py b/src/__init__.py index dbad1477..82174ec9 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -1,9 +1,9 @@ from .account import Account +from .remainingSearches import RemainingSearches from .browser import Browser from .dailySet import DailySet from .login import Login from .morePromotions import MorePromotions from .punchCards import PunchCards from .readToEarn import ReadToEarn -from .remainingSearches import RemainingSearches from .searches import Searches From e237f593d28c39bf2cc9c8148c6b18778cdd5725 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Mon, 19 Aug 2024 22:34:34 -0400 Subject: [PATCH 25/37] Wrap clicks --- src/activities.py | 43 ++++++++++++++++++++++--------------------- src/morePromotions.py | 2 +- 2 files changed, 23 insertions(+), 22 deletions(-) diff --git a/src/activities.py b/src/activities.py index 2194f5c2..bebb0a8c 100644 --- a/src/activities.py +++ b/src/activities.py @@ -4,6 +4,7 @@ from selenium.common import TimeoutException from selenium.webdriver.common.by import By +from selenium.webdriver.remote.webelement import WebElement from src.browser import Browser @@ -43,7 +44,7 @@ def completeQuiz(self): # Simulate completing a quiz activity with contextlib.suppress(TimeoutException): startQuiz = self.browser.utils.waitUntilQuizLoads() - startQuiz.click() + self.browser.utils.click(startQuiz) self.browser.utils.waitUntilVisible( By.ID, "overlayPanel", 5 ) @@ -66,7 +67,8 @@ def completeQuiz(self): if isCorrectOption and isCorrectOption.lower() == "true": answers.append(f"rqAnswerOption{i}") for answer in answers: - self.webdriver.find_element(By.ID, answer).click() + element = self.webdriver.find_element(By.ID, answer) + self.browser.utils.click(element) self.browser.utils.waitUntilQuestionRefresh() elif numberOfOptions in [2, 3, 4]: correctOption = self.webdriver.execute_script( @@ -79,7 +81,8 @@ def completeQuiz(self): ).get_attribute("data-option") == correctOption ): - self.webdriver.find_element(By.ID, f"rqAnswerOption{i}").click() + element = self.webdriver.find_element(By.ID, f"rqAnswerOption{i}") + self.browser.utils.click(element) self.browser.utils.waitUntilQuestionRefresh() break @@ -92,11 +95,11 @@ def completeABC(self): ).text[:-1][1:] numberOfQuestions = max(int(s) for s in counter.split() if s.isdigit()) for question in range(numberOfQuestions): - self.webdriver.find_element( - By.ID, f"questionOptionChoice{question}{random.randint(0, 2)}" - ).click() + element = self.webdriver.find_element(By.ID, f"questionOptionChoice{question}{random.randint(0, 2)}") + self.browser.utils.click(element) time.sleep(random.randint(10, 15)) - self.webdriver.find_element(By.ID, f"nextQuestionbtn{question}").click() + element = self.webdriver.find_element(By.ID, f"nextQuestionbtn{question}") + self.browser.utils.click(element) time.sleep(random.randint(10, 15)) time.sleep(random.randint(1, 7)) self.browser.utils.closeCurrentTab() @@ -104,7 +107,7 @@ def completeABC(self): def completeThisOrThat(self): # Simulate completing a This or That activity startQuiz = self.browser.utils.waitUntilQuizLoads() - startQuiz.click() + self.browser.utils.click(startQuiz) self.browser.utils.waitUntilVisible( By.XPATH, '//*[@id="currentQuestionContainer"]/div/div[1]', 10 ) @@ -115,26 +118,24 @@ def completeThisOrThat(self): ) answer1, answer1Code = self.getAnswerAndCode("rqAnswerOption0") answer2, answer2Code = self.getAnswerAndCode("rqAnswerOption1") + answerToClick: WebElement if answer1Code == correctAnswerCode: - answer1.click() - time.sleep(random.randint(10, 15)) + answerToClick = answer1 elif answer2Code == correctAnswerCode: - answer2.click() - time.sleep(random.randint(10, 15)) + answerToClick = answer2 + + self.browser.utils.click(answerToClick) + time.sleep(random.randint(10, 15)) time.sleep(random.randint(10, 15)) self.browser.utils.closeCurrentTab() - def getAnswerAndCode(self, answerId: str) -> tuple: + def getAnswerAndCode(self, answerId: str) -> tuple[WebElement, str]: # Helper function to get answer element and its code answerEncodeKey = self.webdriver.execute_script("return _G.IG") answer = self.webdriver.find_element(By.ID, answerId) answerTitle = answer.get_attribute("data-option") - if answerTitle is not None: - return ( - answer, - self.browser.utils.getAnswerCode(answerEncodeKey, answerTitle), - ) - else: - # todo - throw exception? - return answer, None + return ( + answer, + self.browser.utils.getAnswerCode(answerEncodeKey, answerTitle), + ) diff --git a/src/morePromotions.py b/src/morePromotions.py index 8715ca9f..336924a4 100644 --- a/src/morePromotions.py +++ b/src/morePromotions.py @@ -45,7 +45,7 @@ def completeMorePromotions(self): searchbar = self.browser.utils.waitUntilClickable( By.ID, "sb_form_q" ) - searchbar.click() + self.browser.utils.click(searchbar) # todo These and following are US-English specific, maybe there's a good way to internationalize if "Search the lyrics of a song" in promotionTitle: searchbar.send_keys("black sabbath supernaut lyrics") From b42e53d1c65911813182c3d31e2a02464b4e22ea Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Mon, 19 Aug 2024 22:45:27 -0400 Subject: [PATCH 26/37] Fix bug when notifying via apprise and assert --- src/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils.py b/src/utils.py index e8b89f1a..c12d7793 100644 --- a/src/utils.py +++ b/src/utils.py @@ -59,7 +59,7 @@ def sendNotification(title, body) -> None: urls: list[str] = Utils.loadConfig("config-private.yaml").get("apprise", {}).get("urls", []) for url in urls: apprise.add(url) - apprise.notify(body=body, title=title) + assert apprise.notify(title=str(title), body=str(body)) def waitUntilVisible( self, by: str, selector: str, timeToWait: float = 10 From 7ecc89a19e931a26b0fc84df80b481d027a28276 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Mon, 19 Aug 2024 23:05:30 -0400 Subject: [PATCH 27/37] Add support for some more promotions --- CHANGELOG.md | 3 +++ src/morePromotions.py | 9 +++++++++ 2 files changed, 12 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d0a0dcd1..5744e892 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Completing quizzes started but not completed in previous runs - Promotions/More activities - Find places to stay + - How's the economy? + - Who won? + - Gaming time ### Changed diff --git a/src/morePromotions.py b/src/morePromotions.py index 336924a4..588bfb4c 100644 --- a/src/morePromotions.py +++ b/src/morePromotions.py @@ -80,6 +80,15 @@ def completeMorePromotions(self): elif "Find places to stay" in promotionTitle: searchbar.send_keys("hotels rome italy") searchbar.submit() + elif "How's the economy?" in promotionTitle: + searchbar.send_keys("sp 500") + searchbar.submit() + elif "Who won?" in promotionTitle: + searchbar.send_keys("braves score") + searchbar.submit() + elif "Gaming time" in promotionTitle: + searchbar.send_keys("vampire survivors video game") + searchbar.submit() elif promotion["promotionType"] == "urlreward": # Complete search for URL reward self.activities.completeSearch() From b91d8785ce7497a4ca026e46f874ebe27a0ee4fb Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Mon, 19 Aug 2024 23:06:13 -0400 Subject: [PATCH 28/37] Get promotions after and loop --- src/morePromotions.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/morePromotions.py b/src/morePromotions.py index 588bfb4c..a1986a62 100644 --- a/src/morePromotions.py +++ b/src/morePromotions.py @@ -108,8 +108,6 @@ def completeMorePromotions(self): self.browser.webdriver.execute_script("window.scrollTo(0, 1080)") time.sleep(random.randint(5, 10)) - if promotion["pointProgress"] < promotion["pointProgressMax"]: - incompletePromotions.append((promotionTitle, promotion["promotionType"])) self.browser.utils.resetTabs() time.sleep(2) except Exception: # pylint: disable=broad-except @@ -117,6 +115,9 @@ def completeMorePromotions(self): # Reset tabs in case of an exception self.browser.utils.resetTabs() continue + for promotion in self.browser.utils.getDashboardData()["morePromotions"]: # Have to refresh + if promotion["pointProgress"] < promotion["pointProgressMax"]: + incompletePromotions.append((promotion["title"], promotion["promotionType"])) if incompletePromotions: Utils.sendNotification("Incomplete promotions(s)", incompletePromotions) logging.info("[MORE PROMOS] Exiting") From 37ddcf776b7777eeb5b267b90683ea57ffbf49ba Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Thu, 22 Aug 2024 09:04:42 -0400 Subject: [PATCH 29/37] Fix bug when send_keys --- src/searches.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/searches.py b/src/searches.py index ca01c766..a96d5cbf 100644 --- a/src/searches.py +++ b/src/searches.py @@ -13,6 +13,7 @@ import requests from selenium.common import TimeoutException from selenium.webdriver.common.by import By +from selenium.webdriver.remote.webelement import WebElement from selenium.webdriver.support import expected_conditions from selenium.webdriver.support.wait import WebDriverWait @@ -159,17 +160,19 @@ def bingSearch(self) -> None: ) time.sleep(sleepTime) - searchbar = self.browser.utils.waitUntilClickable( - By.ID, "sb_form_q", timeToWait=20 - ) + searchbar: WebElement for _ in range(1000): - self.browser.utils.click(searchbar) + searchbar = self.browser.utils.waitUntilClickable( + By.ID, "sb_form_q", timeToWait=40 + ) searchbar.clear() term = next(termsCycle) logging.debug(f"term={term}") + time.sleep(1) searchbar.send_keys(term) + time.sleep(1) with contextlib.suppress(TimeoutException): - WebDriverWait(self.webdriver, 10).until( + WebDriverWait(self.webdriver, 20).until( expected_conditions.text_to_be_present_in_element_value( (By.ID, "sb_form_q"), term ) From b61b4ef9f8e92017b8feba4333d16704bff16e53 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Thu, 22 Aug 2024 09:04:55 -0400 Subject: [PATCH 30/37] Add another exception --- src/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils.py b/src/utils.py index c12d7793..51bd614d 100644 --- a/src/utils.py +++ b/src/utils.py @@ -241,6 +241,6 @@ def saveBrowserConfig(sessionPath: Path, config: dict) -> None: def click(self, element: WebElement) -> None: try: element.click() - except ElementClickInterceptedException: + except (ElementClickInterceptedException, ElementNotInteractableException): self.tryDismissAllMessages() element.click() From cce7697c6e8a39dc2c500b17705eefe39fb29e8e Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Fri, 23 Aug 2024 14:50:53 -0400 Subject: [PATCH 31/37] Reformat and remove unused constant --- src/searches.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/searches.py b/src/searches.py index a96d5cbf..d97e1c6a 100644 --- a/src/searches.py +++ b/src/searches.py @@ -20,8 +20,6 @@ from src.browser import Browser from src.utils import Utils -LOAD_DATE_KEY = "loadDate" - class RetriesStrategy(Enum): """ @@ -80,7 +78,9 @@ def getGoogleTrends(self, wordsCount: int) -> list[str]: f"https://trends.google.com/trends/api/dailytrends?hl={self.browser.localeLang}" f'&ed={(date.today() - timedelta(days=i)).strftime("%Y%m%d")}&geo={self.browser.localeGeo}&ns=15' ) - assert r.status_code == requests.codes.ok # todo Add guidance if assertion fails + assert ( + r.status_code == requests.codes.ok + ) # todo Add guidance if assertion fails trends = json.loads(r.text[6:]) for topic in trends["default"]["trendingSearchesDays"][0][ "trendingSearches" @@ -96,10 +96,14 @@ def getGoogleTrends(self, wordsCount: int) -> list[str]: def getRelatedTerms(self, term: str) -> list[str]: # Function to retrieve related terms from Bing API - relatedTerms: list[str] = Utils.makeRequestsSession().get( - f"https://api.bing.com/osjson.aspx?query={term}", - headers={"User-agent": self.browser.userAgent}, - ).json()[1] # todo Wrap if failed, or assert response? + relatedTerms: list[str] = ( + Utils.makeRequestsSession() + .get( + f"https://api.bing.com/osjson.aspx?query={term}", + headers={"User-agent": self.browser.userAgent}, + ) + .json()[1] + ) # todo Wrap if failed, or assert response? if not relatedTerms: return [term] return relatedTerms @@ -114,7 +118,9 @@ def bingSearches(self) -> None: while (remainingSearches := self.browser.getRemainingSearches()) > 0: logging.info(f"[BING] Remaining searches={remainingSearches}") - desktopAndMobileRemaining = self.browser.getRemainingSearches(desktopAndMobile=True) + desktopAndMobileRemaining = self.browser.getRemainingSearches( + desktopAndMobile=True + ) if desktopAndMobileRemaining.getTotal() > len(self.googleTrendsShelf): # self.googleTrendsShelf.clear() # Maybe needed? logging.debug( From b5c78aa1ed880ff9acb4905cc083e5020c189acc Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Fri, 23 Aug 2024 14:52:12 -0400 Subject: [PATCH 32/37] Update unreleased --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5744e892..70e7e798 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [1.0.0] - 2024-08-23 ### Removed From 0667cf0a633b9d65b3c1c4fa699f2ecdaae94635 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Sun, 25 Aug 2024 12:20:58 -0400 Subject: [PATCH 33/37] Add todo --- src/morePromotions.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/morePromotions.py b/src/morePromotions.py index a1986a62..3523bdf7 100644 --- a/src/morePromotions.py +++ b/src/morePromotions.py @@ -47,6 +47,7 @@ def completeMorePromotions(self): ) self.browser.utils.click(searchbar) # todo These and following are US-English specific, maybe there's a good way to internationalize + # todo Could use dictionary of promotionTitle to search to simplify if "Search the lyrics of a song" in promotionTitle: searchbar.send_keys("black sabbath supernaut lyrics") searchbar.submit() From d694a849e824132657ab9c5f446ebc14fa954d37 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Sun, 25 Aug 2024 12:29:26 -0400 Subject: [PATCH 34/37] Update CHANGELOG.md --- CHANGELOG.md | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 70e7e798..ab9f3beb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,14 +11,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `apprise.urls` from [config.yaml](config.yaml) - This now lives in `config-private.yaml`, see [.template-config-private.yaml](.template-config-private.yaml) on how + to configure - This prevents accidentally leaking sensitive information since `config-private.yaml` is .gitignore'd ### Added - Support for automatic handling of logins with 2FA and for passwordless setups: - - Passwordless login is supported in both visible and headless mode by displaying the code that the user has to select on their phone in the terminal window - - 2FA login with TOTPs is supported in both visible and headless mode by allowing the user to provide their TOTP key in `accounts.json` which automatically generates the one time password - - 2FA login with device-based authentication is supported in theory, BUT doesn't currently work as the undetected chromedriver for some reason does not receive the confirmation signal after the user approves the login + - Passwordless login is supported in both visible and headless mode by displaying the code that the user has to select + on their phone in the terminal window + - 2FA login with TOTPs is supported in both visible and headless mode by allowing the user to provide their TOTP key + in `accounts.json` which automatically generates the one time password + - 2FA login with device-based authentication is supported in theory, BUT doesn't currently work as the undetected + chromedriver for some reason does not receive the confirmation signal after the user approves the login - Completing quizzes started but not completed in previous runs - Promotions/More activities - Find places to stay @@ -31,12 +35,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Incomplete promotions Apprise notifications - How incomplete promotions are determined - Batched into single versus multiple notifications +- Full exception is sent via Apprise versus just error message ### Fixed - Promotions/More activities - Too tired to cook tonight? - Last searches always timing out (#172) +- Quizzes #181 ## [0.2.1] - 2024-08-13 @@ -53,7 +59,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Allows users to choose between Miniconda, Anaconda, and Local Python - Prompts users to input the name of their environment (if using Miniconda or Anaconda) - Uses the script directory as the output path - - Default trigger time is set to 6:00 AM on a specified day, with instructions to modify settings after importing to Task Scheduler + - Default trigger time is set to 6:00 AM on a specified day, with instructions to modify settings after importing to + Task Scheduler - Includes a batch file (`MS_reward.bat`) for automatic execution of the Python script ### Fixed From b0733ba2e8fc0e50374a3944eeb80a3769e4c23f Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Sun, 25 Aug 2024 12:29:42 -0400 Subject: [PATCH 35/37] Move variable closer to use --- src/morePromotions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/morePromotions.py b/src/morePromotions.py index 3523bdf7..1b591f2a 100644 --- a/src/morePromotions.py +++ b/src/morePromotions.py @@ -25,7 +25,6 @@ def completeMorePromotions(self): "morePromotions" ] self.browser.utils.goToRewards() - incompletePromotions: list[tuple[str, str]] = [] for promotion in morePromotions: try: promotionTitle = promotion["title"].replace("\u200b", "").replace("\xa0", " ") @@ -116,6 +115,7 @@ def completeMorePromotions(self): # Reset tabs in case of an exception self.browser.utils.resetTabs() continue + incompletePromotions: list[tuple[str, str]] = [] for promotion in self.browser.utils.getDashboardData()["morePromotions"]: # Have to refresh if promotion["pointProgress"] < promotion["pointProgressMax"]: incompletePromotions.append((promotion["title"], promotion["promotionType"])) From 198586aa578afd9c72f619a5390cb8b1ced30c94 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Sun, 25 Aug 2024 12:30:13 -0400 Subject: [PATCH 36/37] Move variable closer to use --- src/morePromotions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/morePromotions.py b/src/morePromotions.py index 1b591f2a..70f6b00f 100644 --- a/src/morePromotions.py +++ b/src/morePromotions.py @@ -115,7 +115,7 @@ def completeMorePromotions(self): # Reset tabs in case of an exception self.browser.utils.resetTabs() continue - incompletePromotions: list[tuple[str, str]] = [] + incompletePromotions: list[tuple[str, str]] = [] for promotion in self.browser.utils.getDashboardData()["morePromotions"]: # Have to refresh if promotion["pointProgress"] < promotion["pointProgressMax"]: incompletePromotions.append((promotion["title"], promotion["promotionType"])) From e3f1d65aa8f41e7ebe295d0eaa4ae42ff6b32194 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Sun, 25 Aug 2024 12:33:23 -0400 Subject: [PATCH 37/37] Update links --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ab9f3beb..74207e49 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,8 +41,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Promotions/More activities - Too tired to cook tonight? -- Last searches always timing out (#172) -- Quizzes #181 +- [Last searches always timing out](https://github.com/klept0/MS-Rewards-Farmer/issues/172) +- [Quizzes don't complete](https://github.com/klept0/MS-Rewards-Farmer/issues) ## [0.2.1] - 2024-08-13