From 05611c785b786b2425a80fc90ca25997f38a1914 Mon Sep 17 00:00:00 2001 From: Omega Blurz <81272128+OmegaBlurz@users.noreply.github.com> Date: Tue, 3 Sep 2024 09:36:08 +0100 Subject: [PATCH 01/82] Update morePromotions.py --- src/morePromotions.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/morePromotions.py b/src/morePromotions.py index fd686b74..a3b9ddf2 100644 --- a/src/morePromotions.py +++ b/src/morePromotions.py @@ -95,6 +95,9 @@ def completeMorePromotions(self): elif "What time is it?" in promotionTitle: searchbar.send_keys("china time") searchbar.submit() + elif "Houses near you" in promotionTitle: + searchbar.send_keys("apartments manhattan") + searchbar.submit() elif promotion["promotionType"] == "urlreward": # Complete search for URL reward self.activities.completeSearch() From 4400d486475e88468b65cb09191f35a569ecb5ee Mon Sep 17 00:00:00 2001 From: Rippenkneifer <147978010+Rippenkneifer@users.noreply.github.com> Date: Fri, 6 Sep 2024 17:16:40 +0200 Subject: [PATCH 02/82] Update dailySet.py added self.activities.dashboardPopUpModalCloseCross() --- src/dailySet.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dailySet.py b/src/dailySet.py index e16b6146..28b8157a 100644 --- a/src/dailySet.py +++ b/src/dailySet.py @@ -1,7 +1,6 @@ import logging import urllib.parse from datetime import datetime - from src.browser import Browser from .activities import Activities @@ -17,6 +16,7 @@ def completeDailySet(self): logging.info("[DAILY SET] " + "Trying to complete the Daily Set...") data = self.browser.utils.getDashboardData()["dailySetPromotions"] self.browser.utils.goToRewards() + self.activities.dashboardPopUpModalCloseCross() todayDate = datetime.now().strftime("%m/%d/%Y") for activity in data.get(todayDate, []): cardId = int(activity["offerId"][-1:]) From 2b4ea725d40ba9da44708ba0bb16038b01943e3d Mon Sep 17 00:00:00 2001 From: Rippenkneifer <147978010+Rippenkneifer@users.noreply.github.com> Date: Fri, 6 Sep 2024 17:17:52 +0200 Subject: [PATCH 03/82] Update activities.py Added: def click_element_if_visible(self, element): try: if element.is_displayed() and element.is_enabled(): element.click() logging.info("Dashboard pop-up registered and closed, needs to be done once on new accounts") else: pass except (ElementNotInteractableException, NoSuchElementException): pass def dashboardPopUpModalCloseCross(self): try: element = self.webdriver.find_element(By.CSS_SELECTOR, ".dashboardPopUpPopUpSelectButton") self.click_element_if_visible(element) time.sleep(0.25) except NoSuchElementException: return -goodluck --- src/activities.py | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/src/activities.py b/src/activities.py index bebb0a8c..71b0c9e8 100644 --- a/src/activities.py +++ b/src/activities.py @@ -1,11 +1,12 @@ import contextlib import random import time +import logging from selenium.common import TimeoutException from selenium.webdriver.common.by import By from selenium.webdriver.remote.webelement import WebElement - +from selenium.common.exceptions import NoSuchElementException, ElementNotInteractableException from src.browser import Browser @@ -14,6 +15,29 @@ def __init__(self, browser: Browser): self.browser = browser self.webdriver = browser.webdriver + def click_element_if_visible(self, element): + try: + if element.is_displayed() and element.is_enabled(): + element.click() + logging.info("Dashboard pop-up registered and closed, needs to be done once on new accounts") + else: + pass + except (ElementNotInteractableException, NoSuchElementException): + pass + + def dashboardPopUpModalCloseCross(self): + try: + + element = self.webdriver.find_element(By.CSS_SELECTOR, ".dashboardPopUpPopUpSelectButton") + self.click_element_if_visible(element) + time.sleep(0.25) + except NoSuchElementException: + return + + + + + def openDailySetActivity(self, cardId: int): # Open the Daily Set activity for the given cardId element = self.webdriver.find_element(By.XPATH, From cd30d8561098bd21f54c7c10a26288bc42d5cefd Mon Sep 17 00:00:00 2001 From: Rippenkneifer <147978010+Rippenkneifer@users.noreply.github.com> Date: Fri, 6 Sep 2024 23:24:21 +0200 Subject: [PATCH 04/82] Update login.py Lock/Bann checked after Login. --- src/login.py | 91 +++++++++++++++++++++++++++++++++++----------------- 1 file changed, 62 insertions(+), 29 deletions(-) diff --git a/src/login.py b/src/login.py index d42a84f5..f28af567 100644 --- a/src/login.py +++ b/src/login.py @@ -2,12 +2,11 @@ import contextlib import logging from argparse import Namespace - from pyotp import TOTP from selenium.common import TimeoutException from selenium.webdriver.common.by import By from undetected_chromedriver import Chrome - +from selenium.common.exceptions import ElementNotInteractableException, NoSuchElementException from src.browser import Browser @@ -22,17 +21,57 @@ def __init__(self, browser: Browser, args: argparse.Namespace): self.utils = browser.utils self.args = args - def login(self) -> None: - if self.utils.isLoggedIn(): - logging.info("[LOGIN] Already logged-in") - else: - logging.info("[LOGIN] Logging-in...") - self.executeLogin() - logging.info("[LOGIN] Logged-in successfully !") - - assert self.utils.isLoggedIn() + def check_locked_user(self): + try: + element = self.webdriver.find_element(By.XPATH, "//div[@id='serviceAbuseLandingTitle']") + self.locked(element) + except NoSuchElementException: + return + + def check_banned_user(self): + try: + element = self.webdriver.find_element(By.XPATH, '//*[@id="fraudErrorBody"]') + self.banned(element) + except NoSuchElementException: + return + + def locked(self, element): + try: + if element.is_displayed(): + logging.critical("This Account is Locked!") + self.webdriver.close() + raise Exception("Account locked, moving to the next account.") + except (ElementNotInteractableException, NoSuchElementException): + pass + + def banned(self, element): + try: + if element.is_displayed(): + logging.critical("This Account is Banned!") + self.webdriver.close() + raise Exception("Account banned, moving to the next account.") + except (ElementNotInteractableException, NoSuchElementException): + pass - def executeLogin(self) -> None: + def login(self) -> None: + try: + if self.utils.isLoggedIn(): + logging.info("[LOGIN] Already logged-in") + self.check_locked_user() + self.check_banned_user() + else: + logging.info("[LOGIN] Logging-in...") + self.execute_login() + logging.info("[LOGIN] Logged-in successfully!") + self.check_locked_user() + self.check_banned_user() + assert self.utils.isLoggedIn() + except Exception as e: + logging.error(f"Error during login: {e}") + self.webdriver.close() + raise + + def execute_login(self) -> None: # Email field emailField = self.utils.waitUntilVisible(By.ID, "i0116") logging.info("[LOGIN] Entering email...") @@ -52,13 +91,11 @@ def executeLogin(self) -> None: # Passworless login, have user confirm code on phone codeField = self.utils.waitUntilVisible(By.ID, "displaySign") logging.warning( - "[LOGIN] Confirm your login with code %s on your phone (you have" - " one minute)!\a", + "[LOGIN] Confirm your login with code %s on your phone (you have one minute)!\a", codeField.text, ) self.utils.waitUntilVisible(By.NAME, "kmsiForm", 60) logging.info("[LOGIN] Successfully verified!") - else: # Password-based login, enter password from accounts.json passwordField = self.utils.waitUntilClickable(By.NAME, "passwd") @@ -82,13 +119,8 @@ def executeLogin(self) -> None: logging.debug("isTOTPEnabled = %s", isTOTPEnabled) if isDeviceAuthEnabled: - # For some reason, undetected chromedriver doesn't receive the confirmation - # after the user has confirmed the login on their phone. - raise Exception( - "Unfortunatly, device auth is not supported yet. Turn on" - " passwordless login in your account settings, use TOTPs or remove" - " 2FA altogether." - ) + # Device-based authentication not supported + raise Exception("Device authentication not supported. Please use TOTP or disable 2FA.") # Device auth, have user confirm code on phone codeField = self.utils.waitUntilVisible( @@ -112,18 +144,21 @@ def executeLogin(self) -> None: otpField.send_keys(otp) assert otpField.get_attribute("value") == otp self.utils.waitUntilClickable(By.ID, "idSubmit_SAOTCC_Continue").click() - else: # TOTP token not provided, manual intervention required assert self.args.visible, ( - "[LOGIN] 2FA detected, provide token in accounts.json or run in" + "[LOGIN] 2FA detected, provide token in accounts.json or or run in" + "[LOGIN] 2FA detected, provide token in accounts.json or handle manually." " visible mode to handle login." ) print( "[LOGIN] 2FA detected, handle prompts and press enter when on" - " keep me signed in page." - ) + " keep me signed in page.") input() + + + self.check_locked_user() + self.check_banned_user() self.utils.waitUntilVisible(By.NAME, "kmsiForm") self.utils.waitUntilClickable(By.ID, "acceptButton").click() @@ -144,6 +179,4 @@ def executeLogin(self) -> None: ) input() - self.utils.waitUntilVisible( - By.CSS_SELECTOR, 'html[data-role-name="RewardsPortal"]' - ) + self.utils.waitUntilVisible(By.CSS_SELECTOR, 'html[data-role-name="RewardsPortal"]') From 568a721d32b732ad2cb2f1f807e04929afc4ed95 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Fri, 6 Sep 2024 22:57:03 -0400 Subject: [PATCH 05/82] Refactor getRemainingSearches --- src/browser.py | 35 +++++++++++++++++------------------ src/searches.py | 7 +++++-- src/utils.py | 5 +++-- 3 files changed, 25 insertions(+), 22 deletions(-) diff --git a/src/browser.py b/src/browser.py index 93215280..2088452d 100644 --- a/src/browser.py +++ b/src/browser.py @@ -231,27 +231,26 @@ def getChromeVersion() -> str: def getRemainingSearches( self, desktopAndMobile: bool = False ) -> RemainingSearches | int: - dashboard = self.utils.getDashboardData() + bingInfo = self.utils.getBingInfo() searchPoints = 1 - counters = dashboard["userStatus"]["counters"] + counters = bingInfo["flyoutResult"]["userStatus"]["counters"] + pcSearch: dict = counters["PCSearch"][0] + mobileSearch: dict = counters["MobileSearch"][0] + pointProgressMax: int = pcSearch["pointProgressMax"] - progressDesktop = counters["pcSearch"][0]["pointProgress"] - targetDesktop = counters["pcSearch"][0]["pointProgressMax"] - if len(counters["pcSearch"]) >= 2: - progressDesktop = progressDesktop + counters["pcSearch"][1]["pointProgress"] - targetDesktop = targetDesktop + counters["pcSearch"][1]["pointProgressMax"] - if targetDesktop in [30, 90, 102]: + searchPoints: int + if pointProgressMax in [30, 90, 102]: searchPoints = 3 - elif targetDesktop == 50 or targetDesktop >= 170 or targetDesktop == 150: + elif pointProgressMax in [50, 150] or pointProgressMax >= 170: searchPoints = 5 - remainingDesktop = int((targetDesktop - progressDesktop) / searchPoints) - remainingMobile = 0 - if dashboard["userStatus"]["levelInfo"]["activeLevel"] != "Level1": - progressMobile = counters["mobileSearch"][0]["pointProgress"] - targetMobile = counters["mobileSearch"][0]["pointProgressMax"] - remainingMobile = int((targetMobile - progressMobile) / searchPoints) + pcPointsRemaining = pcSearch["pointProgressMax"] - pcSearch["pointProgress"] + assert pcPointsRemaining % searchPoints == 0 + remainingDesktopSearches: int = int(pcPointsRemaining / searchPoints) + mobilePointsRemaining = mobileSearch["pointProgressMax"] - mobileSearch["pointProgress"] + assert mobilePointsRemaining % searchPoints == 0 + remainingMobileSearches: int = int(mobilePointsRemaining / searchPoints) if desktopAndMobile: - return RemainingSearches(desktop=remainingDesktop, mobile=remainingMobile) + return RemainingSearches(desktop=remainingDesktopSearches, mobile=remainingMobileSearches) if self.mobile: - return remainingMobile - return remainingDesktop + return remainingMobileSearches + return remainingDesktopSearches diff --git a/src/searches.py b/src/searches.py index d97e1c6a..07c96a8d 100644 --- a/src/searches.py +++ b/src/searches.py @@ -116,11 +116,14 @@ def bingSearches(self) -> None: self.browser.utils.goToSearch() - while (remainingSearches := self.browser.getRemainingSearches()) > 0: - logging.info(f"[BING] Remaining searches={remainingSearches}") + while True: desktopAndMobileRemaining = self.browser.getRemainingSearches( desktopAndMobile=True ) + if (self.browser.browserType == "desktop" and desktopAndMobileRemaining.desktop == 0) \ + or (self.browser.browserType == "mobile" and desktopAndMobileRemaining.mobile == 0): + break + if desktopAndMobileRemaining.getTotal() > len(self.googleTrendsShelf): # self.googleTrendsShelf.clear() # Maybe needed? logging.debug( diff --git a/src/utils.py b/src/utils.py index 5b2fc555..cd0a7900 100644 --- a/src/utils.py +++ b/src/utils.py @@ -125,6 +125,7 @@ def getAnswerCode(key: str, string: str) -> str: t += int(key[-2:], 16) return str(t) + # Prefer getBingInfo if possible def getDashboardData(self) -> dict: urlBefore = self.webdriver.current_url try: @@ -145,7 +146,7 @@ def getBingInfo(self) -> Any: response = session.get("https://www.bing.com/rewards/panelflyout/getuserinfo") assert response.status_code == requests.codes.ok - return response.json()["userInfo"] + return response.json() @staticmethod def makeRequestsSession(session: Session = requests.session()) -> Session: @@ -173,7 +174,7 @@ def isLoggedIn(self) -> bool: return False def getAccountPoints(self) -> int: - return self.getBingInfo()["balance"] + return self.getBingInfo()["userInfo"]["balance"] def getGoalPoints(self) -> int: return self.getDashboardData()["userStatus"]["redeemGoal"]["price"] From cb3c6683a8e51424c5a44719930f336369a7b668 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Fri, 6 Sep 2024 22:57:18 -0400 Subject: [PATCH 06/82] Add fixme --- src/searches.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/searches.py b/src/searches.py index 07c96a8d..29b5bc79 100644 --- a/src/searches.py +++ b/src/searches.py @@ -136,6 +136,8 @@ def bingSearches(self) -> None: logging.debug( f"google_trends after load = {list(self.googleTrendsShelf.items())}" ) + + # fixme Multiple tabs are getting opened self.bingSearch() time.sleep(random.randint(10, 15)) From 4560eae21d56e2ecae753ed40cbf4fd11da822ee Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Sat, 7 Sep 2024 02:33:35 -0400 Subject: [PATCH 07/82] Put logging back --- src/searches.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/searches.py b/src/searches.py index 29b5bc79..a5b1a3b6 100644 --- a/src/searches.py +++ b/src/searches.py @@ -120,6 +120,7 @@ def bingSearches(self) -> None: desktopAndMobileRemaining = self.browser.getRemainingSearches( desktopAndMobile=True ) + logging.info(f"[BING] Remaining searches={desktopAndMobileRemaining}") if (self.browser.browserType == "desktop" and desktopAndMobileRemaining.desktop == 0) \ or (self.browser.browserType == "mobile" and desktopAndMobileRemaining.mobile == 0): break From 29863961633aded0a1ce303f925314a4ff28583e Mon Sep 17 00:00:00 2001 From: Guido30 Date: Sat, 7 Sep 2024 21:57:16 +0200 Subject: [PATCH 08/82] .gitignore --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 7302770d..d3a9e9eb 100644 --- a/.gitignore +++ b/.gitignore @@ -186,4 +186,4 @@ runbot.bat /google_trends.dat /google_trends.dir /google_trends.bak -/config-private.yaml +/config.yaml From f8f29b318acc505879229e6132888e289b6b3552 Mon Sep 17 00:00:00 2001 From: Guido30 Date: Sat, 7 Sep 2024 21:59:12 +0200 Subject: [PATCH 09/82] removed config.yaml --- config.yaml | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 config.yaml diff --git a/config.yaml b/config.yaml deleted file mode 100644 index 7e45c418..00000000 --- a/config.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# config.yaml -apprise: - summary: ON_ERROR -retries: - base_delay_in_seconds: 14.0625 # base_delay_in_seconds * 2^max = 14.0625 * 2^6 = 900 = 15 minutes - max: 8 - strategy: EXPONENTIAL From 102d90b57a4b3f2c69e843cc0b0d73f0b3260d7e Mon Sep 17 00:00:00 2001 From: Guido30 Date: Sat, 7 Sep 2024 22:45:25 +0200 Subject: [PATCH 10/82] Merged config-private into config.yaml Added configuration to set the logger level in config Also configs for toggling incomplete promotions and exceptions from apprise notifications Added chrome option --disable-http2 as suggested in (#185) --- .template-config-private.yaml | 5 ----- README.md | 8 +++++--- config.yaml.sample | 13 +++++++++++++ main.py | 27 +++++++++++++++++++-------- src/browser.py | 6 ++++-- src/morePromotions.py | 4 ++-- src/utils.py | 2 +- 7 files changed, 44 insertions(+), 21 deletions(-) delete mode 100644 .template-config-private.yaml create mode 100644 config.yaml.sample diff --git a/.template-config-private.yaml b/.template-config-private.yaml deleted file mode 100644 index 60b62b9d..00000000 --- a/.template-config-private.yaml +++ /dev/null @@ -1,5 +0,0 @@ -# config-private.yaml -# Copy this file to config-private.yaml to use -apprise: - urls: - - 'discord://WebhookID/WebhookToken' # Replace with your actual Apprise service URLs diff --git a/README.md b/README.md index d48f509e..336af401 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,9 @@ this [link](https://learn.microsoft.com/en-GB/cpp/windows/latest-supported-vc-redist?view=msvc-170) and reboot your computer -4. Edit the `accounts.json.sample` with your accounts credentials and rename it by removing `.sample` at the end. +4. Edit the `config.yaml.sample` accordingly and rename it by removing `.sample` at the end. + +5. Edit the `accounts.json.sample` with your accounts credentials and rename it by removing `.sample` at the end. The "totp" field is not mandatory, only enter your TOTP key if you use it for 2FA (if ommitting, don't keep it as an empty string, remove the line completely). @@ -72,11 +74,11 @@ ] ``` -5. Run the script: +6. Run the script: `python main.py` -6. (Windows Only) You can set up automatic execution by generating a Task Scheduler XML file. +7. (Windows Only) You can set up automatic execution by generating a Task Scheduler XML file. If you are a Windows user, run the `generate_task_xml.py` script to create a `.xml` file. After generating the file, import it into Task Scheduler to schedule automatic execution of the script. This will allow the script to run at the specified time without manual intervention. diff --git a/config.yaml.sample b/config.yaml.sample new file mode 100644 index 00000000..3a6dc166 --- /dev/null +++ b/config.yaml.sample @@ -0,0 +1,13 @@ +# RENAME THIS FILE TO config.yaml ONCE CONFIGURED +apprise: + summary: ON_ERROR + exceptions: True # True or False (Whether to send or not exceptions) + promotions: True # True or False (Whether to send or not incomplete promotions) + urls: + - 'discord://{WebhookID}/{WebhookToken}' # Replace with your actual Apprise service URLs +retries: + base_delay_in_seconds: 14.0625 # base_delay_in_seconds * 2^max = 14.0625 * 2^6 = 900 = 15 minutes + max: 8 + strategy: EXPONENTIAL +default_geolocation: US # Replace with your country code https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2 +logging: DEBUG # DEBUG or INFO diff --git a/main.py b/main.py index 813c0382..041685d9 100644 --- a/main.py +++ b/main.py @@ -40,10 +40,11 @@ def main(): earned_points = executeBot(currentAccount, args) except Exception as e1: logging.error("", exc_info=True) - Utils.sendNotification( - f"⚠️ Error executing {currentAccount.username}, please check the log", - traceback.format_exc(), - ) + if Utils.loadConfig().get("apprise", {}).get("exceptions", True): + Utils.sendNotification( + f"⚠️ Error executing {currentAccount.username}, please check the log", + traceback.format_exc(), + ) continue previous_points = previous_points_data.get(currentAccount.username, 0) @@ -99,6 +100,15 @@ def setupLogging(): # so only our code is logged if level=logging.DEBUG or finer # if not working see https://stackoverflow.com/a/48891485/4164390 + _levels = { + 'DEBUG': logging.DEBUG, + 'INFO': logging.INFO, + 'WARNING': logging.WARNING, + 'ERROR': logging.ERROR, + 'CRITICAL': logging.CRITICAL + } + log_level_str = Utils.loadConfig().get("logging", "DEBUG").upper() + log_level = _levels.get(log_level_str, logging.DEBUG) logging.config.dictConfig( { "version": 1, @@ -106,7 +116,7 @@ def setupLogging(): } ) logging.basicConfig( - level=logging.DEBUG, + level=log_level, format=_format, handlers=[ handlers.TimedRotatingFileHandler( @@ -356,6 +366,7 @@ def save_previous_points_data(data): main() except Exception as e: logging.exception("") - Utils.sendNotification( - "⚠️ Error occurred, please check the log", traceback.format_exc() - ) + if Utils.loadConfig().get("apprise", {}).get("exceptions", True): + Utils.sendNotification( + "⚠️ Error occurred, please check the log", traceback.format_exc() + ) diff --git a/src/browser.py b/src/browser.py index 93215280..df08d118 100644 --- a/src/browser.py +++ b/src/browser.py @@ -90,6 +90,7 @@ def browserSetup( options.add_argument("--disable-default-apps") options.add_argument("--disable-features=Translate") options.add_argument("--disable-features=PrivacySandboxSettings4") + options.add_argument("--disable-http2") options.add_argument("--disable-search-engine-choice-screen") #153 seleniumwireOptions: dict[str, Any] = {"verify_ssl": False} @@ -205,8 +206,9 @@ def getCCodeLang(lang: str, geo: str) -> tuple: try: nfo = ipapi.location() except RateLimited: - logging.warning("Returning default", exc_info=True) - return "en", "US" + geo = Utils.loadConfig().get("default_geolocation", "US").upper() + logging.warning(f"Returning default geolocation {geo}", exc_info=True) + return "en", geo if isinstance(nfo, dict): if lang is None: lang = nfo["languages"].split(",")[0].split("-")[0] diff --git a/src/morePromotions.py b/src/morePromotions.py index fd686b74..047ed7cb 100644 --- a/src/morePromotions.py +++ b/src/morePromotions.py @@ -125,6 +125,6 @@ def completeMorePromotions(self): for promotion in self.browser.utils.getDashboardData()["morePromotions"]: # Have to refresh if promotion["pointProgress"] < promotion["pointProgressMax"]: incompletePromotions.append((promotion["title"], promotion["promotionType"])) - if incompletePromotions: - Utils.sendNotification("Incomplete promotions(s)", incompletePromotions) + if incompletePromotions and Utils.loadConfig().get("apprise", {}).get("promotions", True): + Utils.sendNotification(f"Incomplete promotions(s) for {self.browser.username}", incompletePromotions) logging.info("[MORE PROMOS] Exiting") diff --git a/src/utils.py b/src/utils.py index 5b2fc555..7e4acc53 100644 --- a/src/utils.py +++ b/src/utils.py @@ -60,7 +60,7 @@ def sendNotification(title, body) -> None: if Utils.args.disable_apprise: return apprise = Apprise() - urls: list[str] = Utils.loadConfig("config-private.yaml").get("apprise", {}).get("urls", []) + urls: list[str] = Utils.loadConfig().get("apprise", {}).get("urls", []) if not urls: logging.debug("No urls found, not sending notification") return From dcdfdce68966b61689130ea24d6b765585074b84 Mon Sep 17 00:00:00 2001 From: Guido30 Date: Sun, 8 Sep 2024 18:32:44 +0200 Subject: [PATCH 11/82] Restored configurarion files --- .gitignore | 2 +- .template-config-private.yaml | 5 +++++ README.md | 2 +- config.yaml | 11 +++++++++++ config.yaml.sample | 13 ------------- main.py | 17 ++++++++--------- src/browser.py | 2 +- src/morePromotions.py | 2 +- src/utils.py | 7 ++++--- 9 files changed, 32 insertions(+), 29 deletions(-) create mode 100644 .template-config-private.yaml create mode 100644 config.yaml delete mode 100644 config.yaml.sample diff --git a/.gitignore b/.gitignore index d3a9e9eb..7302770d 100644 --- a/.gitignore +++ b/.gitignore @@ -186,4 +186,4 @@ runbot.bat /google_trends.dat /google_trends.dir /google_trends.bak -/config.yaml +/config-private.yaml diff --git a/.template-config-private.yaml b/.template-config-private.yaml new file mode 100644 index 00000000..75f03a62 --- /dev/null +++ b/.template-config-private.yaml @@ -0,0 +1,5 @@ +# config-private.yaml +apprise: + urls: + - "discord://{WebhookID}/{WebhookToken}" # Replace with your actual Apprise service URLs +default_geolocation: US # Replace with your country code https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2 diff --git a/README.md b/README.md index 336af401..3536a601 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ this [link](https://learn.microsoft.com/en-GB/cpp/windows/latest-supported-vc-redist?view=msvc-170) and reboot your computer -4. Edit the `config.yaml.sample` accordingly and rename it by removing `.sample` at the end. +4. Edit the `.template-config-private.yaml` accordingly and rename it to `config-private.yaml`. 5. Edit the `accounts.json.sample` with your accounts credentials and rename it by removing `.sample` at the end. diff --git a/config.yaml b/config.yaml new file mode 100644 index 00000000..dcada04d --- /dev/null +++ b/config.yaml @@ -0,0 +1,11 @@ +# config.yaml +apprise: + summary: ON_ERROR + notify: + uncaught-exceptions: True # True or False + incomplete-promotions: True # True or False +retries: + base_delay_in_seconds: 14.0625 # base_delay_in_seconds * 2^max = 14.0625 * 2^6 = 900 = 15 minutes + max: 8 + strategy: EXPONENTIAL +logging: DEBUG # DEBUG or INFO diff --git a/config.yaml.sample b/config.yaml.sample deleted file mode 100644 index 3a6dc166..00000000 --- a/config.yaml.sample +++ /dev/null @@ -1,13 +0,0 @@ -# RENAME THIS FILE TO config.yaml ONCE CONFIGURED -apprise: - summary: ON_ERROR - exceptions: True # True or False (Whether to send or not exceptions) - promotions: True # True or False (Whether to send or not incomplete promotions) - urls: - - 'discord://{WebhookID}/{WebhookToken}' # Replace with your actual Apprise service URLs -retries: - base_delay_in_seconds: 14.0625 # base_delay_in_seconds * 2^max = 14.0625 * 2^6 = 900 = 15 minutes - max: 8 - strategy: EXPONENTIAL -default_geolocation: US # Replace with your country code https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2 -logging: DEBUG # DEBUG or INFO diff --git a/main.py b/main.py index 041685d9..1a809ac3 100644 --- a/main.py +++ b/main.py @@ -40,11 +40,11 @@ def main(): earned_points = executeBot(currentAccount, args) except Exception as e1: logging.error("", exc_info=True) - if Utils.loadConfig().get("apprise", {}).get("exceptions", True): - Utils.sendNotification( - f"⚠️ Error executing {currentAccount.username}, please check the log", - traceback.format_exc(), - ) + Utils.sendNotification( + f"⚠️ Error executing {currentAccount.username}, please check the log", + traceback.format_exc(), + True + ) continue previous_points = previous_points_data.get(currentAccount.username, 0) @@ -366,7 +366,6 @@ def save_previous_points_data(data): main() except Exception as e: logging.exception("") - if Utils.loadConfig().get("apprise", {}).get("exceptions", True): - Utils.sendNotification( - "⚠️ Error occurred, please check the log", traceback.format_exc() - ) + Utils.sendNotification( + "⚠️ Error occurred, please check the log", traceback.format_exc(), True + ) diff --git a/src/browser.py b/src/browser.py index df08d118..7f95a88f 100644 --- a/src/browser.py +++ b/src/browser.py @@ -206,7 +206,7 @@ def getCCodeLang(lang: str, geo: str) -> tuple: try: nfo = ipapi.location() except RateLimited: - geo = Utils.loadConfig().get("default_geolocation", "US").upper() + geo = Utils.loadConfig("config-private.yaml").get("default_geolocation", "US").upper() logging.warning(f"Returning default geolocation {geo}", exc_info=True) return "en", geo if isinstance(nfo, dict): diff --git a/src/morePromotions.py b/src/morePromotions.py index 047ed7cb..065b8778 100644 --- a/src/morePromotions.py +++ b/src/morePromotions.py @@ -125,6 +125,6 @@ def completeMorePromotions(self): for promotion in self.browser.utils.getDashboardData()["morePromotions"]: # Have to refresh if promotion["pointProgress"] < promotion["pointProgressMax"]: incompletePromotions.append((promotion["title"], promotion["promotionType"])) - if incompletePromotions and Utils.loadConfig().get("apprise", {}).get("promotions", True): + if incompletePromotions and Utils.loadConfig().get("apprise", {}).get("notify", {}).get("incomplete-promotions", True): Utils.sendNotification(f"Incomplete promotions(s) for {self.browser.username}", incompletePromotions) logging.info("[MORE PROMOS] Exiting") diff --git a/src/utils.py b/src/utils.py index 7e4acc53..13ff6c51 100644 --- a/src/utils.py +++ b/src/utils.py @@ -56,11 +56,12 @@ def loadConfig(configFilename="config.yaml") -> dict: return {} @staticmethod - def sendNotification(title, body) -> None: - if Utils.args.disable_apprise: + def sendNotification(title, body, is_exception=False) -> None: + is_exception_allowed = Utils.loadConfig().get("apprise", {}).get("notify", {}).get("uncaught-exceptions", True) + if Utils.args.disable_apprise or (is_exception and not is_exception_allowed): return apprise = Apprise() - urls: list[str] = Utils.loadConfig().get("apprise", {}).get("urls", []) + urls: list[str] = Utils.loadConfig("config-private.yaml").get("apprise", {}).get("urls", []) if not urls: logging.debug("No urls found, not sending notification") return From cc7d4c288f114f9a5f181415a3720f578991ae04 Mon Sep 17 00:00:00 2001 From: Omega Blurz <81272128+OmegaBlurz@users.noreply.github.com> Date: Mon, 9 Sep 2024 10:09:05 +0100 Subject: [PATCH 12/82] Update morePromotions.py Couple more again, tried to get another called "Find somewhere new to explore" but I couldn't get it to trigger. --- src/morePromotions.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/morePromotions.py b/src/morePromotions.py index a3b9ddf2..d63f99af 100644 --- a/src/morePromotions.py +++ b/src/morePromotions.py @@ -98,6 +98,12 @@ def completeMorePromotions(self): elif "Houses near you" in promotionTitle: searchbar.send_keys("apartments manhattan") searchbar.submit() + elif "Prepare for the weather" in promotionTitle: + searchbar.send_keys("weather") + searchbar.submit() + elif "Get your shopping done faster" in promotionTitle: + searchbar.send_keys("chicken tenders") + searchbar.submit() elif promotion["promotionType"] == "urlreward": # Complete search for URL reward self.activities.completeSearch() From 4d8ed734ff892e789bc33550f58d87894d2cc4d5 Mon Sep 17 00:00:00 2001 From: belgio99 <17623601+belgio99@users.noreply.github.com> Date: Mon, 9 Sep 2024 12:40:09 +0200 Subject: [PATCH 13/82] First Docker implementation (cherry picked from commit 33ee0e69ba8e18e4db67b91178a7b725125fc45a) --- Dockerfile | 7 +++++++ docker.sh | 32 ++++++++++++++++++++++++++++++++ src/browser.py | 36 ++++++++++++++++++++++++++---------- 3 files changed, 65 insertions(+), 10 deletions(-) create mode 100644 Dockerfile create mode 100644 docker.sh diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..87dbdb4a --- /dev/null +++ b/Dockerfile @@ -0,0 +1,7 @@ +FROM python:slim +COPY . /app +WORKDIR /app +RUN apt update && apt install -y cron chromium chromium-driver +RUN pip install -r requirements.txt +ENV DOCKER=1 +CMD ["sh", "docker.sh"] \ No newline at end of file diff --git a/docker.sh b/docker.sh new file mode 100644 index 00000000..08dedc2e --- /dev/null +++ b/docker.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# Check if RUN_ONCE environment variable is set. In case, running the script now and exiting. +if [ "$RUN_ONCE" = "true" ] +then + echo "RUN_ONCE environment variable is set. Running the script now and exiting." + python main.py + exit 0 +fi +# Check if CRON_SCHEDULE environment variable is set +if [ -z "$CRON_SCHEDULE" ] +then + echo "CRON_SCHEDULE environment variable is not set. Setting it to 4 AM everyday by default" + CRON_SCHEDULE="0 4 * * *" +fi + +# Setting up cron job +echo "$CRON_SCHEDULE root python /app/main.py >> /var/log/cron.log 2>&1" > /etc/cron.d/rewards-cron-job + +# Give execution rights on the cron job +chmod 0644 /etc/cron.d/rewards-cron-job + +# Apply cron job +crontab /etc/cron.d/rewards-cron-job + +# Create the log file to be able to run tail +touch /var/log/cron.log + +echo "Cron job is set to run at $CRON_SCHEDULE. Waiting for the cron to run..." + +# Run the cron +cron && tail -f /var/log/cron.log diff --git a/src/browser.py b/src/browser.py index 93215280..10f67abd 100644 --- a/src/browser.py +++ b/src/browser.py @@ -1,5 +1,6 @@ import argparse import logging +import os import random from pathlib import Path from types import TracebackType @@ -83,6 +84,12 @@ def browserSetup( options.add_argument("--ignore-certificate-errors") options.add_argument("--ignore-certificate-errors-spki-list") options.add_argument("--ignore-ssl-errors") + if os.environ.get("DOCKER"): + options.add_argument("--no-sandbox") + options.add_argument("--disable-dev-shm-usage") + options.add_argument("--disable-gpu") + options.add_argument("--disable-extensions") + options.add_argument("--headless=new") options.add_argument("--no-sandbox") options.add_argument("--disable-extensions") options.add_argument("--dns-prefetch-disable") @@ -101,17 +108,26 @@ def browserSetup( "https": self.proxy, "no_proxy": "localhost,127.0.0.1", } + driver = None + + if os.environ.get("DOCKER"): + driver = webdriver.Chrome( + options=options, + seleniumwire_options=seleniumwireOptions, + user_data_dir=self.userDataDir.as_posix(), + driver_executable_path="/usr/bin/chromedriver", + ) + else: + # Obtain webdriver chrome driver version + version = self.getChromeVersion() + major = int(version.split(".")[0]) - # Obtain webdriver chrome driver version - version = self.getChromeVersion() - major = int(version.split(".")[0]) - - driver = webdriver.Chrome( - options=options, - seleniumwire_options=seleniumwireOptions, - user_data_dir=self.userDataDir.as_posix(), - version_main=major, - ) + driver = webdriver.Chrome( + options=options, + seleniumwire_options=seleniumwireOptions, + user_data_dir=self.userDataDir.as_posix(), + version_main=major, + ) seleniumLogger = logging.getLogger("seleniumwire") seleniumLogger.setLevel(logging.ERROR) From ab079fc7346faf9f5efe85dac9eb94f728ca4ce1 Mon Sep 17 00:00:00 2001 From: belgio99 <17623601+belgio99@users.noreply.github.com> Date: Mon, 9 Sep 2024 12:40:26 +0200 Subject: [PATCH 14/82] Removed duplicate options from cherry-pick --- src/browser.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/browser.py b/src/browser.py index 10f67abd..5054bf8a 100644 --- a/src/browser.py +++ b/src/browser.py @@ -85,10 +85,7 @@ def browserSetup( options.add_argument("--ignore-certificate-errors-spki-list") options.add_argument("--ignore-ssl-errors") if os.environ.get("DOCKER"): - options.add_argument("--no-sandbox") options.add_argument("--disable-dev-shm-usage") - options.add_argument("--disable-gpu") - options.add_argument("--disable-extensions") options.add_argument("--headless=new") options.add_argument("--no-sandbox") options.add_argument("--disable-extensions") From 245adb1574516759950bbebf14d8839cb5a29886 Mon Sep 17 00:00:00 2001 From: belgio99 <17623601+belgio99@users.noreply.github.com> Date: Mon, 9 Sep 2024 12:40:29 +0200 Subject: [PATCH 15/82] Added first GitHub Action to push on Docker Hub (cherry picked from commit 1e4977ba4ca7522d71aedaf44d9965d0f7f8183a) --- .github/workflows/build_push_docker_image.yml | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 .github/workflows/build_push_docker_image.yml diff --git a/.github/workflows/build_push_docker_image.yml b/.github/workflows/build_push_docker_image.yml new file mode 100644 index 00000000..5bc3323c --- /dev/null +++ b/.github/workflows/build_push_docker_image.yml @@ -0,0 +1,31 @@ +name: Build and Push Docker Image + +on: + push: + branches: + - docker + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - name: Checkout Code + uses: actions/checkout@v4 + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Build and push + uses: docker/build-push-action@v5 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: true + tags: supercar99/ms-rewards:latest From b2160cd7df39b9afa93350bc4a1d7188448f7ecd Mon Sep 17 00:00:00 2001 From: belgio99 <17623601+belgio99@users.noreply.github.com> Date: Mon, 9 Sep 2024 12:40:36 +0200 Subject: [PATCH 16/82] Added Docker image versioning based on GitHub tags (cherry picked from commit 8bb53810d5516560104c9646d1ec7a9813174de8) --- .github/workflows/build_push_docker_image.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_push_docker_image.yml b/.github/workflows/build_push_docker_image.yml index 5bc3323c..4dc09c64 100644 --- a/.github/workflows/build_push_docker_image.yml +++ b/.github/workflows/build_push_docker_image.yml @@ -2,8 +2,8 @@ name: Build and Push Docker Image on: push: - branches: - - docker + tags: + - "v*" workflow_dispatch: jobs: @@ -13,6 +13,11 @@ jobs: steps: - name: Checkout Code uses: actions/checkout@v4 + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: supercar99/ms-rewards - name: Set up QEMU uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx @@ -28,4 +33,5 @@ jobs: context: . platforms: linux/amd64,linux/arm64 push: true - tags: supercar99/ms-rewards:latest + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} From 97a4616db7a352f0d22894013b04c4d3f6eefd4b Mon Sep 17 00:00:00 2001 From: belgio99 <17623601+belgio99@users.noreply.github.com> Date: Mon, 9 Sep 2024 12:44:15 +0200 Subject: [PATCH 17/82] Added 15s timeout to support 2FA (cherry picked from commit a0a9651c8264b894331c9d63527a4dec52a3d7c1) --- src/login.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/login.py b/src/login.py index d42a84f5..30ca4d9a 100644 --- a/src/login.py +++ b/src/login.py @@ -1,6 +1,7 @@ import argparse import contextlib import logging +import os from argparse import Namespace from pyotp import TOTP From 7893e121bcddb91e05103e694634e2190891028c Mon Sep 17 00:00:00 2001 From: belgio99 <17623601+belgio99@users.noreply.github.com> Date: Mon, 9 Sep 2024 12:44:28 +0200 Subject: [PATCH 18/82] chore: Update Dockerfile to use apt-get instead of apt, and pip --no-cache-dir instead of pip --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 87dbdb4a..bc12f2f5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM python:slim COPY . /app WORKDIR /app -RUN apt update && apt install -y cron chromium chromium-driver -RUN pip install -r requirements.txt +RUN apt-get update && apt-get install -y cron chromium chromium-driver +RUN pip install --no-cache-dir -r requirements.txt ENV DOCKER=1 CMD ["sh", "docker.sh"] \ No newline at end of file From 653392f4bd06ad5c35baf698fe8a4df01b3269e1 Mon Sep 17 00:00:00 2001 From: belgio99 <17623601+belgio99@users.noreply.github.com> Date: Mon, 9 Sep 2024 12:44:29 +0200 Subject: [PATCH 19/82] Delete the apt-get lists after installing packages --- Dockerfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index bc12f2f5..e4595adf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,9 @@ FROM python:slim COPY . /app WORKDIR /app -RUN apt-get update && apt-get install -y cron chromium chromium-driver -RUN pip install --no-cache-dir -r requirements.txt +RUN apt-get update && apt-get install -y cron chromium chromium-driver \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* +RUN pip install --no-cache-dir -r requirements.txt ENV DOCKER=1 CMD ["sh", "docker.sh"] \ No newline at end of file From b08ae39f91dbf5120c5888f79c4f8b49b57ba7ed Mon Sep 17 00:00:00 2001 From: belgio99 <17623601+belgio99@users.noreply.github.com> Date: Mon, 9 Sep 2024 12:54:50 +0200 Subject: [PATCH 20/82] Removed unused 2FA import --- src/login.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/login.py b/src/login.py index 30ca4d9a..d42a84f5 100644 --- a/src/login.py +++ b/src/login.py @@ -1,7 +1,6 @@ import argparse import contextlib import logging -import os from argparse import Namespace from pyotp import TOTP From c8082efaa6e999ff7068932deb4c8cbee7ce6848 Mon Sep 17 00:00:00 2001 From: belgio99 <17623601+belgio99@users.noreply.github.com> Date: Mon, 9 Sep 2024 13:00:32 +0200 Subject: [PATCH 21/82] Fixed Docker semantic versioning --- .github/workflows/build_push_docker_image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_push_docker_image.yml b/.github/workflows/build_push_docker_image.yml index 4dc09c64..5dee025d 100644 --- a/.github/workflows/build_push_docker_image.yml +++ b/.github/workflows/build_push_docker_image.yml @@ -3,7 +3,7 @@ name: Build and Push Docker Image on: push: tags: - - "v*" + - "*" workflow_dispatch: jobs: From ab4a8a7d245c6ebdbd3f08c8fc6afd6f4e2cb5ec Mon Sep 17 00:00:00 2001 From: belgio99 <17623601+belgio99@users.noreply.github.com> Date: Mon, 9 Sep 2024 13:35:25 +0200 Subject: [PATCH 22/82] Change action to match username specified in secret --- .github/workflows/build_push_docker_image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_push_docker_image.yml b/.github/workflows/build_push_docker_image.yml index 5dee025d..4bf039a6 100644 --- a/.github/workflows/build_push_docker_image.yml +++ b/.github/workflows/build_push_docker_image.yml @@ -17,7 +17,7 @@ jobs: id: meta uses: docker/metadata-action@v5 with: - images: supercar99/ms-rewards + images: ${{ secrets.DOCKERHUB_USERNAME }}/ms-rewards - name: Set up QEMU uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx From fa5194ff4897495bc6cb56825d1b0a7c08b893c2 Mon Sep 17 00:00:00 2001 From: klept0 Date: Mon, 9 Sep 2024 08:58:26 -0400 Subject: [PATCH 23/82] Update morePromotions.py edited some wording on apprise notification --- src/morePromotions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/morePromotions.py b/src/morePromotions.py index 795fd40e..ddb362a8 100644 --- a/src/morePromotions.py +++ b/src/morePromotions.py @@ -135,5 +135,5 @@ def completeMorePromotions(self): if promotion["pointProgress"] < promotion["pointProgressMax"]: incompletePromotions.append((promotion["title"], promotion["promotionType"])) if incompletePromotions and Utils.loadConfig().get("apprise", {}).get("notify", {}).get("incomplete-promotions", True): - Utils.sendNotification(f"Incomplete promotions(s) for {self.browser.username}", incompletePromotions) + Utils.sendNotification(f"We found some incomplete promotions for {self.browser.username} to do!", incompletePromotions) logging.info("[MORE PROMOS] Exiting") From ed8c7d6c27e6002f3c73369e6e917141e6c4481b Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Mon, 9 Sep 2024 13:29:27 -0400 Subject: [PATCH 24/82] Add config singletons, combine default configs --- .template-config-private.yaml | 1 - config.yaml | 11 +++-- main.py | 25 +++------- src/browser.py | 7 +-- src/morePromotions.py | 16 +++--- src/searches.py | 19 ++------ src/utils.py | 91 +++++++++++++++++++++++++++-------- 7 files changed, 101 insertions(+), 69 deletions(-) diff --git a/.template-config-private.yaml b/.template-config-private.yaml index 75f03a62..2c8b441a 100644 --- a/.template-config-private.yaml +++ b/.template-config-private.yaml @@ -2,4 +2,3 @@ apprise: urls: - "discord://{WebhookID}/{WebhookToken}" # Replace with your actual Apprise service URLs -default_geolocation: US # Replace with your country code https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2 diff --git a/config.yaml b/config.yaml index dcada04d..7321d4d8 100644 --- a/config.yaml +++ b/config.yaml @@ -1,11 +1,14 @@ # config.yaml apprise: - summary: ON_ERROR notify: - uncaught-exceptions: True # True or False incomplete-promotions: True # True or False + uncaught-exceptions: True # True or False + summary: ON_ERROR +default: + geolocation: US # Replace with your country code https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2 +logging: + level: DEBUG # See https://docs.python.org/3/library/logging.html#logging-levels retries: base_delay_in_seconds: 14.0625 # base_delay_in_seconds * 2^max = 14.0625 * 2^6 = 900 = 15 minutes - max: 8 + max: 4 strategy: EXPONENTIAL -logging: DEBUG # DEBUG or INFO diff --git a/main.py b/main.py index 1a809ac3..ccdb141d 100644 --- a/main.py +++ b/main.py @@ -23,7 +23,7 @@ ) from src.browser import RemainingSearches from src.loggingColoredFormatter import ColoredFormatter -from src.utils import Utils +from src.utils import Utils, CONFIG def main(): @@ -40,11 +40,8 @@ def main(): earned_points = executeBot(currentAccount, args) except Exception as e1: logging.error("", exc_info=True) - Utils.sendNotification( - f"⚠️ Error executing {currentAccount.username}, please check the log", - traceback.format_exc(), - True - ) + Utils.sendNotification(f"⚠️ Error executing {currentAccount.username}, please check the log", + traceback.format_exc(), e1) continue previous_points = previous_points_data.get(currentAccount.username, 0) @@ -99,16 +96,6 @@ def setupLogging(): logs_directory.mkdir(parents=True, exist_ok=True) # so only our code is logged if level=logging.DEBUG or finer - # if not working see https://stackoverflow.com/a/48891485/4164390 - _levels = { - 'DEBUG': logging.DEBUG, - 'INFO': logging.INFO, - 'WARNING': logging.WARNING, - 'ERROR': logging.ERROR, - 'CRITICAL': logging.CRITICAL - } - log_level_str = Utils.loadConfig().get("logging", "DEBUG").upper() - log_level = _levels.get(log_level_str, logging.DEBUG) logging.config.dictConfig( { "version": 1, @@ -116,7 +103,7 @@ def setupLogging(): } ) logging.basicConfig( - level=log_level, + level=logging.getLevelName(CONFIG.get("logging").get("level").upper()), format=_format, handlers=[ handlers.TimedRotatingFileHandler( @@ -291,7 +278,7 @@ def executeBot(currentAccount: Account, args: argparse.Namespace): f"[POINTS] You are now at {Utils.formatNumber(accountPoints)} points !" ) appriseSummary = AppriseSummary[ - Utils.loadConfig().get("apprise", {}).get("summary", AppriseSummary.ALWAYS.name) + CONFIG.get("apprise").get("summary") ] if appriseSummary == AppriseSummary.ALWAYS: goalStatus = "" @@ -367,5 +354,5 @@ def save_previous_points_data(data): except Exception as e: logging.exception("") Utils.sendNotification( - "⚠️ Error occurred, please check the log", traceback.format_exc(), True + "⚠️ Error occurred, please check the log", traceback.format_exc(), e ) diff --git a/src/browser.py b/src/browser.py index fb64837f..4931a775 100644 --- a/src/browser.py +++ b/src/browser.py @@ -15,7 +15,7 @@ from src import Account, RemainingSearches from src.userAgentGenerator import GenerateUserAgent -from src.utils import Utils +from src.utils import Utils, CONFIG class Browser: @@ -217,9 +217,10 @@ def setupProfiles(self) -> Path: def getCCodeLang(lang: str, geo: str) -> tuple: if lang is None or geo is None: try: + # fixme Find better way to get this that doesn't involve ip nfo = ipapi.location() except RateLimited: - geo = Utils.loadConfig("config-private.yaml").get("default_geolocation", "US").upper() + geo = CONFIG.get("default").get("geolocation", "US") logging.warning(f"Returning default geolocation {geo}", exc_info=True) return "en", geo if isinstance(nfo, dict): @@ -249,7 +250,7 @@ def getRemainingSearches( bingInfo = self.utils.getBingInfo() searchPoints = 1 counters = bingInfo["flyoutResult"]["userStatus"]["counters"] - pcSearch: dict = counters["PCSearch"][0] + pcSearch: dict = counters["PCSearch"][0] mobileSearch: dict = counters["MobileSearch"][0] pointProgressMax: int = pcSearch["pointProgressMax"] diff --git a/src/morePromotions.py b/src/morePromotions.py index ddb362a8..9c10f556 100644 --- a/src/morePromotions.py +++ b/src/morePromotions.py @@ -8,7 +8,7 @@ from src.browser import Browser from .activities import Activities -from .utils import Utils +from .utils import Utils, CONFIG # todo Rename MoreActivities? @@ -130,10 +130,12 @@ def completeMorePromotions(self): # Reset tabs in case of an exception self.browser.utils.resetTabs() continue - incompletePromotions: list[tuple[str, str]] = [] - for promotion in self.browser.utils.getDashboardData()["morePromotions"]: # Have to refresh - if promotion["pointProgress"] < promotion["pointProgressMax"]: - incompletePromotions.append((promotion["title"], promotion["promotionType"])) - if incompletePromotions and Utils.loadConfig().get("apprise", {}).get("notify", {}).get("incomplete-promotions", True): - Utils.sendNotification(f"We found some incomplete promotions for {self.browser.username} to do!", incompletePromotions) + if CONFIG.get("apprise").get("notify").get("incomplete-promotions"): + incompletePromotions: list[tuple[str, str]] = [] + for promotion in self.browser.utils.getDashboardData()["morePromotions"]: # Have to refresh + if promotion["pointProgress"] < promotion["pointProgressMax"]: + incompletePromotions.append((promotion["title"], promotion["promotionType"])) + if incompletePromotions: + Utils.sendNotification(f"We found some incomplete promotions for {self.browser.username} to do!", + incompletePromotions) logging.info("[MORE PROMOS] Exiting") diff --git a/src/searches.py b/src/searches.py index a5b1a3b6..08d4c749 100644 --- a/src/searches.py +++ b/src/searches.py @@ -1,4 +1,3 @@ -import contextlib import dbm.dumb import json import logging @@ -11,14 +10,10 @@ from typing import Final import requests -from selenium.common import TimeoutException from selenium.webdriver.common.by import By -from selenium.webdriver.remote.webelement import WebElement -from selenium.webdriver.support import expected_conditions -from selenium.webdriver.support.wait import WebDriverWait from src.browser import Browser -from src.utils import Utils +from src.utils import Utils, CONFIG class RetriesStrategy(Enum): @@ -37,21 +32,16 @@ class RetriesStrategy(Enum): class Searches: - config = Utils.loadConfig() - maxRetries: Final[int] = config.get("retries", {}).get("max", 8) + maxRetries: Final[int] = CONFIG.get("retries").get("max") """ the max amount of retries to attempt """ - baseDelay: Final[float] = config.get("retries", {}).get( - "base_delay_in_seconds", 14.0625 - ) + baseDelay: Final[float] = CONFIG.get("retries").get("base_delay_in_seconds") """ how many seconds to delay """ # retriesStrategy = Final[ # todo Figure why doesn't work with equality below - retriesStrategy = RetriesStrategy[ - config.get("retries", {}).get("strategy", RetriesStrategy.CONSTANT.name) - ] + retriesStrategy = RetriesStrategy[CONFIG.get("retries").get("strategy")] def __init__(self, browser: Browser): self.browser = browser @@ -138,7 +128,6 @@ def bingSearches(self) -> None: f"google_trends after load = {list(self.googleTrendsShelf.items())}" ) - # fixme Multiple tabs are getting opened self.bingSearch() time.sleep(random.randint(10, 15)) diff --git a/src/utils.py b/src/utils.py index 6d3df45f..2f15cf13 100644 --- a/src/utils.py +++ b/src/utils.py @@ -6,6 +6,7 @@ import time from argparse import Namespace from pathlib import Path +from types import MappingProxyType from typing import Any import requests @@ -13,8 +14,12 @@ from apprise import Apprise from requests import Session from requests.adapters import HTTPAdapter -from selenium.common import NoSuchElementException, TimeoutException, ElementClickInterceptedException, \ - ElementNotInteractableException +from selenium.common import ( + NoSuchElementException, + TimeoutException, + ElementClickInterceptedException, + ElementNotInteractableException, +) from selenium.webdriver.chrome.webdriver import WebDriver from selenium.webdriver.common.by import By from selenium.webdriver.remote.webelement import WebElement @@ -25,6 +30,23 @@ from .constants import REWARDS_URL from .constants import SEARCH_URL +DEFAULT_CONFIG: MappingProxyType = MappingProxyType( + { + "apprise": { + "notify": {"incomplete-promotions": True, "uncaught-exceptions": True}, + "summary": "ALWAYS", + }, + "default": None, + "logging": {"level": "INFO"}, + "retries": {"base_delay_in_seconds": 14.0625, "max": 4, "strategy": "EXPONENTIAL"}, + }) +DEFAULT_PRIVATE_CONFIG: MappingProxyType = MappingProxyType( + { + "apprise": { + "urls": [], + }, + }) + class Utils: args: Namespace @@ -35,33 +57,43 @@ def __init__(self, webdriver: WebDriver): locale = pylocale.getdefaultlocale()[0] pylocale.setlocale(pylocale.LC_NUMERIC, locale) - self.config = self.loadConfig() + # self.config = self.loadConfig() @staticmethod def getProjectRoot() -> Path: return Path(__file__).parent.parent @staticmethod - def loadConfig(configFilename="config.yaml") -> dict: + def loadYaml(path: Path) -> dict: + with open(path, "r") as file: + yamlContents = yaml.safe_load(file) + if not yamlContents: + logging.info(f"{yamlContents} is empty") + yamlContents = {} + return yamlContents + + @staticmethod + def loadConfig(configFilename="config.yaml", defaultConfig=DEFAULT_CONFIG) -> MappingProxyType: configFile = Utils.getProjectRoot() / configFilename try: - with open(configFile, "r") as file: - config = yaml.safe_load(file) - if not config: - logging.info(f"{file} doesn't exist") - return {} - return config + return MappingProxyType(defaultConfig | Utils.loadYaml(configFile)) except OSError: - logging.warning(f"{configFilename} doesn't exist") - return {} + logging.info(f"{configFile} doesn't exist, returning defaults") + return defaultConfig + + @staticmethod + def loadPrivateConfig() -> MappingProxyType: + return Utils.loadConfig("config-private.yaml", DEFAULT_PRIVATE_CONFIG) @staticmethod - def sendNotification(title, body, is_exception=False) -> None: - is_exception_allowed = Utils.loadConfig().get("apprise", {}).get("notify", {}).get("uncaught-exceptions", True) - if Utils.args.disable_apprise or (is_exception and not is_exception_allowed): + def sendNotification(title, body, e: Exception = None) -> None: + if Utils.args.disable_apprise or (e and not CONFIG.get("apprise").get("notify").get("uncaught-exceptions")): return apprise = Apprise() - urls: list[str] = Utils.loadConfig("config-private.yaml").get("apprise", {}).get("urls", []) + urls: list[str] = ( + # Utils.loadConfig("config-private.yaml").get("apprise", {}).get("urls", []) + PRIVATE_CONFIG.get("apprise").get("urls") + ) if not urls: logging.debug("No urls found, not sending notification") return @@ -147,12 +179,20 @@ def getBingInfo(self) -> Any: response = session.get("https://www.bing.com/rewards/panelflyout/getuserinfo") assert response.status_code == requests.codes.ok + # fixme Add more asserts return response.json() @staticmethod def makeRequestsSession(session: Session = requests.session()) -> Session: retry = Retry( - total=5, backoff_factor=1, status_forcelist=[500, 502, 503, 504] + total=5, + backoff_factor=1, + status_forcelist=[ + 500, + 502, + 503, + 504, + ], # todo Use global retries from config ) session.mount( "https://", HTTPAdapter(max_retries=retry) @@ -196,7 +236,10 @@ def tryDismissAllMessages(self) -> None: for button in buttons: try: elements = self.webdriver.find_elements(by=button[0], value=button[1]) - except (NoSuchElementException, ElementNotInteractableException): # Expected? + except ( + NoSuchElementException, + ElementNotInteractableException, + ): # Expected? logging.debug("", exc_info=True) continue for element in elements: @@ -205,13 +248,17 @@ def tryDismissAllMessages(self) -> None: self.tryDismissBingCookieBanner() def tryDismissCookieBanner(self) -> None: - with contextlib.suppress(NoSuchElementException, ElementNotInteractableException): # Expected + with contextlib.suppress( + NoSuchElementException, ElementNotInteractableException + ): # Expected self.webdriver.find_element(By.ID, "cookie-banner").find_element( By.TAG_NAME, "button" ).click() def tryDismissBingCookieBanner(self) -> None: - with contextlib.suppress(NoSuchElementException, ElementNotInteractableException): # Expected + with contextlib.suppress( + NoSuchElementException, ElementNotInteractableException + ): # Expected self.webdriver.find_element(By.ID, "bnp_btn_accept").click() def switchToNewTab(self, timeToWait: float = 0) -> None: @@ -253,3 +300,7 @@ def click(self, element: WebElement) -> None: except (ElementClickInterceptedException, ElementNotInteractableException): self.tryDismissAllMessages() element.click() + + +CONFIG = Utils.loadConfig() +PRIVATE_CONFIG = Utils.loadPrivateConfig() From f726e578594aa18e2f5262105119c3c754305adc Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Mon, 9 Sep 2024 13:30:14 -0400 Subject: [PATCH 25/82] Make search simpler and remove failing terms completely --- src/searches.py | 34 ++++++++++------------------------ 1 file changed, 10 insertions(+), 24 deletions(-) diff --git a/src/searches.py b/src/searches.py index 08d4c749..3d9ffd03 100644 --- a/src/searches.py +++ b/src/searches.py @@ -146,6 +146,7 @@ def bingSearch(self) -> None: baseDelay = Searches.baseDelay logging.debug(f"rootTerm={rootTerm}") + # todo If first 3 searches of day, don't retry since points register differently, will be a bit quicker for i in range(self.maxRetries + 1): if i != 0: sleepTime: float @@ -161,28 +162,15 @@ def bingSearch(self) -> None: ) time.sleep(sleepTime) - searchbar: WebElement - for _ in range(1000): - searchbar = self.browser.utils.waitUntilClickable( - By.ID, "sb_form_q", timeToWait=40 - ) - searchbar.clear() - term = next(termsCycle) - logging.debug(f"term={term}") - time.sleep(1) - searchbar.send_keys(term) - time.sleep(1) - with contextlib.suppress(TimeoutException): - WebDriverWait(self.webdriver, 20).until( - expected_conditions.text_to_be_present_in_element_value( - (By.ID, "sb_form_q"), term - ) - ) - break - logging.debug("error send_keys") - else: - # todo Still happens occasionally, gotta be a fix - raise TimeoutException + searchbar = self.browser.utils.waitUntilClickable( + By.ID, "sb_form_q", timeToWait=40 + ) + searchbar.clear() + term = next(termsCycle) + logging.debug(f"term={term}") + time.sleep(1) + searchbar.send_keys(term) + time.sleep(1) searchbar.submit() pointsAfter = self.browser.utils.getAccountPoints() @@ -196,6 +184,4 @@ def bingSearch(self) -> None: # self.webdriver.proxy = self.browser.giveMeProxy() logging.error("[BING] Reached max search attempt retries") - logging.debug("Moving passedInTerm to end of list") del self.googleTrendsShelf[rootTerm] - self.googleTrendsShelf[rootTerm] = None From f8d5ba9287eb10ae6be389573f061087bda3b5cc Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Mon, 9 Sep 2024 13:32:10 -0400 Subject: [PATCH 26/82] Change log level to info --- config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.yaml b/config.yaml index 7321d4d8..1d65792a 100644 --- a/config.yaml +++ b/config.yaml @@ -7,7 +7,7 @@ apprise: default: geolocation: US # Replace with your country code https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2 logging: - level: DEBUG # See https://docs.python.org/3/library/logging.html#logging-levels + level: INFO # See https://docs.python.org/3/library/logging.html#logging-levels retries: base_delay_in_seconds: 14.0625 # base_delay_in_seconds * 2^max = 14.0625 * 2^6 = 900 = 15 minutes max: 4 From 8558132a152dbcebdf3c60944213731f122bee80 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Mon, 9 Sep 2024 13:36:06 -0400 Subject: [PATCH 27/82] Add assertion message --- src/searches.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/searches.py b/src/searches.py index 3d9ffd03..0a3f6bc0 100644 --- a/src/searches.py +++ b/src/searches.py @@ -70,7 +70,7 @@ def getGoogleTrends(self, wordsCount: int) -> list[str]: ) assert ( r.status_code == requests.codes.ok - ) # todo Add guidance if assertion fails + ), "Adjust retry config in src.utils.Utils.makeRequestsSession" trends = json.loads(r.text[6:]) for topic in trends["default"]["trendingSearchesDays"][0][ "trendingSearches" From 04e9e36640bc6a6465d18a907819feebc2e81606 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Mon, 9 Sep 2024 13:47:22 -0400 Subject: [PATCH 28/82] Make quicker --- src/utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/utils.py b/src/utils.py index 2f15cf13..7cf2453c 100644 --- a/src/utils.py +++ b/src/utils.py @@ -180,6 +180,7 @@ def getBingInfo(self) -> Any: assert response.status_code == requests.codes.ok # fixme Add more asserts + # todo Add fallback to src.utils.Utils.getDashboardData (slower but more reliable) return response.json() @staticmethod @@ -204,6 +205,8 @@ def makeRequestsSession(session: Session = requests.session()) -> Session: def isLoggedIn(self) -> bool: # return self.getBingInfo()["isRewardsUser"] # todo For some reason doesn't work, but doesn't involve changing url so preferred + if self.getBingInfo()["isRewardsUser"]: # faster, if it works + return True self.webdriver.get( "https://rewards.bing.com/Signin/" ) # changed site to allow bypassing when M$ blocks access to login.live.com randomly From 8a9df3da63470c04a263abe8164b5880eb5162a1 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Mon, 9 Sep 2024 13:47:33 -0400 Subject: [PATCH 29/82] Fix indent --- src/browser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/browser.py b/src/browser.py index 4931a775..4e8cc7ee 100644 --- a/src/browser.py +++ b/src/browser.py @@ -250,7 +250,7 @@ def getRemainingSearches( bingInfo = self.utils.getBingInfo() searchPoints = 1 counters = bingInfo["flyoutResult"]["userStatus"]["counters"] - pcSearch: dict = counters["PCSearch"][0] + pcSearch: dict = counters["PCSearch"][0] mobileSearch: dict = counters["MobileSearch"][0] pointProgressMax: int = pcSearch["pointProgressMax"] From 1e90f13a76230a30fec8243784f5c0ac4abfcee9 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Mon, 9 Sep 2024 21:09:37 -0400 Subject: [PATCH 30/82] Add new promo and fix some --- src/morePromotions.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/morePromotions.py b/src/morePromotions.py index 9c10f556..27dbcb83 100644 --- a/src/morePromotions.py +++ b/src/morePromotions.py @@ -89,21 +89,24 @@ def completeMorePromotions(self): elif "Gaming time" in promotionTitle: searchbar.send_keys("vampire survivors video game") searchbar.submit() - elif "Expand your vocabulary" in promotionTitle: - searchbar.send_keys("definition definition") - searchbar.submit() elif "What time is it?" in promotionTitle: searchbar.send_keys("china time") searchbar.submit() elif "Houses near you" in promotionTitle: searchbar.send_keys("apartments manhattan") searchbar.submit() - elif "Prepare for the weather" in promotionTitle: - searchbar.send_keys("weather") - searchbar.submit() elif "Get your shopping done faster" in promotionTitle: searchbar.send_keys("chicken tenders") searchbar.submit() + elif "Expand your vocabulary" in promotionTitle: + searchbar.send_keys("define polymorphism") + searchbar.submit() + elif "Stay on top of the elections" in promotionTitle: + searchbar.send_keys("election news latest") + searchbar.submit() + elif "Prepare for the weather" in promotionTitle: + searchbar.send_keys("weather tomorrow") + searchbar.submit() elif promotion["promotionType"] == "urlreward": # Complete search for URL reward self.activities.completeSearch() From de769836134315c128a13b5b02a042cdb204647e Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Mon, 9 Sep 2024 21:30:50 -0400 Subject: [PATCH 31/82] Reformat and add PROMOTION_TITLE_TO_SEARCH --- src/morePromotions.py | 107 +++++++++++++++--------------------------- 1 file changed, 39 insertions(+), 68 deletions(-) diff --git a/src/morePromotions.py b/src/morePromotions.py index 27dbcb83..2589b020 100644 --- a/src/morePromotions.py +++ b/src/morePromotions.py @@ -10,6 +10,29 @@ from .activities import Activities from .utils import Utils, CONFIG +PROMOTION_TITLE_TO_SEARCH = { + "Search the lyrics of a song": "black sabbath supernaut lyrics", + "Translate anything": "translate pencil sharpener to spanish", + "Let's watch that movie again!": "aliens movie", + "Discover open job roles": "walmart open job roles", + "Plan a quick getaway": "flights nyc to paris", + "You can track your package": "usps tracking", + "Find somewhere new to explore": "directions to new york", + "Too tired to cook tonight?": "Pizza Hut near me", + "Quickly convert your money": "convert 374 usd to yen", + "Learn to cook a new recipe": "how cook pierogi", + "Find places to stay": "hotels rome italy", + "How's the economy?": "sp 500", + "Who won?": "braves score", + "Gaming time": "vampire survivors video game", + "What time is it?": "china time", + "Houses near you": "apartments manhattan", + "Get your shopping done faster": "chicken tenders", + "Expand your vocabulary": "define polymorphism", + "Stay on top of the elections": "election news latest", + "Prepare for the weather": "weather tomorrow", +} + # todo Rename MoreActivities? class MorePromotions: @@ -27,7 +50,9 @@ def completeMorePromotions(self): self.browser.utils.goToRewards() for promotion in morePromotions: try: - promotionTitle = promotion["title"].replace("\u200b", "").replace("\xa0", " ") + promotionTitle = ( + promotion["title"].replace("\u200b", "").replace("\xa0", " ") + ) logging.debug(f"promotionTitle={promotionTitle}") # Open the activity for the promotion if ( @@ -46,73 +71,13 @@ def completeMorePromotions(self): ) self.browser.utils.click(searchbar) # todo These and following are US-English specific, maybe there's a good way to internationalize - # todo Could use dictionary of promotionTitle to search to simplify - if "Search the lyrics of a song" in promotionTitle: - searchbar.send_keys("black sabbath supernaut lyrics") - searchbar.submit() - elif "Translate anything" in promotionTitle: - searchbar.send_keys("translate pencil sharpener to spanish") - searchbar.submit() - elif "Let's watch that movie again!" in promotionTitle: - searchbar.send_keys("aliens movie") - searchbar.submit() - elif "Discover open job roles" in promotionTitle: - searchbar.send_keys("walmart open job roles") - searchbar.submit() - elif "Plan a quick getaway" in promotionTitle: - searchbar.send_keys("flights nyc to paris") - searchbar.submit() - elif "You can track your package" in promotionTitle: - searchbar.send_keys("usps tracking") - searchbar.submit() - elif "Find somewhere new to explore" in promotionTitle: - searchbar.send_keys("directions to new york") - searchbar.submit() - elif "Too tired to cook tonight?" in promotionTitle: - searchbar.send_keys("Pizza Hut near me") - searchbar.submit() - elif "Quickly convert your money" in promotionTitle: - searchbar.send_keys("convert 374 usd to yen") - searchbar.submit() - elif "Learn to cook a new recipe" in promotionTitle: - searchbar.send_keys("how cook pierogi") - searchbar.submit() - elif "Find places to stay" in promotionTitle: - searchbar.send_keys("hotels rome italy") - searchbar.submit() - elif "How's the economy?" in promotionTitle: - searchbar.send_keys("sp 500") - searchbar.submit() - elif "Who won?" in promotionTitle: - searchbar.send_keys("braves score") - searchbar.submit() - elif "Gaming time" in promotionTitle: - searchbar.send_keys("vampire survivors video game") - searchbar.submit() - elif "What time is it?" in promotionTitle: - searchbar.send_keys("china time") - searchbar.submit() - elif "Houses near you" in promotionTitle: - searchbar.send_keys("apartments manhattan") - searchbar.submit() - elif "Get your shopping done faster" in promotionTitle: - searchbar.send_keys("chicken tenders") - searchbar.submit() - elif "Expand your vocabulary" in promotionTitle: - searchbar.send_keys("define polymorphism") - searchbar.submit() - elif "Stay on top of the elections" in promotionTitle: - searchbar.send_keys("election news latest") - searchbar.submit() - elif "Prepare for the weather" in promotionTitle: - searchbar.send_keys("weather tomorrow") + if promotionTitle in PROMOTION_TITLE_TO_SEARCH: + searchbar.send_keys(PROMOTION_TITLE_TO_SEARCH[promotionTitle]) searchbar.submit() elif promotion["promotionType"] == "urlreward": # Complete search for URL reward self.activities.completeSearch() - elif ( - promotion["promotionType"] == "quiz" - ): + elif promotion["promotionType"] == "quiz": # Complete different types of quizzes based on point progress max if promotion["pointProgressMax"] == 10: self.activities.completeABC() @@ -135,10 +100,16 @@ def completeMorePromotions(self): continue if CONFIG.get("apprise").get("notify").get("incomplete-promotions"): incompletePromotions: list[tuple[str, str]] = [] - for promotion in self.browser.utils.getDashboardData()["morePromotions"]: # Have to refresh + for promotion in self.browser.utils.getDashboardData()[ + "morePromotions" + ]: # Have to refresh if promotion["pointProgress"] < promotion["pointProgressMax"]: - incompletePromotions.append((promotion["title"], promotion["promotionType"])) + incompletePromotions.append( + (promotion["title"], promotion["promotionType"]) + ) if incompletePromotions: - Utils.sendNotification(f"We found some incomplete promotions for {self.browser.username} to do!", - incompletePromotions) + Utils.sendNotification( + f"We found some incomplete promotions for {self.browser.username} to do!", + incompletePromotions, + ) logging.info("[MORE PROMOS] Exiting") From 3798ecbd61de46b0c026640e8b0aa66474dc277d Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Tue, 10 Sep 2024 21:21:42 -0400 Subject: [PATCH 32/82] Add comment --- src/activities.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/activities.py b/src/activities.py index 71b0c9e8..75521d07 100644 --- a/src/activities.py +++ b/src/activities.py @@ -1,12 +1,13 @@ import contextlib +import logging import random import time -import logging from selenium.common import TimeoutException +from selenium.common.exceptions import NoSuchElementException, ElementNotInteractableException from selenium.webdriver.common.by import By from selenium.webdriver.remote.webelement import WebElement -from selenium.common.exceptions import NoSuchElementException, ElementNotInteractableException + from src.browser import Browser @@ -69,6 +70,7 @@ def completeQuiz(self): with contextlib.suppress(TimeoutException): startQuiz = self.browser.utils.waitUntilQuizLoads() self.browser.utils.click(startQuiz) + # this is bugged on Chrome for some reason self.browser.utils.waitUntilVisible( By.ID, "overlayPanel", 5 ) From 14cc1d7391e7da1ea3f34149073f6d0713e2a495 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Tue, 10 Sep 2024 21:22:08 -0400 Subject: [PATCH 33/82] Add jitter and remove delete --- src/searches.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/searches.py b/src/searches.py index 0a3f6bc0..4c120b00 100644 --- a/src/searches.py +++ b/src/searches.py @@ -129,6 +129,7 @@ def bingSearches(self) -> None: ) self.bingSearch() + del self.googleTrendsShelf[list(self.googleTrendsShelf.keys())[0]] time.sleep(random.randint(10, 15)) logging.info( @@ -156,6 +157,7 @@ def bingSearch(self) -> None: sleepTime = baseDelay else: raise AssertionError + sleepTime += baseDelay * random.random() # Add jitter logging.debug( f"[BING] Search attempt not counted {i}/{Searches.maxRetries}, sleeping {sleepTime}" f" seconds..." @@ -175,7 +177,6 @@ def bingSearch(self) -> None: pointsAfter = self.browser.utils.getAccountPoints() if pointsBefore < pointsAfter: - del self.googleTrendsShelf[rootTerm] return # todo @@ -183,5 +184,3 @@ def bingSearch(self) -> None: # logging.info("[BING] " + "TIMED OUT GETTING NEW PROXY") # self.webdriver.proxy = self.browser.giveMeProxy() logging.error("[BING] Reached max search attempt retries") - - del self.googleTrendsShelf[rootTerm] From 93375b10ea9a7ffa4a2cffd99f3fcd2a0d326682 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Wed, 11 Sep 2024 13:59:50 -0400 Subject: [PATCH 34/82] Fix error when switching to new tab and optionally close it --- src/activities.py | 2 +- src/punchCards.py | 4 ++-- src/utils.py | 9 ++++----- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/activities.py b/src/activities.py index 75521d07..bfaa14b5 100644 --- a/src/activities.py +++ b/src/activities.py @@ -51,7 +51,7 @@ def openMorePromotionsActivity(self, cardId: int): element = self.webdriver.find_element(By.CSS_SELECTOR, f"#more-activities > .m-card-group > .ng-scope:nth-child({cardId + 1}) .ds-card-sec") self.browser.utils.click(element) - self.browser.utils.switchToNewTab(timeToWait=5) + self.browser.utils.switchToNewTab(timeToWait=8) def completeSearch(self): # Simulate completing a search activity diff --git a/src/punchCards.py b/src/punchCards.py index 28d24fb9..1561d020 100644 --- a/src/punchCards.py +++ b/src/punchCards.py @@ -23,7 +23,7 @@ def completePunchCard(self, url: str, childPromotions: dict): self.webdriver.find_element( By.XPATH, "//a[@class='offer-cta']/div" ).click() - self.browser.utils.visitNewTab(random.randint(13, 17)) + self.browser.utils.switchToNewTab(random.randint(13, 17), True) if child["promotionType"] == "quiz": self.webdriver.find_element( By.XPATH, "//a[@class='offer-cta']/div" @@ -99,6 +99,6 @@ def completePromotionalItems(self): self.webdriver.find_element( By.XPATH, '//*[@id="promo-item"]/section/div/div/div/span' ).click() - self.browser.utils.visitNewTab(8) + self.browser.utils.switchToNewTab(8, True) except Exception: logging.debug("", exc_info=True) diff --git a/src/utils.py b/src/utils.py index 7cf2453c..50e4a744 100644 --- a/src/utils.py +++ b/src/utils.py @@ -264,8 +264,11 @@ def tryDismissBingCookieBanner(self) -> None: ): # Expected self.webdriver.find_element(By.ID, "bnp_btn_accept").click() - def switchToNewTab(self, timeToWait: float = 0) -> None: + def switchToNewTab(self, timeToWait: float = 0, closeTab: bool = False) -> None: + time.sleep(timeToWait) self.webdriver.switch_to.window(window_name=self.webdriver.window_handles[1]) + if closeTab: + self.closeCurrentTab() def closeCurrentTab(self) -> None: self.webdriver.close() @@ -273,10 +276,6 @@ def closeCurrentTab(self) -> None: self.webdriver.switch_to.window(window_name=self.webdriver.window_handles[0]) time.sleep(0.5) - def visitNewTab(self, timeToWait: float = 0) -> None: - self.switchToNewTab(timeToWait) - self.closeCurrentTab() - @staticmethod def formatNumber(number, num_decimals=2) -> str: return pylocale.format_string( From 885f19c48fd17723880821bec5e11e8bf1db7245 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Wed, 11 Sep 2024 22:15:27 -0400 Subject: [PATCH 35/82] Clarify lang and geo options --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 3536a601..ba486a9b 100644 --- a/README.md +++ b/README.md @@ -88,8 +88,9 @@ ## Launch arguments - `-v/--visible` to disable headless -- `-l/--lang` to force a language (ex: en) +- `-l/--lang` to force a language (ex: en) see https://serpapi.com/google-languages for options - `-g/--geo` to force a searching geolocation (ex: US) + see https://serpapi.com/google-trends-locations for options `https://trends.google.com/trends/ for proper geolocation abbreviation for your choice. These MUST be uppercase!!!` - `-p/--proxy` to add a proxy to the whole program, supports http/https/socks4/socks5 (overrides per-account proxy in accounts.json) From 6b3c39ad67e5b006130135a05e4cb2254745ae82 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Wed, 11 Sep 2024 22:16:53 -0400 Subject: [PATCH 36/82] Reformat --- README.md | 43 +++++++++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index ba486a9b..ddd72413 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,8 @@ > [!IMPORTANT] -> If you are multi-accounting and abusing the service for which this is intended - **_DO NOT COMPLAIN ABOUT BANS!!!_** +> If you are multi-accounting and abusing the service for which this is intended - * +*_DO NOT COMPLAIN ABOUT BANS!!!_** @@ -42,20 +43,21 @@ 3. (Windows Only) Make sure Visual C++ redistributable DLLs are installed If they're not, install the current "vc_redist.exe" from - this [link](https://learn.microsoft.com/en-GB/cpp/windows/latest-supported-vc-redist?view=msvc-170) and reboot your - computer + this [link](https://learn.microsoft.com/en-GB/cpp/windows/latest-supported-vc-redist?view=msvc-170) + and reboot your computer 4. Edit the `.template-config-private.yaml` accordingly and rename it to `config-private.yaml`. -5. Edit the `accounts.json.sample` with your accounts credentials and rename it by removing `.sample` at the end. +5. Edit the `accounts.json.sample` with your accounts credentials and rename it by removing + `.sample` at the end. - The "totp" field is not mandatory, only enter your TOTP key if you use it for 2FA (if ommitting, don't keep - it as an empty string, remove the line completely). + The "totp" field is not mandatory, only enter your TOTP key if you use it for 2FA (if + ommitting, don't keep it as an empty string, remove the line completely). - The "proxy" field is not mandatory, you can omit it if you don't want to use proxy (don't keep it as an empty string, - remove the line completely). + The "proxy" field is not mandatory, you can omit it if you don't want to use proxy (don't + keep it as an empty string, remove the line completely). - - If you want to add more than one account, the syntax is the following: + - If you want to add more than one account, the syntax is the following: ```json [ @@ -80,10 +82,13 @@ 7. (Windows Only) You can set up automatic execution by generating a Task Scheduler XML file. - If you are a Windows user, run the `generate_task_xml.py` script to create a `.xml` file. After generating the file, import it into Task Scheduler to schedule automatic execution of the script. This will allow the script to run at the specified time without manual intervention. - - To import the XML file into Task Scheduler, see [this guide](https://superuser.com/a/485565/709704). + If you are a Windows user, run the `generate_task_xml.py` script to create a `.xml` file. + After generating the file, import it into Task Scheduler to schedule automatic execution of + the script. This will allow the script to run at the specified time without manual + intervention. + To import the XML file into Task Scheduler, + see [this guide](https://superuser.com/a/485565/709704). ## Launch arguments @@ -92,12 +97,13 @@ - `-g/--geo` to force a searching geolocation (ex: US) see https://serpapi.com/google-trends-locations for options `https://trends.google.com/trends/ for proper geolocation abbreviation for your choice. These MUST be uppercase!!!` -- `-p/--proxy` to add a proxy to the whole program, supports http/https/socks4/socks5 (overrides per-account proxy in - accounts.json) +- `-p/--proxy` to add a proxy to the whole program, supports http/https/socks4/socks5 ( + overrides per-account proxy in accounts.json) `(ex: http://user:pass@host:port)` - `-cv/--chromeversion` to use a specific version of chrome `(ex: 118)` -- `-da/--disable-apprise` disables Apprise notifications for the session, overriding [config.yaml](config.yaml). +- `-da/--disable-apprise` disables Apprise notifications for the session, + overriding [config.yaml](config.yaml). Useful when running manually as opposed to on a schedule. - `-t/--searchtype` to only do `desktop` or `mobile` searches, `(ex: --searchtype=mobile)` @@ -111,7 +117,8 @@ - Multi-Account Management - Session storing - 2FA Support -- Notifications via [Apprise](https://github.com/caronc/apprise) - no longer limited to Telegram or Discord +- Notifications via [Apprise](https://github.com/caronc/apprise) - no longer limited to + Telegram or Discord - Proxy Support (3.0) - they need to be **high quality** proxies - Logs to CSV file for point tracking @@ -120,8 +127,8 @@ Fork this repo and: * if providing a bugfix, create a pull request into master. -* if providing a new feature, please create a pull request into develop. Extra points if you update - the [CHANGELOG.md](CHANGELOG.md). +* if providing a new feature, please create a pull request into develop. Extra points if you + update the [CHANGELOG.md](CHANGELOG.md). ## To Do List (When time permits or someone makes a PR) From d3bbaa6e3408a777536f9a2047beeea821570530 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Wed, 11 Sep 2024 22:24:38 -0400 Subject: [PATCH 37/82] Get language and country from locale --- src/browser.py | 37 +++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/src/browser.py b/src/browser.py index 4e8cc7ee..52de0531 100644 --- a/src/browser.py +++ b/src/browser.py @@ -1,4 +1,5 @@ import argparse +import locale import logging import os import random @@ -6,10 +7,9 @@ from types import TracebackType from typing import Any, Type -import ipapi +import pycountry import seleniumwire.undetected_chromedriver as webdriver import undetected_chromedriver -from ipapi.exceptions import RateLimited from selenium.webdriver import ChromeOptions from selenium.webdriver.chrome.webdriver import WebDriver @@ -215,20 +215,25 @@ def setupProfiles(self) -> Path: @staticmethod def getCCodeLang(lang: str, geo: str) -> tuple: - if lang is None or geo is None: - try: - # fixme Find better way to get this that doesn't involve ip - nfo = ipapi.location() - except RateLimited: - geo = CONFIG.get("default").get("geolocation", "US") - logging.warning(f"Returning default geolocation {geo}", exc_info=True) - return "en", geo - if isinstance(nfo, dict): - if lang is None: - lang = nfo["languages"].split(",")[0].split("-")[0] - if geo is None: - geo = nfo["country"] - return lang, geo + currentLocale = locale.getlocale() + language, country = currentLocale[0].split("_") + + if not lang: + language = pycountry.languages.get(name=language).alpha_2 + else: + language = lang + + configCountry = CONFIG.get("default").get("location") + if not geo and not configCountry: + country = pycountry.countries.get(name=country).alpha_2 + elif geo: + country = geo + elif configCountry: + country = configCountry + else: + raise AssertionError + + return language, country @staticmethod def getChromeVersion() -> str: From ee86e6d149626d9800d0eaf4b4ada1f8557fd62f Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Wed, 11 Sep 2024 22:40:11 -0400 Subject: [PATCH 38/82] Add pycountry and sort alphabetically --- requirements.txt | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/requirements.txt b/requirements.txt index 859f29d1..d25a0ac5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,15 +1,16 @@ -requests~=2.32.3 -selenium>=4.15.2 # not directly required, pinned by Snyk to avoid a vulnerability +apprise~=1.8.1 +blinker==1.7.0 # prevents issues on newer versions ipapi~=1.0.4 -undetected-chromedriver==3.5.5 -selenium-wire~=5.1.0 numpy>=1.22.2 # not directly required, pinned by Snyk to avoid a vulnerability -setuptools psutil -blinker==1.7.0 # prevents issues on newer versions -apprise~=1.8.1 +pycountry~=24.6.1 +pyotp~=2.9.0 pyyaml~=6.0.2 -urllib3>=2.2.2 # not directly required, pinned by Snyk to avoid a vulnerability requests-oauthlib~=2.0.0 -zipp>=3.19.1 # not directly required, pinned by Snyk to avoid a vulnerability -pyotp~=2.9.0 +requests~=2.32.3 +selenium-wire~=5.1.0 +selenium>=4.15.2 # not directly required, pinned by Snyk to avoid a vulnerability +setuptools +undetected-chromedriver==3.5.5 +urllib3>=2.2.2 # not directly required, pinned by Snyk to avoid a vulnerability +zipp>=3.19.1 # not directly required, pinned by Snyk to avoid a vulnerability \ No newline at end of file From ac792f5c30f8c641b5320d7a3cbe66f2998826b9 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Thu, 12 Sep 2024 00:40:38 -0400 Subject: [PATCH 39/82] Remove ipapi and upgrade apprise --- requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index d25a0ac5..febde036 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,5 @@ -apprise~=1.8.1 +apprise~=1.9.0 blinker==1.7.0 # prevents issues on newer versions -ipapi~=1.0.4 numpy>=1.22.2 # not directly required, pinned by Snyk to avoid a vulnerability psutil pycountry~=24.6.1 From f79be7164ac5e0d5060815e722310aa83d8ac95b Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Sat, 14 Sep 2024 10:22:42 -0400 Subject: [PATCH 40/82] Refactor getting language and country --- src/browser.py | 57 +++++++++++++++++++++++++++++++++----------------- 1 file changed, 38 insertions(+), 19 deletions(-) diff --git a/src/browser.py b/src/browser.py index 52de0531..e5a3dbdf 100644 --- a/src/browser.py +++ b/src/browser.py @@ -1,4 +1,5 @@ import argparse +import contextlib import locale import logging import os @@ -7,9 +8,11 @@ from types import TracebackType from typing import Any, Type +import ipapi import pycountry import seleniumwire.undetected_chromedriver as webdriver import undetected_chromedriver +from ipapi.exceptions import RateLimited from selenium.webdriver import ChromeOptions from selenium.webdriver.chrome.webdriver import WebDriver @@ -34,7 +37,7 @@ def __init__( self.username = account.username self.password = account.password self.totp = account.totp - self.localeLang, self.localeGeo = self.getCCodeLang(args.lang, args.geo) + self.localeLang, self.localeGeo = self.getLanguageCountry(args.lang, args.geo) self.proxy = None if args.proxy: self.proxy = args.proxy @@ -214,24 +217,40 @@ def setupProfiles(self) -> Path: return sessionsDir @staticmethod - def getCCodeLang(lang: str, geo: str) -> tuple: - currentLocale = locale.getlocale() - language, country = currentLocale[0].split("_") - - if not lang: - language = pycountry.languages.get(name=language).alpha_2 - else: - language = lang - - configCountry = CONFIG.get("default").get("location") - if not geo and not configCountry: - country = pycountry.countries.get(name=country).alpha_2 - elif geo: - country = geo - elif configCountry: - country = configCountry - else: - raise AssertionError + def getLanguageCountry(language: str, country: str) -> tuple[str, str]: + + if not country: + country = CONFIG.get("default").get("location") + + if not language or not country: + currentLocale = locale.getlocale() + if not language: + with contextlib.suppress(ValueError): + language = pycountry.languages.get( + name=currentLocale[0].split("_")[0]).alpha_2 + if not country: + with contextlib.suppress(ValueError): + country = pycountry.countries.get( + name=currentLocale[0].split("_")[1]).alpha_2 + + if not language or not country: + try: + ipapiLocation = ipapi.location() + if not language: + language = ipapiLocation["languages"].split(",")[0].split("-")[0] + if not country: + country = ipapiLocation["country"] + except RateLimited: + logging.warning( + "Rate limited, explore alternative ways to specify location above in " + "code. Returning (en, US)", + exc_info=True) + + if not language: + language = "en" + + if not country: + country = "US" return language, country From f6a9feb05a2f1de07377219283509fede79c43da Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Sat, 14 Sep 2024 10:24:59 -0400 Subject: [PATCH 41/82] Refactor warning logging --- src/browser.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/browser.py b/src/browser.py index e5a3dbdf..1ae5417c 100644 --- a/src/browser.py +++ b/src/browser.py @@ -218,7 +218,6 @@ def setupProfiles(self) -> Path: @staticmethod def getLanguageCountry(language: str, country: str) -> tuple[str, str]: - if not country: country = CONFIG.get("default").get("location") @@ -241,16 +240,15 @@ def getLanguageCountry(language: str, country: str) -> tuple[str, str]: if not country: country = ipapiLocation["country"] except RateLimited: - logging.warning( - "Rate limited, explore alternative ways to specify location above in " - "code. Returning (en, US)", - exc_info=True) + logging.warning(exc_info=True) if not language: language = "en" + logging.warning(f"Not able to figure language returning default: {language}") if not country: country = "US" + logging.warning(f"Not able to figure country returning default: {country}") return language, country From 49e4930345c033769adc231723458e5ff342b731 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Sat, 14 Sep 2024 10:26:34 -0400 Subject: [PATCH 42/82] Put ipapi back --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index febde036..61f520c6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ apprise~=1.9.0 blinker==1.7.0 # prevents issues on newer versions numpy>=1.22.2 # not directly required, pinned by Snyk to avoid a vulnerability +ipapi~=1.0.4 psutil pycountry~=24.6.1 pyotp~=2.9.0 From 7f6080a3c2390a3dfa237fe03428e96ec2106eae Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Fri, 27 Sep 2024 11:48:27 -0400 Subject: [PATCH 43/82] Handle all activities in single loop --- config.yaml | 1 + main.py | 10 ++-- src/activities.py | 113 ++++++++++++++++++++++++++++++++++++++++++ src/dailySet.py | 3 ++ src/morePromotions.py | 2 + src/utils.py | 7 +++ 6 files changed, 129 insertions(+), 7 deletions(-) diff --git a/config.yaml b/config.yaml index 1d65792a..349fc663 100644 --- a/config.yaml +++ b/config.yaml @@ -1,6 +1,7 @@ # config.yaml apprise: notify: + # todo Rename to incomplete-activities incomplete-promotions: True # True or False uncaught-exceptions: True # True or False summary: ON_ERROR diff --git a/main.py b/main.py index ccdb141d..41731014 100644 --- a/main.py +++ b/main.py @@ -14,13 +14,12 @@ from src import ( Browser, Login, - MorePromotions, PunchCards, Searches, ReadToEarn, - DailySet, Account, ) +from src.activities import Activities from src.browser import RemainingSearches from src.loggingColoredFormatter import ColoredFormatter from src.utils import Utils, CONFIG @@ -234,12 +233,9 @@ def executeBot(currentAccount: Account, args: argparse.Namespace): Login(desktopBrowser, args).login() startingPoints = utils.getAccountPoints() logging.info( - f"[POINTS] You have {utils.formatNumber(startingPoints)} points on your account" - ) - # todo Combine these classes so main loop isn't duplicated - DailySet(desktopBrowser).completeDailySet() + f"[POINTS] You have {utils.formatNumber(startingPoints)} points on your account") + Activities(desktopBrowser).completeActivities() PunchCards(desktopBrowser).completePunchCards() - MorePromotions(desktopBrowser).completeMorePromotions() # VersusGame(desktopBrowser).completeVersusGame() with Searches(desktopBrowser) as searches: diff --git a/src/activities.py b/src/activities.py index bfaa14b5..056e3eac 100644 --- a/src/activities.py +++ b/src/activities.py @@ -9,6 +9,30 @@ from selenium.webdriver.remote.webelement import WebElement from src.browser import Browser +from src.utils import CONFIG, Utils + +ACTIVITY_TITLE_TO_SEARCH = { + "Search the lyrics of a song": "black sabbath supernaut lyrics", + "Translate anything": "translate pencil sharpener to spanish", + "Let's watch that movie again!": "aliens movie", + "Discover open job roles": "walmart open job roles", + "Plan a quick getaway": "flights nyc to paris", + "You can track your package": "usps tracking", + "Find somewhere new to explore": "directions to new york", + "Too tired to cook tonight?": "Pizza Hut near me", + "Quickly convert your money": "convert 374 usd to yen", + "Learn to cook a new recipe": "how cook pierogi", + "Find places to stay": "hotels rome italy", + "How's the economy?": "sp 500", + "Who won?": "braves score", + "Gaming time": "vampire survivors video game", + "What time is it?": "china time", + "Houses near you": "apartments manhattan", + "Get your shopping done faster": "new iphone", + "Expand your vocabulary": "define polymorphism", + "Stay on top of the elections": "election news latest", + "Prepare for the weather": "weather tomorrow", +} class Activities: @@ -41,8 +65,13 @@ def dashboardPopUpModalCloseCross(self): def openDailySetActivity(self, cardId: int): # Open the Daily Set activity for the given cardId + cardId += 1 element = self.webdriver.find_element(By.XPATH, f'//*[@id="daily-sets"]/mee-card-group[1]/div/mee-card[{cardId}]/div/card-content/mee-rewards-daily-set-item-content/div/a', ) + # element = self.webdriver.find_element(By.CSS_SELECTOR, + # f".ng-scope:nth-child(8) .ng-scope:nth-child({cardId}) .contentContainer:nth-child(3) > .ng-binding:nth-child({cardId})") + # element = self.webdriver.find_element(By.XPATH, + # f"//div[@id=\'daily-sets\']/mee-card-group/div/mee-card[{cardId}]/div/card-content/mee-rewards-daily-set-item-content/div/a/div[{cardId}]/p") self.browser.utils.click(element) self.browser.utils.switchToNewTab(timeToWait=8) @@ -165,3 +194,87 @@ def getAnswerAndCode(self, answerId: str) -> tuple[WebElement, str]: answer, self.browser.utils.getAnswerCode(answerEncodeKey, answerTitle), ) + + def doActivity(self, activity: dict, activities: list[dict]) -> None: + try: + activityTitle = ( + activity["title"].replace("\u200b", "").replace("\xa0", " ") + ) + logging.debug(f"activityTitle={activityTitle}") + if activity["complete"] is True or activity["pointProgressMax"] == 0: + logging.debug("Already done, returning") + return + # Open the activity for the activity + cardId = activities.index(activity) + isDailySet = "daily_set_date" in activity["attributes"] + if isDailySet: + self.openDailySetActivity(cardId) + else: + self.openMorePromotionsActivity(cardId) + self.browser.webdriver.execute_script("window.scrollTo(0, 1080)") + with contextlib.suppress(TimeoutException): + searchbar = self.browser.utils.waitUntilClickable( + By.ID, "sb_form_q" + ) + self.browser.utils.click(searchbar) + # todo These and following are US-English specific, maybe there's a good way to internationalize + if activityTitle in ACTIVITY_TITLE_TO_SEARCH: + searchbar.send_keys(ACTIVITY_TITLE_TO_SEARCH[activityTitle]) + searchbar.submit() + elif "poll" in activityTitle: + logging.info(f"[ACTIVITY] Completing poll of card {cardId}") + # Complete survey for a specific scenario + self.completeSurvey() + elif activity["promotionType"] == "urlreward": + # Complete search for URL reward + self.completeSearch() + elif activity["promotionType"] == "quiz": + # Complete different types of quizzes based on point progress max + if activity["pointProgressMax"] == 10: + self.completeABC() + elif activity["pointProgressMax"] in [30, 40]: + self.completeQuiz() + elif activity["pointProgressMax"] == 50: + self.completeThisOrThat() + else: + # Default to completing search + self.completeSearch() + self.browser.webdriver.execute_script("window.scrollTo(0, 1080)") + time.sleep(random.randint(5, 10)) + except Exception: + logging.error( + f"[ACTIVITY] Error doing {activityTitle}", exc_info=True + ) + self.browser.utils.resetTabs() + time.sleep(2) + + def completeActivities(self): + logging.info("[DAILY SET] " + "Trying to complete the Daily Set...") + dailySetPromotions = self.browser.utils.getDailySetPromotions() + self.browser.utils.goToRewards() + self.dashboardPopUpModalCloseCross() + for activity in dailySetPromotions: + self.doActivity(activity, dailySetPromotions) + logging.info("[DAILY SET] Done") + + logging.info("[MORE PROMOS] " + "Trying to complete More Promotions...") + morePromotions: list[dict] = self.browser.utils.getMorePromotions() + self.browser.utils.goToRewards() + for activity in morePromotions: + self.doActivity(activity, morePromotions) + logging.info("[MORE PROMOS] Done") + + if CONFIG.get("apprise").get("notify").get("incomplete-promotions"): + incompleteActivities: list[tuple[str, str]] = [] + for activity in (self.browser.utils.getDailySetPromotions() + + self.browser.utils.getMorePromotions()): # Have to refresh + if activity["pointProgress"] < activity["pointProgressMax"]: + incompleteActivities.append( + (activity["title"], activity["promotionType"]) + ) + if incompleteActivities: + Utils.sendNotification( + f"We found some incomplete activities for {self.browser.username} to do!", + incompleteActivities, + ) + diff --git a/src/dailySet.py b/src/dailySet.py index 28b8157a..53e55a5f 100644 --- a/src/dailySet.py +++ b/src/dailySet.py @@ -1,10 +1,13 @@ import logging import urllib.parse from datetime import datetime +from warnings import deprecated + from src.browser import Browser from .activities import Activities +@deprecated("Use Activities") class DailySet: def __init__(self, browser: Browser): self.browser = browser diff --git a/src/morePromotions.py b/src/morePromotions.py index 2589b020..9e5a24f6 100644 --- a/src/morePromotions.py +++ b/src/morePromotions.py @@ -2,6 +2,7 @@ import logging import random import time +from warnings import deprecated from selenium.common import TimeoutException from selenium.webdriver.common.by import By @@ -34,6 +35,7 @@ } +@deprecated("Use Activities") # todo Rename MoreActivities? class MorePromotions: def __init__(self, browser: Browser): diff --git a/src/utils.py b/src/utils.py index 50e4a744..c26358ff 100644 --- a/src/utils.py +++ b/src/utils.py @@ -5,6 +5,7 @@ import re import time from argparse import Namespace +from datetime import date from pathlib import Path from types import MappingProxyType from typing import Any @@ -170,6 +171,12 @@ def getDashboardData(self) -> dict: except TimeoutException: self.goToRewards() + def getDailySetPromotions(self) -> list[dict]: + return self.getDashboardData()["dailySetPromotions"][date.today().strftime("%m/%d/%Y")] + + def getMorePromotions(self) -> list[dict]: + return self.getDashboardData()["morePromotions"] + def getBingInfo(self) -> Any: session = self.makeRequestsSession() From 9a3dc6c20c5d9a5ba148fa3896def95a7eadcaff Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Fri, 27 Sep 2024 11:50:11 -0400 Subject: [PATCH 44/82] Refactor config --- config.yaml | 5 ++--- src/activities.py | 2 +- src/morePromotions.py | 2 +- src/utils.py | 4 ++-- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/config.yaml b/config.yaml index 349fc663..4f231545 100644 --- a/config.yaml +++ b/config.yaml @@ -1,9 +1,8 @@ # config.yaml apprise: notify: - # todo Rename to incomplete-activities - incomplete-promotions: True # True or False - uncaught-exceptions: True # True or False + incomplete-activity: True # True or False + uncaught-exception: True # True or False summary: ON_ERROR default: geolocation: US # Replace with your country code https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2 diff --git a/src/activities.py b/src/activities.py index 056e3eac..d641a6e4 100644 --- a/src/activities.py +++ b/src/activities.py @@ -264,7 +264,7 @@ def completeActivities(self): self.doActivity(activity, morePromotions) logging.info("[MORE PROMOS] Done") - if CONFIG.get("apprise").get("notify").get("incomplete-promotions"): + if CONFIG.get("apprise").get("notify").get("incomplete-activity"): incompleteActivities: list[tuple[str, str]] = [] for activity in (self.browser.utils.getDailySetPromotions() + self.browser.utils.getMorePromotions()): # Have to refresh diff --git a/src/morePromotions.py b/src/morePromotions.py index 9e5a24f6..3e095ca6 100644 --- a/src/morePromotions.py +++ b/src/morePromotions.py @@ -100,7 +100,7 @@ def completeMorePromotions(self): # Reset tabs in case of an exception self.browser.utils.resetTabs() continue - if CONFIG.get("apprise").get("notify").get("incomplete-promotions"): + if CONFIG.get("apprise").get("notify").get("incomplete-activity"): incompletePromotions: list[tuple[str, str]] = [] for promotion in self.browser.utils.getDashboardData()[ "morePromotions" diff --git a/src/utils.py b/src/utils.py index c26358ff..7ad26c99 100644 --- a/src/utils.py +++ b/src/utils.py @@ -34,7 +34,7 @@ DEFAULT_CONFIG: MappingProxyType = MappingProxyType( { "apprise": { - "notify": {"incomplete-promotions": True, "uncaught-exceptions": True}, + "notify": {"incomplete-activity": True, "uncaught-exception": True}, "summary": "ALWAYS", }, "default": None, @@ -88,7 +88,7 @@ def loadPrivateConfig() -> MappingProxyType: @staticmethod def sendNotification(title, body, e: Exception = None) -> None: - if Utils.args.disable_apprise or (e and not CONFIG.get("apprise").get("notify").get("uncaught-exceptions")): + if Utils.args.disable_apprise or (e and not CONFIG.get("apprise").get("notify").get("uncaught-exception")): return apprise = Apprise() urls: list[str] = ( From bdd0c8665fda22f5b9b71c7868868686de5d68ec Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Fri, 27 Sep 2024 11:52:05 -0400 Subject: [PATCH 45/82] Clean up and make consistent --- src/activities.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/activities.py b/src/activities.py index d641a6e4..73aa4b27 100644 --- a/src/activities.py +++ b/src/activities.py @@ -68,17 +68,14 @@ def openDailySetActivity(self, cardId: int): cardId += 1 element = self.webdriver.find_element(By.XPATH, f'//*[@id="daily-sets"]/mee-card-group[1]/div/mee-card[{cardId}]/div/card-content/mee-rewards-daily-set-item-content/div/a', ) - # element = self.webdriver.find_element(By.CSS_SELECTOR, - # f".ng-scope:nth-child(8) .ng-scope:nth-child({cardId}) .contentContainer:nth-child(3) > .ng-binding:nth-child({cardId})") - # element = self.webdriver.find_element(By.XPATH, - # f"//div[@id=\'daily-sets\']/mee-card-group/div/mee-card[{cardId}]/div/card-content/mee-rewards-daily-set-item-content/div/a/div[{cardId}]/p") self.browser.utils.click(element) self.browser.utils.switchToNewTab(timeToWait=8) def openMorePromotionsActivity(self, cardId: int): + cardId += 1 # Open the More Promotions activity for the given cardId element = self.webdriver.find_element(By.CSS_SELECTOR, - f"#more-activities > .m-card-group > .ng-scope:nth-child({cardId + 1}) .ds-card-sec") + f"#more-activities > .m-card-group > .ng-scope:nth-child({cardId}) .ds-card-sec") self.browser.utils.click(element) self.browser.utils.switchToNewTab(timeToWait=8) From 38f6c12ab743dd53d0e7214ab1891288ea7d4725 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Fri, 27 Sep 2024 12:12:25 -0400 Subject: [PATCH 46/82] Simplify message dismiss --- src/activities.py | 24 ------------------------ src/utils.py | 39 +++++++++++---------------------------- 2 files changed, 11 insertions(+), 52 deletions(-) diff --git a/src/activities.py b/src/activities.py index 73aa4b27..a310fce7 100644 --- a/src/activities.py +++ b/src/activities.py @@ -4,7 +4,6 @@ import time from selenium.common import TimeoutException -from selenium.common.exceptions import NoSuchElementException, ElementNotInteractableException from selenium.webdriver.common.by import By from selenium.webdriver.remote.webelement import WebElement @@ -40,29 +39,6 @@ def __init__(self, browser: Browser): self.browser = browser self.webdriver = browser.webdriver - def click_element_if_visible(self, element): - try: - if element.is_displayed() and element.is_enabled(): - element.click() - logging.info("Dashboard pop-up registered and closed, needs to be done once on new accounts") - else: - pass - except (ElementNotInteractableException, NoSuchElementException): - pass - - def dashboardPopUpModalCloseCross(self): - try: - - element = self.webdriver.find_element(By.CSS_SELECTOR, ".dashboardPopUpPopUpSelectButton") - self.click_element_if_visible(element) - time.sleep(0.25) - except NoSuchElementException: - return - - - - - def openDailySetActivity(self, cardId: int): # Open the Daily Set activity for the given cardId cardId += 1 diff --git a/src/utils.py b/src/utils.py index 7ad26c99..37c0ed25 100644 --- a/src/utils.py +++ b/src/utils.py @@ -234,7 +234,7 @@ def getGoalTitle(self) -> str: return self.getDashboardData()["userStatus"]["redeemGoal"]["title"] def tryDismissAllMessages(self) -> None: - buttons = [ + byValues = [ (By.ID, "iLandingViewAction"), (By.ID, "iShowSkip"), (By.ID, "iNext"), @@ -242,34 +242,17 @@ def tryDismissAllMessages(self) -> None: (By.ID, "idSIButton9"), (By.ID, "bnp_btn_accept"), (By.ID, "acceptButton"), + (By.CSS_SELECTOR, ".dashboardPopUpPopUpSelectButton") ] - for button in buttons: - try: - elements = self.webdriver.find_elements(by=button[0], value=button[1]) - except ( - NoSuchElementException, - ElementNotInteractableException, - ): # Expected? - logging.debug("", exc_info=True) - continue - for element in elements: - element.click() - self.tryDismissCookieBanner() - self.tryDismissBingCookieBanner() - - def tryDismissCookieBanner(self) -> None: - with contextlib.suppress( - NoSuchElementException, ElementNotInteractableException - ): # Expected - self.webdriver.find_element(By.ID, "cookie-banner").find_element( - By.TAG_NAME, "button" - ).click() - - def tryDismissBingCookieBanner(self) -> None: - with contextlib.suppress( - NoSuchElementException, ElementNotInteractableException - ): # Expected - self.webdriver.find_element(By.ID, "bnp_btn_accept").click() + for byValue in byValues: + dismissButtons = [] + with contextlib.suppress(NoSuchElementException): + dismissButtons = self.webdriver.find_elements(by=byValue[0], value=byValue[1]) + for dismissButton in dismissButtons: + dismissButton.click() + with contextlib.suppress(NoSuchElementException): + self.webdriver.find_element(By.ID, "cookie-banner").find_element(By.TAG_NAME, + "button").click() def switchToNewTab(self, timeToWait: float = 0, closeTab: bool = False) -> None: time.sleep(timeToWait) From fed306858293d97a23091cfe4adaee984580e8ff Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Fri, 27 Sep 2024 12:12:51 -0400 Subject: [PATCH 47/82] Simplify message dismiss --- src/activities.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/activities.py b/src/activities.py index a310fce7..71ffb90d 100644 --- a/src/activities.py +++ b/src/activities.py @@ -225,7 +225,6 @@ def completeActivities(self): logging.info("[DAILY SET] " + "Trying to complete the Daily Set...") dailySetPromotions = self.browser.utils.getDailySetPromotions() self.browser.utils.goToRewards() - self.dashboardPopUpModalCloseCross() for activity in dailySetPromotions: self.doActivity(activity, dailySetPromotions) logging.info("[DAILY SET] Done") From 478557d0657515f32b932186dccfa129f0b11628 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Fri, 27 Sep 2024 12:13:52 -0400 Subject: [PATCH 48/82] Simplify message dismiss --- src/activities.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/activities.py b/src/activities.py index 71ffb90d..e43d4e45 100644 --- a/src/activities.py +++ b/src/activities.py @@ -244,6 +244,7 @@ def completeActivities(self): incompleteActivities.append( (activity["title"], activity["promotionType"]) ) + # todo Allow option to ignore identity protection if incompleteActivities: Utils.sendNotification( f"We found some incomplete activities for {self.browser.username} to do!", From 0e2c966b80e6f08d87446836e982df6b344b34f3 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Fri, 27 Sep 2024 12:15:10 -0400 Subject: [PATCH 49/82] Use typing_extensions --- src/dailySet.py | 3 ++- src/morePromotions.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/dailySet.py b/src/dailySet.py index 53e55a5f..b4f09eea 100644 --- a/src/dailySet.py +++ b/src/dailySet.py @@ -1,7 +1,8 @@ import logging import urllib.parse from datetime import datetime -from warnings import deprecated + +from typing_extensions import deprecated from src.browser import Browser from .activities import Activities diff --git a/src/morePromotions.py b/src/morePromotions.py index 3e095ca6..52cfb875 100644 --- a/src/morePromotions.py +++ b/src/morePromotions.py @@ -2,10 +2,10 @@ import logging import random import time -from warnings import deprecated from selenium.common import TimeoutException from selenium.webdriver.common.by import By +from typing_extensions import deprecated from src.browser import Browser from .activities import Activities From f89f2d266ed80bf87ce59428544b4917dbb960b6 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Fri, 27 Sep 2024 12:53:50 -0400 Subject: [PATCH 50/82] Add option to ignore identity activity --- config.yaml | 7 +++++-- src/activities.py | 31 ++++++++++++++++------------- src/utils.py | 50 ++++++++++++++++++++++++++++++++++------------- 3 files changed, 58 insertions(+), 30 deletions(-) diff --git a/config.yaml b/config.yaml index 4f231545..48cf0aaf 100644 --- a/config.yaml +++ b/config.yaml @@ -1,8 +1,11 @@ # config.yaml apprise: notify: - incomplete-activity: True # True or False - uncaught-exception: True # True or False + incomplete-activity: + enabled: True # True or False + ignore-safeguard-info: True + uncaught-exception: + enabled: True # True or False summary: ON_ERROR default: geolocation: US # Replace with your country code https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2 diff --git a/src/activities.py b/src/activities.py index e43d4e45..301f6e4e 100644 --- a/src/activities.py +++ b/src/activities.py @@ -10,6 +10,7 @@ from src.browser import Browser from src.utils import CONFIG, Utils +# todo These are US-English specific, maybe there's a good way to internationalize ACTIVITY_TITLE_TO_SEARCH = { "Search the lyrics of a song": "black sabbath supernaut lyrics", "Translate anything": "translate pencil sharpener to spanish", @@ -104,10 +105,10 @@ def completeQuiz(self): ) for i in range(numberOfOptions): if ( - self.webdriver.find_element( - By.ID, f"rqAnswerOption{i}" - ).get_attribute("data-option") - == correctOption + self.webdriver.find_element( + By.ID, f"rqAnswerOption{i}" + ).get_attribute("data-option") + == correctOption ): element = self.webdriver.find_element(By.ID, f"rqAnswerOption{i}") self.browser.utils.click(element) @@ -123,7 +124,8 @@ def completeABC(self): ).text[:-1][1:] numberOfQuestions = max(int(s) for s in counter.split() if s.isdigit()) for question in range(numberOfQuestions): - element = self.webdriver.find_element(By.ID, f"questionOptionChoice{question}{random.randint(0, 2)}") + element = self.webdriver.find_element(By.ID, + f"questionOptionChoice{question}{random.randint(0, 2)}") self.browser.utils.click(element) time.sleep(random.randint(10, 15)) element = self.webdriver.find_element(By.ID, f"nextQuestionbtn{question}") @@ -190,7 +192,6 @@ def doActivity(self, activity: dict, activities: list[dict]) -> None: By.ID, "sb_form_q" ) self.browser.utils.click(searchbar) - # todo These and following are US-English specific, maybe there's a good way to internationalize if activityTitle in ACTIVITY_TITLE_TO_SEARCH: searchbar.send_keys(ACTIVITY_TITLE_TO_SEARCH[activityTitle]) searchbar.submit() @@ -236,18 +237,20 @@ def completeActivities(self): self.doActivity(activity, morePromotions) logging.info("[MORE PROMOS] Done") - if CONFIG.get("apprise").get("notify").get("incomplete-activity"): - incompleteActivities: list[tuple[str, str]] = [] + # todo Send one email for all accounts? + if CONFIG.get("apprise").get("notify").get("incomplete-activity").get("enabled"): + incompleteActivities: dict[str, tuple[str, str, str]] = {} for activity in (self.browser.utils.getDailySetPromotions() + self.browser.utils.getMorePromotions()): # Have to refresh if activity["pointProgress"] < activity["pointProgressMax"]: - incompleteActivities.append( - (activity["title"], activity["promotionType"]) - ) - # todo Allow option to ignore identity protection + incompleteActivities[activity["title"]] = ( + activity["promotionType"], activity["pointProgress"], + activity["pointProgressMax"]) + if CONFIG.get("apprise").get("notify").get("incomplete-activity").get( + "ignore-safeguard-info"): + incompleteActivities.pop("Safeguard your family's info", None) if incompleteActivities: Utils.sendNotification( - f"We found some incomplete activities for {self.browser.username} to do!", + f"We found some incomplete activities for {self.browser.username}", incompleteActivities, ) - diff --git a/src/utils.py b/src/utils.py index 37c0ed25..22626b61 100644 --- a/src/utils.py +++ b/src/utils.py @@ -34,19 +34,28 @@ DEFAULT_CONFIG: MappingProxyType = MappingProxyType( { "apprise": { - "notify": {"incomplete-activity": True, "uncaught-exception": True}, - "summary": "ALWAYS", + "notify": { + "incomplete-activity": {"enabled": True, "ignore-safeguard-info": True}, + "uncaught-exception": {"enabled": True}, + }, + "summary": "ON_ERROR", }, - "default": None, + "default": {"geolocation": "US"}, "logging": {"level": "INFO"}, - "retries": {"base_delay_in_seconds": 14.0625, "max": 4, "strategy": "EXPONENTIAL"}, - }) + "retries": { + "base_delay_in_seconds": 14.0625, + "max": 4, + "strategy": "EXPONENTIAL", + }, + } +) DEFAULT_PRIVATE_CONFIG: MappingProxyType = MappingProxyType( { "apprise": { "urls": [], }, - }) + } +) class Utils: @@ -74,7 +83,9 @@ def loadYaml(path: Path) -> dict: return yamlContents @staticmethod - def loadConfig(configFilename="config.yaml", defaultConfig=DEFAULT_CONFIG) -> MappingProxyType: + def loadConfig( + configFilename="config.yaml", defaultConfig=DEFAULT_CONFIG + ) -> MappingProxyType: configFile = Utils.getProjectRoot() / configFilename try: return MappingProxyType(defaultConfig | Utils.loadYaml(configFile)) @@ -88,7 +99,13 @@ def loadPrivateConfig() -> MappingProxyType: @staticmethod def sendNotification(title, body, e: Exception = None) -> None: - if Utils.args.disable_apprise or (e and not CONFIG.get("apprise").get("notify").get("uncaught-exception")): + if Utils.args.disable_apprise or ( + e + and not CONFIG.get("apprise") + .get("notify") + .get("uncaught-exception") + .get("enabled") + ): return apprise = Apprise() urls: list[str] = ( @@ -172,7 +189,9 @@ def getDashboardData(self) -> dict: self.goToRewards() def getDailySetPromotions(self) -> list[dict]: - return self.getDashboardData()["dailySetPromotions"][date.today().strftime("%m/%d/%Y")] + return self.getDashboardData()["dailySetPromotions"][ + date.today().strftime("%m/%d/%Y") + ] def getMorePromotions(self) -> list[dict]: return self.getDashboardData()["morePromotions"] @@ -212,7 +231,7 @@ def makeRequestsSession(session: Session = requests.session()) -> Session: def isLoggedIn(self) -> bool: # return self.getBingInfo()["isRewardsUser"] # todo For some reason doesn't work, but doesn't involve changing url so preferred - if self.getBingInfo()["isRewardsUser"]: # faster, if it works + if self.getBingInfo()["isRewardsUser"]: # faster, if it works return True self.webdriver.get( "https://rewards.bing.com/Signin/" @@ -242,17 +261,20 @@ def tryDismissAllMessages(self) -> None: (By.ID, "idSIButton9"), (By.ID, "bnp_btn_accept"), (By.ID, "acceptButton"), - (By.CSS_SELECTOR, ".dashboardPopUpPopUpSelectButton") + (By.CSS_SELECTOR, ".dashboardPopUpPopUpSelectButton"), ] for byValue in byValues: dismissButtons = [] with contextlib.suppress(NoSuchElementException): - dismissButtons = self.webdriver.find_elements(by=byValue[0], value=byValue[1]) + dismissButtons = self.webdriver.find_elements( + by=byValue[0], value=byValue[1] + ) for dismissButton in dismissButtons: dismissButton.click() with contextlib.suppress(NoSuchElementException): - self.webdriver.find_element(By.ID, "cookie-banner").find_element(By.TAG_NAME, - "button").click() + self.webdriver.find_element(By.ID, "cookie-banner").find_element( + By.TAG_NAME, "button" + ).click() def switchToNewTab(self, timeToWait: float = 0, closeTab: bool = False) -> None: time.sleep(timeToWait) From 494c9df467a11af9a984114ac39c7f0419747f4c Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Fri, 27 Sep 2024 14:13:40 -0400 Subject: [PATCH 51/82] Reformat using Black --- main.py | 14 ++++--- src/activities.py | 72 ++++++++++++++++++++--------------- src/browser.py | 45 +++++++++++++--------- src/dailySet.py | 4 +- src/login.py | 31 +++++++++++---- src/readToEarn.py | 97 ++++++++++++++++++++++++++++------------------- src/searches.py | 9 ++++- 7 files changed, 166 insertions(+), 106 deletions(-) diff --git a/main.py b/main.py index 41731014..82b7c27f 100644 --- a/main.py +++ b/main.py @@ -39,8 +39,11 @@ def main(): earned_points = executeBot(currentAccount, args) except Exception as e1: logging.error("", exc_info=True) - Utils.sendNotification(f"⚠️ Error executing {currentAccount.username}, please check the log", - traceback.format_exc(), e1) + Utils.sendNotification( + f"⚠️ Error executing {currentAccount.username}, please check the log", + traceback.format_exc(), + e1, + ) continue previous_points = previous_points_data.get(currentAccount.username, 0) @@ -233,7 +236,8 @@ def executeBot(currentAccount: Account, args: argparse.Namespace): Login(desktopBrowser, args).login() startingPoints = utils.getAccountPoints() logging.info( - f"[POINTS] You have {utils.formatNumber(startingPoints)} points on your account") + f"[POINTS] You have {utils.formatNumber(startingPoints)} points on your account" + ) Activities(desktopBrowser).completeActivities() PunchCards(desktopBrowser).completePunchCards() # VersusGame(desktopBrowser).completeVersusGame() @@ -273,9 +277,7 @@ def executeBot(currentAccount: Account, args: argparse.Namespace): logging.info( f"[POINTS] You are now at {Utils.formatNumber(accountPoints)} points !" ) - appriseSummary = AppriseSummary[ - CONFIG.get("apprise").get("summary") - ] + appriseSummary = AppriseSummary[CONFIG.get("apprise").get("summary")] if appriseSummary == AppriseSummary.ALWAYS: goalStatus = "" if goalPoints > 0: diff --git a/src/activities.py b/src/activities.py index 301f6e4e..e4e1fb1d 100644 --- a/src/activities.py +++ b/src/activities.py @@ -43,16 +43,20 @@ def __init__(self, browser: Browser): def openDailySetActivity(self, cardId: int): # Open the Daily Set activity for the given cardId cardId += 1 - element = self.webdriver.find_element(By.XPATH, - f'//*[@id="daily-sets"]/mee-card-group[1]/div/mee-card[{cardId}]/div/card-content/mee-rewards-daily-set-item-content/div/a', ) + element = self.webdriver.find_element( + By.XPATH, + f'//*[@id="daily-sets"]/mee-card-group[1]/div/mee-card[{cardId}]/div/card-content/mee-rewards-daily-set-item-content/div/a', + ) self.browser.utils.click(element) self.browser.utils.switchToNewTab(timeToWait=8) def openMorePromotionsActivity(self, cardId: int): cardId += 1 # Open the More Promotions activity for the given cardId - element = self.webdriver.find_element(By.CSS_SELECTOR, - f"#more-activities > .m-card-group > .ng-scope:nth-child({cardId}) .ds-card-sec") + element = self.webdriver.find_element( + By.CSS_SELECTOR, + f"#more-activities > .m-card-group > .ng-scope:nth-child({cardId}) .ds-card-sec", + ) self.browser.utils.click(element) self.browser.utils.switchToNewTab(timeToWait=8) @@ -74,9 +78,7 @@ def completeQuiz(self): startQuiz = self.browser.utils.waitUntilQuizLoads() self.browser.utils.click(startQuiz) # this is bugged on Chrome for some reason - self.browser.utils.waitUntilVisible( - By.ID, "overlayPanel", 5 - ) + self.browser.utils.waitUntilVisible(By.ID, "overlayPanel", 5) currentQuestionNumber: int = self.webdriver.execute_script( "return _w.rewardsQuizRenderInfo.currentQuestionNumber" ) @@ -105,12 +107,14 @@ def completeQuiz(self): ) for i in range(numberOfOptions): if ( - self.webdriver.find_element( - By.ID, f"rqAnswerOption{i}" - ).get_attribute("data-option") - == correctOption + self.webdriver.find_element( + By.ID, f"rqAnswerOption{i}" + ).get_attribute("data-option") + == correctOption ): - element = self.webdriver.find_element(By.ID, f"rqAnswerOption{i}") + element = self.webdriver.find_element( + By.ID, f"rqAnswerOption{i}" + ) self.browser.utils.click(element) self.browser.utils.waitUntilQuestionRefresh() @@ -124,8 +128,9 @@ def completeABC(self): ).text[:-1][1:] numberOfQuestions = max(int(s) for s in counter.split() if s.isdigit()) for question in range(numberOfQuestions): - element = self.webdriver.find_element(By.ID, - f"questionOptionChoice{question}{random.randint(0, 2)}") + element = self.webdriver.find_element( + By.ID, f"questionOptionChoice{question}{random.randint(0, 2)}" + ) self.browser.utils.click(element) time.sleep(random.randint(10, 15)) element = self.webdriver.find_element(By.ID, f"nextQuestionbtn{question}") @@ -172,9 +177,7 @@ def getAnswerAndCode(self, answerId: str) -> tuple[WebElement, str]: def doActivity(self, activity: dict, activities: list[dict]) -> None: try: - activityTitle = ( - activity["title"].replace("\u200b", "").replace("\xa0", " ") - ) + activityTitle = activity["title"].replace("\u200b", "").replace("\xa0", " ") logging.debug(f"activityTitle={activityTitle}") if activity["complete"] is True or activity["pointProgressMax"] == 0: logging.debug("Already done, returning") @@ -188,9 +191,7 @@ def doActivity(self, activity: dict, activities: list[dict]) -> None: self.openMorePromotionsActivity(cardId) self.browser.webdriver.execute_script("window.scrollTo(0, 1080)") with contextlib.suppress(TimeoutException): - searchbar = self.browser.utils.waitUntilClickable( - By.ID, "sb_form_q" - ) + searchbar = self.browser.utils.waitUntilClickable(By.ID, "sb_form_q") self.browser.utils.click(searchbar) if activityTitle in ACTIVITY_TITLE_TO_SEARCH: searchbar.send_keys(ACTIVITY_TITLE_TO_SEARCH[activityTitle]) @@ -216,9 +217,7 @@ def doActivity(self, activity: dict, activities: list[dict]) -> None: self.browser.webdriver.execute_script("window.scrollTo(0, 1080)") time.sleep(random.randint(5, 10)) except Exception: - logging.error( - f"[ACTIVITY] Error doing {activityTitle}", exc_info=True - ) + logging.error(f"[ACTIVITY] Error doing {activityTitle}", exc_info=True) self.browser.utils.resetTabs() time.sleep(2) @@ -238,16 +237,29 @@ def completeActivities(self): logging.info("[MORE PROMOS] Done") # todo Send one email for all accounts? - if CONFIG.get("apprise").get("notify").get("incomplete-activity").get("enabled"): + if ( + CONFIG.get("apprise") + .get("notify") + .get("incomplete-activity") + .get("enabled") + ): incompleteActivities: dict[str, tuple[str, str, str]] = {} - for activity in (self.browser.utils.getDailySetPromotions() + - self.browser.utils.getMorePromotions()): # Have to refresh + for activity in ( + self.browser.utils.getDailySetPromotions() + + self.browser.utils.getMorePromotions() + ): # Have to refresh if activity["pointProgress"] < activity["pointProgressMax"]: incompleteActivities[activity["title"]] = ( - activity["promotionType"], activity["pointProgress"], - activity["pointProgressMax"]) - if CONFIG.get("apprise").get("notify").get("incomplete-activity").get( - "ignore-safeguard-info"): + activity["promotionType"], + activity["pointProgress"], + activity["pointProgressMax"], + ) + if ( + CONFIG.get("apprise") + .get("notify") + .get("incomplete-activity") + .get("ignore-safeguard-info") + ): incompleteActivities.pop("Safeguard your family's info", None) if incompleteActivities: Utils.sendNotification( diff --git a/src/browser.py b/src/browser.py index 1ae5417c..a73ea91f 100644 --- a/src/browser.py +++ b/src/browser.py @@ -15,6 +15,7 @@ from ipapi.exceptions import RateLimited from selenium.webdriver import ChromeOptions from selenium.webdriver.chrome.webdriver import WebDriver +from selenium.webdriver.common.by import By from src import Account, RemainingSearches from src.userAgentGenerator import GenerateUserAgent @@ -62,10 +63,10 @@ def __enter__(self): return self def __exit__( - self, - exc_type: Type[BaseException] | None, - exc_value: BaseException | None, - traceback: TracebackType | None, + self, + exc_type: Type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, ): # Cleanup actions when exiting the browser context logging.debug( @@ -83,7 +84,9 @@ def browserSetup( options.headless = self.headless options.add_argument(f"--lang={self.localeLang}") options.add_argument("--log-level=3") - options.add_argument("--blink-settings=imagesEnabled=false") #If you are having MFA sign in issues comment this line out + options.add_argument( + "--blink-settings=imagesEnabled=false" + ) # If you are having MFA sign in issues comment this line out options.add_argument("--ignore-certificate-errors") options.add_argument("--ignore-certificate-errors-spki-list") options.add_argument("--ignore-ssl-errors") @@ -98,7 +101,7 @@ def browserSetup( options.add_argument("--disable-features=Translate") options.add_argument("--disable-features=PrivacySandboxSettings4") options.add_argument("--disable-http2") - options.add_argument("--disable-search-engine-choice-screen") #153 + options.add_argument("--disable-search-engine-choice-screen") # 153 seleniumwireOptions: dict[str, Any] = {"verify_ssl": False} @@ -110,7 +113,7 @@ def browserSetup( "no_proxy": "localhost,127.0.0.1", } driver = None - + if os.environ.get("DOCKER"): driver = webdriver.Chrome( options=options, @@ -124,10 +127,10 @@ def browserSetup( major = int(version.split(".")[0]) driver = webdriver.Chrome( - options=options, - seleniumwire_options=seleniumwireOptions, - user_data_dir=self.userDataDir.as_posix(), - version_main=major, + options=options, + seleniumwire_options=seleniumwireOptions, + user_data_dir=self.userDataDir.as_posix(), + version_main=major, ) seleniumLogger = logging.getLogger("seleniumwire") @@ -226,11 +229,13 @@ def getLanguageCountry(language: str, country: str) -> tuple[str, str]: if not language: with contextlib.suppress(ValueError): language = pycountry.languages.get( - name=currentLocale[0].split("_")[0]).alpha_2 + name=currentLocale[0].split("_")[0] + ).alpha_2 if not country: with contextlib.suppress(ValueError): country = pycountry.countries.get( - name=currentLocale[0].split("_")[1]).alpha_2 + name=currentLocale[0].split("_")[1] + ).alpha_2 if not language or not country: try: @@ -244,7 +249,9 @@ def getLanguageCountry(language: str, country: str) -> tuple[str, str]: if not language: language = "en" - logging.warning(f"Not able to figure language returning default: {language}") + logging.warning( + f"Not able to figure language returning default: {language}" + ) if not country: country = "US" @@ -267,7 +274,7 @@ def getChromeVersion() -> str: return version def getRemainingSearches( - self, desktopAndMobile: bool = False + self, desktopAndMobile: bool = False ) -> RemainingSearches | int: bingInfo = self.utils.getBingInfo() searchPoints = 1 @@ -284,11 +291,15 @@ def getRemainingSearches( pcPointsRemaining = pcSearch["pointProgressMax"] - pcSearch["pointProgress"] assert pcPointsRemaining % searchPoints == 0 remainingDesktopSearches: int = int(pcPointsRemaining / searchPoints) - mobilePointsRemaining = mobileSearch["pointProgressMax"] - mobileSearch["pointProgress"] + mobilePointsRemaining = ( + mobileSearch["pointProgressMax"] - mobileSearch["pointProgress"] + ) assert mobilePointsRemaining % searchPoints == 0 remainingMobileSearches: int = int(mobilePointsRemaining / searchPoints) if desktopAndMobile: - return RemainingSearches(desktop=remainingDesktopSearches, mobile=remainingMobileSearches) + return RemainingSearches( + desktop=remainingDesktopSearches, mobile=remainingMobileSearches + ) if self.mobile: return remainingMobileSearches return remainingDesktopSearches diff --git a/src/dailySet.py b/src/dailySet.py index b4f09eea..187f3213 100644 --- a/src/dailySet.py +++ b/src/dailySet.py @@ -43,9 +43,7 @@ def completeDailySet(self): ) # Complete This or That for a specific point progress max self.activities.completeThisOrThat() - elif ( - activity["pointProgressMax"] in [40, 30] - ): + elif activity["pointProgressMax"] in [40, 30]: logging.info(f"[DAILY SET] Completing quiz of card {cardId}") # Complete quiz for specific point progress max self.activities.completeQuiz() diff --git a/src/login.py b/src/login.py index f28af567..01978342 100644 --- a/src/login.py +++ b/src/login.py @@ -2,11 +2,16 @@ import contextlib import logging from argparse import Namespace + from pyotp import TOTP from selenium.common import TimeoutException +from selenium.common.exceptions import ( + ElementNotInteractableException, + NoSuchElementException, +) from selenium.webdriver.common.by import By from undetected_chromedriver import Chrome -from selenium.common.exceptions import ElementNotInteractableException, NoSuchElementException + from src.browser import Browser @@ -23,7 +28,9 @@ def __init__(self, browser: Browser, args: argparse.Namespace): def check_locked_user(self): try: - element = self.webdriver.find_element(By.XPATH, "//div[@id='serviceAbuseLandingTitle']") + element = self.webdriver.find_element( + By.XPATH, "//div[@id='serviceAbuseLandingTitle']" + ) self.locked(element) except NoSuchElementException: return @@ -120,7 +127,9 @@ def execute_login(self) -> None: if isDeviceAuthEnabled: # Device-based authentication not supported - raise Exception("Device authentication not supported. Please use TOTP or disable 2FA.") + raise Exception( + "Device authentication not supported. Please use TOTP or disable 2FA." + ) # Device auth, have user confirm code on phone codeField = self.utils.waitUntilVisible( @@ -140,10 +149,14 @@ def execute_login(self) -> None: # TOTP token provided logging.info("[LOGIN] Entering OTP...") otp = TOTP(self.browser.totp.replace(" ", "")).now() - otpField = self.utils.waitUntilClickable(By.ID, "idTxtBx_SAOTCC_OTC") + otpField = self.utils.waitUntilClickable( + By.ID, "idTxtBx_SAOTCC_OTC" + ) otpField.send_keys(otp) assert otpField.get_attribute("value") == otp - self.utils.waitUntilClickable(By.ID, "idSubmit_SAOTCC_Continue").click() + self.utils.waitUntilClickable( + By.ID, "idSubmit_SAOTCC_Continue" + ).click() else: # TOTP token not provided, manual intervention required assert self.args.visible, ( @@ -153,9 +166,9 @@ def execute_login(self) -> None: ) print( "[LOGIN] 2FA detected, handle prompts and press enter when on" - " keep me signed in page.") + " keep me signed in page." + ) input() - self.check_locked_user() self.check_banned_user() @@ -179,4 +192,6 @@ def execute_login(self) -> None: ) input() - self.utils.waitUntilVisible(By.CSS_SELECTOR, 'html[data-role-name="RewardsPortal"]') + self.utils.waitUntilVisible( + By.CSS_SELECTOR, 'html[data-role-name="RewardsPortal"]' + ) diff --git a/src/readToEarn.py b/src/readToEarn.py index 408382ca..cc70645c 100644 --- a/src/readToEarn.py +++ b/src/readToEarn.py @@ -9,91 +9,108 @@ from .activities import Activities from .utils import Utils -client_id = '0000000040170455' -authorization_base_url = 'https://login.live.com/oauth20_authorize.srf' -token_url = 'https://login.microsoftonline.com/consumers/oauth2/v2.0/token' -redirect_uri = ' https://login.live.com/oauth20_desktop.srf' -scope = [ "service::prod.rewardsplatform.microsoft.com::MBI_SSL"] +# todo Use constant naming style +client_id = "0000000040170455" +authorization_base_url = "https://login.live.com/oauth20_authorize.srf" +token_url = "https://login.microsoftonline.com/consumers/oauth2/v2.0/token" +redirect_uri = " https://login.live.com/oauth20_desktop.srf" +scope = ["service::prod.rewardsplatform.microsoft.com::MBI_SSL"] + class ReadToEarn: def __init__(self, browser: Browser): self.browser = browser self.webdriver = browser.webdriver self.activities = Activities(browser) - + def completeReadToEarn(self): - + logging.info("[READ TO EARN] " + "Trying to complete Read to Earn...") accountName = self.browser.username - + # Should Really Cache Token and load it in. # To Save token - #with open('token.pickle', 'wb') as f: + # with open('token.pickle', 'wb') as f: # pickle.dump(token, f) # To Load token - #with open('token.pickle', 'rb') as f: + # with open('token.pickle', 'rb') as f: # token = pickle.load(f) - #mobileApp = OAuth2Session(client_id, scope=scope, token=token) - + # mobileApp = OAuth2Session(client_id, scope=scope, token=token) + # Use Webdriver to get OAuth2 Token # This works, since you already logged into Bing, so no user interaction needed - mobileApp = Utils.makeRequestsSession(OAuth2Session(client_id, scope=scope, redirect_uri=redirect_uri)) - authorization_url, state = mobileApp.authorization_url(authorization_base_url, access_type="offline_access", login_hint=accountName) - + mobileApp = Utils.makeRequestsSession( + OAuth2Session(client_id, scope=scope, redirect_uri=redirect_uri) + ) + authorization_url, state = mobileApp.authorization_url( + authorization_base_url, access_type="offline_access", login_hint=accountName + ) + # Get Referer URL from webdriver self.webdriver.get(authorization_url) while True: logging.info("[READ TO EARN] Waiting for Login") - if self.webdriver.current_url[:48] == "https://login.live.com/oauth20_desktop.srf?code=": + if ( + self.webdriver.current_url[:48] + == "https://login.live.com/oauth20_desktop.srf?code=" + ): redirect_response = self.webdriver.current_url break time.sleep(1) - + logging.info("[READ TO EARN] Logged-in successfully !") # Use returned URL to create a token - token = mobileApp.fetch_token(token_url, authorization_response=redirect_response,include_client_id=True) - + token = mobileApp.fetch_token( + token_url, authorization_response=redirect_response, include_client_id=True + ) + # Do Daily Check in json_data = { - 'amount': 1, - 'country': self.browser.localeGeo.lower(), - 'id': 1, - 'type': 101, - 'attributes': { - 'offerid': 'Gamification_Sapphire_DailyCheckIn', + "amount": 1, + "country": self.browser.localeGeo.lower(), + "id": 1, + "type": 101, + "attributes": { + "offerid": "Gamification_Sapphire_DailyCheckIn", }, } - json_data['id'] = secrets.token_hex(64) + json_data["id"] = secrets.token_hex(64) logging.info("[READ TO EARN] Daily App Check In") - r = mobileApp.post("https://prod.rewardsplatform.microsoft.com/dapi/me/activities",json=json_data) + r = mobileApp.post( + "https://prod.rewardsplatform.microsoft.com/dapi/me/activities", + json=json_data, + ) balance = r.json().get("response").get("balance") time.sleep(random.randint(10, 20)) # json data to confirm an article is read json_data = { - 'amount': 1, - 'country': self.browser.localeGeo.lower(), - 'id': 1, - 'type': 101, - 'attributes': { - 'offerid': 'ENUS_readarticle3_30points', - }, - } + "amount": 1, + "country": self.browser.localeGeo.lower(), + "id": 1, + "type": 101, + "attributes": { + "offerid": "ENUS_readarticle3_30points", + }, + } # 10 is the most articles you can read. Sleep time is a guess, not tuned for i in range(10): # Replace ID with a random value so get credit for a new article - json_data['id'] = secrets.token_hex(64) - r = mobileApp.post("https://prod.rewardsplatform.microsoft.com/dapi/me/activities",json=json_data) + json_data["id"] = secrets.token_hex(64) + r = mobileApp.post( + "https://prod.rewardsplatform.microsoft.com/dapi/me/activities", + json=json_data, + ) newbalance = r.json().get("response").get("balance") if newbalance == balance: logging.info("[READ TO EARN] Read All Available Articles !") break else: - logging.info("[READ TO EARN] Read Article " + str(i+1)) + logging.info("[READ TO EARN] Read Article " + str(i + 1)) balance = newbalance time.sleep(random.randint(10, 20)) - - logging.info("[READ TO EARN] Completed the Read to Earn successfully !") + + logging.info("[READ TO EARN] Completed the Read to Earn successfully !") diff --git a/src/searches.py b/src/searches.py index 4c120b00..9d91b0e8 100644 --- a/src/searches.py +++ b/src/searches.py @@ -111,8 +111,13 @@ def bingSearches(self) -> None: desktopAndMobile=True ) logging.info(f"[BING] Remaining searches={desktopAndMobileRemaining}") - if (self.browser.browserType == "desktop" and desktopAndMobileRemaining.desktop == 0) \ - or (self.browser.browserType == "mobile" and desktopAndMobileRemaining.mobile == 0): + if ( + self.browser.browserType == "desktop" + and desktopAndMobileRemaining.desktop == 0 + ) or ( + self.browser.browserType == "mobile" + and desktopAndMobileRemaining.mobile == 0 + ): break if desktopAndMobileRemaining.getTotal() > len(self.googleTrendsShelf): From 44b01162323017071c91dd939e14921c7af5596c Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Wed, 23 Oct 2024 00:08:24 -0400 Subject: [PATCH 52/82] Alphabetize --- src/activities.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/src/activities.py b/src/activities.py index e4e1fb1d..d326a1c5 100644 --- a/src/activities.py +++ b/src/activities.py @@ -12,26 +12,26 @@ # todo These are US-English specific, maybe there's a good way to internationalize ACTIVITY_TITLE_TO_SEARCH = { - "Search the lyrics of a song": "black sabbath supernaut lyrics", - "Translate anything": "translate pencil sharpener to spanish", - "Let's watch that movie again!": "aliens movie", "Discover open job roles": "walmart open job roles", - "Plan a quick getaway": "flights nyc to paris", - "You can track your package": "usps tracking", - "Find somewhere new to explore": "directions to new york", - "Too tired to cook tonight?": "Pizza Hut near me", - "Quickly convert your money": "convert 374 usd to yen", - "Learn to cook a new recipe": "how cook pierogi", + "Expand your vocabulary": "define demure", "Find places to stay": "hotels rome italy", - "How's the economy?": "sp 500", - "Who won?": "braves score", + "Find somewhere new to explore": "directions to new york", "Gaming time": "vampire survivors video game", - "What time is it?": "china time", - "Houses near you": "apartments manhattan", "Get your shopping done faster": "new iphone", - "Expand your vocabulary": "define polymorphism", - "Stay on top of the elections": "election news latest", + "Houses near you": "apartments manhattan", + "How's the economy?": "sp 500", + "Learn to cook a new recipe": "how cook pierogi", + "Let's watch that movie again!": "aliens movie", + "Plan a quick getaway": "flights nyc to paris", "Prepare for the weather": "weather tomorrow", + "Quickly convert your money": "convert 374 usd to yen", + "Search the lyrics of a song": "black sabbath supernaut lyrics", + "Stay on top of the elections": "election news latest", + "Too tired to cook tonight?": "Pizza Hut near me", + "Translate anything": "translate pencil sharpener to spanish", + "What time is it?": "china time", + "Who won?": "braves score", + "You can track your package": "usps tracking", } From bc30d82017d3720f67f650274df52b22736b8e00 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Wed, 23 Oct 2024 00:09:04 -0400 Subject: [PATCH 53/82] Clean up activity title for emails --- src/activities.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/activities.py b/src/activities.py index d326a1c5..f6f1d655 100644 --- a/src/activities.py +++ b/src/activities.py @@ -177,7 +177,7 @@ def getAnswerAndCode(self, answerId: str) -> tuple[WebElement, str]: def doActivity(self, activity: dict, activities: list[dict]) -> None: try: - activityTitle = activity["title"].replace("\u200b", "").replace("\xa0", " ") + activityTitle = cleanupActivityTitle(activity["title"]) logging.debug(f"activityTitle={activityTitle}") if activity["complete"] is True or activity["pointProgressMax"] == 0: logging.debug("Already done, returning") @@ -249,7 +249,7 @@ def completeActivities(self): + self.browser.utils.getMorePromotions() ): # Have to refresh if activity["pointProgress"] < activity["pointProgressMax"]: - incompleteActivities[activity["title"]] = ( + incompleteActivities[cleanupActivityTitle(activity["title"])] = ( activity["promotionType"], activity["pointProgress"], activity["pointProgressMax"], @@ -266,3 +266,7 @@ def completeActivities(self): f"We found some incomplete activities for {self.browser.username}", incompleteActivities, ) + + +def cleanupActivityTitle(activityTitle: str) -> str: + return activityTitle.replace("\u200b", "").replace("\xa0", " ") From 92d272088d5e5018dca0600f3ac155ec322c147c Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Wed, 23 Oct 2024 00:09:54 -0400 Subject: [PATCH 54/82] Fix issue where page load timed out --- src/browser.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/browser.py b/src/browser.py index a73ea91f..eb4b6a40 100644 --- a/src/browser.py +++ b/src/browser.py @@ -102,6 +102,7 @@ def browserSetup( options.add_argument("--disable-features=PrivacySandboxSettings4") options.add_argument("--disable-http2") options.add_argument("--disable-search-engine-choice-screen") # 153 + options.page_load_strategy = 'eager' seleniumwireOptions: dict[str, Any] = {"verify_ssl": False} From 4572c600ff35af56c48a2dbb59d0a9eb3c555bd3 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Wed, 23 Oct 2024 00:10:24 -0400 Subject: [PATCH 55/82] Refactor imports and use method name only --- src/activities.py | 33 ++++++++++++++++------------- src/readToEarn.py | 4 ++-- src/searches.py | 23 ++++++++++---------- src/userAgentGenerator.py | 4 ++-- src/utils.py | 44 ++++++++++++++++++++------------------- 5 files changed, 57 insertions(+), 51 deletions(-) diff --git a/src/activities.py b/src/activities.py index f6f1d655..6d1a7d12 100644 --- a/src/activities.py +++ b/src/activities.py @@ -1,13 +1,14 @@ import contextlib import logging -import random -import time +from random import randint +from time import sleep from selenium.common import TimeoutException from selenium.webdriver.common.by import By from selenium.webdriver.remote.webelement import WebElement from src.browser import Browser +from src.constants import REWARDS_URL from src.utils import CONFIG, Utils # todo These are US-English specific, maybe there's a good way to internationalize @@ -62,14 +63,15 @@ def openMorePromotionsActivity(self, cardId: int): def completeSearch(self): # Simulate completing a search activity - time.sleep(random.randint(10, 15)) + sleep(randint(20, 30)) + # WebDriverWait(self.webdriver, 30).until() self.browser.utils.closeCurrentTab() def completeSurvey(self): # Simulate completing a survey activity # noinspection SpellCheckingInspection - self.webdriver.find_element(By.ID, f"btoption{random.randint(0, 1)}").click() - time.sleep(random.randint(10, 15)) + self.webdriver.find_element(By.ID, f"btoption{randint(0, 1)}").click() + sleep(randint(10, 15)) self.browser.utils.closeCurrentTab() def completeQuiz(self): @@ -129,14 +131,14 @@ def completeABC(self): numberOfQuestions = max(int(s) for s in counter.split() if s.isdigit()) for question in range(numberOfQuestions): element = self.webdriver.find_element( - By.ID, f"questionOptionChoice{question}{random.randint(0, 2)}" + By.ID, f"questionOptionChoice{question}{randint(0, 2)}" ) self.browser.utils.click(element) - time.sleep(random.randint(10, 15)) + sleep(randint(10, 15)) element = self.webdriver.find_element(By.ID, f"nextQuestionbtn{question}") self.browser.utils.click(element) - time.sleep(random.randint(10, 15)) - time.sleep(random.randint(1, 7)) + sleep(randint(10, 15)) + sleep(randint(1, 7)) self.browser.utils.closeCurrentTab() def completeThisOrThat(self): @@ -146,7 +148,7 @@ def completeThisOrThat(self): self.browser.utils.waitUntilVisible( By.XPATH, '//*[@id="currentQuestionContainer"]/div/div[1]', 10 ) - time.sleep(random.randint(10, 15)) + sleep(randint(10, 15)) for _ in range(10): correctAnswerCode = self.webdriver.execute_script( "return _w.rewardsQuizRenderInfo.correctAnswer" @@ -160,9 +162,9 @@ def completeThisOrThat(self): answerToClick = answer2 self.browser.utils.click(answerToClick) - time.sleep(random.randint(10, 15)) + sleep(randint(10, 15)) - time.sleep(random.randint(10, 15)) + sleep(randint(10, 15)) self.browser.utils.closeCurrentTab() def getAnswerAndCode(self, answerId: str) -> tuple[WebElement, str]: @@ -190,6 +192,7 @@ def doActivity(self, activity: dict, activities: list[dict]) -> None: else: self.openMorePromotionsActivity(cardId) self.browser.webdriver.execute_script("window.scrollTo(0, 1080)") + sleep(1) with contextlib.suppress(TimeoutException): searchbar = self.browser.utils.waitUntilClickable(By.ID, "sb_form_q") self.browser.utils.click(searchbar) @@ -215,11 +218,11 @@ def doActivity(self, activity: dict, activities: list[dict]) -> None: # Default to completing search self.completeSearch() self.browser.webdriver.execute_script("window.scrollTo(0, 1080)") - time.sleep(random.randint(5, 10)) + # sleep(randint(5, 10)) except Exception: logging.error(f"[ACTIVITY] Error doing {activityTitle}", exc_info=True) self.browser.utils.resetTabs() - time.sleep(2) + # sleep(randint(900, 1200)) def completeActivities(self): logging.info("[DAILY SET] " + "Trying to complete the Daily Set...") @@ -264,7 +267,7 @@ def completeActivities(self): if incompleteActivities: Utils.sendNotification( f"We found some incomplete activities for {self.browser.username}", - incompleteActivities, + str(incompleteActivities) + "\n" + REWARDS_URL, ) diff --git a/src/readToEarn.py b/src/readToEarn.py index cc70645c..67979f6e 100644 --- a/src/readToEarn.py +++ b/src/readToEarn.py @@ -7,7 +7,7 @@ from src.browser import Browser from .activities import Activities -from .utils import Utils +from .utils import makeRequestsSession # todo Use constant naming style client_id = "0000000040170455" @@ -41,7 +41,7 @@ def completeReadToEarn(self): # Use Webdriver to get OAuth2 Token # This works, since you already logged into Bing, so no user interaction needed - mobileApp = Utils.makeRequestsSession( + mobileApp = makeRequestsSession( OAuth2Session(client_id, scope=scope, redirect_uri=redirect_uri) ) authorization_url, state = mobileApp.authorization_url( diff --git a/src/searches.py b/src/searches.py index 9d91b0e8..410d93a2 100644 --- a/src/searches.py +++ b/src/searches.py @@ -1,19 +1,19 @@ import dbm.dumb import json import logging -import random import shelve -import time from datetime import date, timedelta from enum import Enum, auto from itertools import cycle +from random import random, randint, shuffle +from time import sleep from typing import Final import requests from selenium.webdriver.common.by import By from src.browser import Browser -from src.utils import Utils, CONFIG +from src.utils import Utils, CONFIG, makeRequestsSession class RetriesStrategy(Enum): @@ -60,7 +60,7 @@ def getGoogleTrends(self, wordsCount: int) -> list[str]: # Function to retrieve Google Trends search terms searchTerms: list[str] = [] i = 0 - session = Utils.makeRequestsSession() + session = makeRequestsSession() while len(searchTerms) < wordsCount: i += 1 # Fetching daily trends from Google Trends API @@ -87,7 +87,7 @@ def getGoogleTrends(self, wordsCount: int) -> list[str]: def getRelatedTerms(self, term: str) -> list[str]: # Function to retrieve related terms from Bing API relatedTerms: list[str] = ( - Utils.makeRequestsSession() + makeRequestsSession() .get( f"https://api.bing.com/osjson.aspx?query={term}", headers={"User-agent": self.browser.userAgent}, @@ -126,7 +126,7 @@ def bingSearches(self) -> None: f"google_trends before load = {list(self.googleTrendsShelf.items())}" ) trends = self.getGoogleTrends(desktopAndMobileRemaining.getTotal()) - random.shuffle(trends) + shuffle(trends) for trend in trends: self.googleTrendsShelf[trend] = None logging.debug( @@ -135,7 +135,7 @@ def bingSearches(self) -> None: self.bingSearch() del self.googleTrendsShelf[list(self.googleTrendsShelf.keys())[0]] - time.sleep(random.randint(10, 15)) + sleep(randint(10, 15)) logging.info( f"[BING] Finished {self.browser.browserType.capitalize()} Edge Bing searches !" @@ -162,12 +162,12 @@ def bingSearch(self) -> None: sleepTime = baseDelay else: raise AssertionError - sleepTime += baseDelay * random.random() # Add jitter + sleepTime += baseDelay * random() # Add jitter logging.debug( f"[BING] Search attempt not counted {i}/{Searches.maxRetries}, sleeping {sleepTime}" f" seconds..." ) - time.sleep(sleepTime) + sleep(sleepTime) searchbar = self.browser.utils.waitUntilClickable( By.ID, "sb_form_q", timeToWait=40 @@ -175,13 +175,14 @@ def bingSearch(self) -> None: searchbar.clear() term = next(termsCycle) logging.debug(f"term={term}") - time.sleep(1) + sleep(1) searchbar.send_keys(term) - time.sleep(1) + sleep(1) searchbar.submit() pointsAfter = self.browser.utils.getAccountPoints() if pointsBefore < pointsAfter: + # sleep(randint(900, 1200)) return # todo diff --git a/src/userAgentGenerator.py b/src/userAgentGenerator.py index 6cbbaee9..854de78e 100644 --- a/src/userAgentGenerator.py +++ b/src/userAgentGenerator.py @@ -4,7 +4,7 @@ import requests from requests import HTTPError, Response -from src.utils import Utils +from src.utils import makeRequestsSession class GenerateUserAgent: @@ -180,7 +180,7 @@ def getChromeVersion(self) -> str: @staticmethod def getWebdriverPage(url: str) -> Response: - response = Utils.makeRequestsSession().get(url) + response = makeRequestsSession().get(url) if response.status_code != requests.codes.ok: # pylint: disable=no-member raise HTTPError( f"Failed to get webdriver page {url}. " diff --git a/src/utils.py b/src/utils.py index 22626b61..c8b0c27e 100644 --- a/src/utils.py +++ b/src/utils.py @@ -197,7 +197,7 @@ def getMorePromotions(self) -> list[dict]: return self.getDashboardData()["morePromotions"] def getBingInfo(self) -> Any: - session = self.makeRequestsSession() + session = makeRequestsSession() for cookie in self.webdriver.get_cookies(): session.cookies.set(cookie["name"], cookie["value"]) @@ -209,26 +209,6 @@ def getBingInfo(self) -> Any: # todo Add fallback to src.utils.Utils.getDashboardData (slower but more reliable) return response.json() - @staticmethod - def makeRequestsSession(session: Session = requests.session()) -> Session: - retry = Retry( - total=5, - backoff_factor=1, - status_forcelist=[ - 500, - 502, - 503, - 504, - ], # todo Use global retries from config - ) - session.mount( - "https://", HTTPAdapter(max_retries=retry) - ) # See https://stackoverflow.com/a/35504626/4164390 to finetune - session.mount( - "http://", HTTPAdapter(max_retries=retry) - ) # See https://stackoverflow.com/a/35504626/4164390 to finetune - return session - def isLoggedIn(self) -> bool: # return self.getBingInfo()["isRewardsUser"] # todo For some reason doesn't work, but doesn't involve changing url so preferred if self.getBingInfo()["isRewardsUser"]: # faster, if it works @@ -310,11 +290,33 @@ def saveBrowserConfig(sessionPath: Path, config: dict) -> None: def click(self, element: WebElement) -> None: try: + WebDriverWait(self.webdriver, 10).until(expected_conditions.element_to_be_clickable(element)) element.click() except (ElementClickInterceptedException, ElementNotInteractableException): self.tryDismissAllMessages() + WebDriverWait(self.webdriver, 10).until(expected_conditions.element_to_be_clickable(element)) element.click() +def makeRequestsSession(session: Session = requests.session()) -> Session: + retry = Retry( + total=5, + backoff_factor=1, + status_forcelist=[ + 500, + 502, + 503, + 504, + ], # todo Use global retries from config + ) + session.mount( + "https://", HTTPAdapter(max_retries=retry) + ) # See https://stackoverflow.com/a/35504626/4164390 to finetune + session.mount( + "http://", HTTPAdapter(max_retries=retry) + ) # See https://stackoverflow.com/a/35504626/4164390 to finetune + return session + + CONFIG = Utils.loadConfig() PRIVATE_CONFIG = Utils.loadPrivateConfig() From 9291a6e46e2d1fe1a5e7f9b0a7a6106b1c9b3f4e Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Wed, 23 Oct 2024 13:16:41 -0400 Subject: [PATCH 56/82] Remove staticmethod in favor of function https://stackoverflow.com/a/11788267/4164390 --- main.py | 40 +++++------ src/activities.py | 6 +- src/browser.py | 10 +-- src/morePromotions.py | 4 +- src/searches.py | 4 +- src/utils.py | 159 +++++++++++++++++++++--------------------- test/test_utils.py | 4 +- 7 files changed, 111 insertions(+), 116 deletions(-) diff --git a/main.py b/main.py index 82b7c27f..60053a0a 100644 --- a/main.py +++ b/main.py @@ -22,7 +22,7 @@ from src.activities import Activities from src.browser import RemainingSearches from src.loggingColoredFormatter import ColoredFormatter -from src.utils import Utils, CONFIG +from src.utils import Utils, CONFIG, sendNotification, getProjectRoot, formatNumber def main(): @@ -39,7 +39,7 @@ def main(): earned_points = executeBot(currentAccount, args) except Exception as e1: logging.error("", exc_info=True) - Utils.sendNotification( + sendNotification( f"⚠️ Error executing {currentAccount.username}, please check the log", traceback.format_exc(), e1, @@ -66,7 +66,7 @@ def main(): def log_daily_points_to_csv(earned_points, points_difference): - logs_directory = Utils.getProjectRoot() / "logs" + logs_directory = getProjectRoot() / "logs" csv_filename = logs_directory / "points_data.csv" # Create a new row with the date, daily points, and points difference @@ -94,7 +94,7 @@ def setupLogging(): terminalHandler = logging.StreamHandler(sys.stdout) terminalHandler.setFormatter(ColoredFormatter(_format)) - logs_directory = Utils.getProjectRoot() / "logs" + logs_directory = getProjectRoot() / "logs" logs_directory.mkdir(parents=True, exist_ok=True) # so only our code is logged if level=logging.DEBUG or finer @@ -175,7 +175,7 @@ def validEmail(email: str) -> bool: pattern = r"^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$" return bool(re.match(pattern, email)) - accountPath = Utils.getProjectRoot() / "accounts.json" + accountPath = getProjectRoot() / "accounts.json" if not accountPath.exists(): accountPath.write_text( json.dumps( @@ -236,7 +236,7 @@ def executeBot(currentAccount: Account, args: argparse.Namespace): Login(desktopBrowser, args).login() startingPoints = utils.getAccountPoints() logging.info( - f"[POINTS] You have {utils.formatNumber(startingPoints)} points on your account" + f"[POINTS] You have {formatNumber(startingPoints)} points on your account" ) Activities(desktopBrowser).completeActivities() PunchCards(desktopBrowser).completePunchCards() @@ -272,38 +272,36 @@ def executeBot(currentAccount: Account, args: argparse.Namespace): accountPoints = utils.getAccountPoints() logging.info( - f"[POINTS] You have earned {Utils.formatNumber(accountPoints - startingPoints)} points this run !" - ) - logging.info( - f"[POINTS] You are now at {Utils.formatNumber(accountPoints)} points !" + f"[POINTS] You have earned {formatNumber(accountPoints - startingPoints)} points this run !" ) + logging.info(f"[POINTS] You are now at {formatNumber(accountPoints)} points !") appriseSummary = AppriseSummary[CONFIG.get("apprise").get("summary")] if appriseSummary == AppriseSummary.ALWAYS: goalStatus = "" if goalPoints > 0: logging.info( - f"[POINTS] You are now at {(Utils.formatNumber((accountPoints / goalPoints) * 100))}% of your " + f"[POINTS] You are now at {(formatNumber((accountPoints / goalPoints) * 100))}% of your " f"goal ({goalTitle}) !" ) goalStatus = ( - f"🎯 Goal reached: {(Utils.formatNumber((accountPoints / goalPoints) * 100))}%" + f"🎯 Goal reached: {(formatNumber((accountPoints / goalPoints) * 100))}%" f" ({goalTitle})" ) - Utils.sendNotification( + sendNotification( "Daily Points Update", "\n".join( [ f"👤 Account: {currentAccount.username}", - f"⭐️ Points earned today: {Utils.formatNumber(accountPoints - startingPoints)}", - f"💰 Total points: {Utils.formatNumber(accountPoints)}", + f"⭐️ Points earned today: {formatNumber(accountPoints - startingPoints)}", + f"💰 Total points: {formatNumber(accountPoints)}", goalStatus, ] ), ) elif appriseSummary == AppriseSummary.ON_ERROR: if remainingSearches.getTotal() > 0: - Utils.sendNotification( + sendNotification( "Error: remaining searches", f"account username: {currentAccount.username}, {remainingSearches}", ) @@ -314,7 +312,7 @@ def executeBot(currentAccount: Account, args: argparse.Namespace): def export_points_to_csv(points_data): - logs_directory = Utils.getProjectRoot() / "logs" + logs_directory = getProjectRoot() / "logs" csv_filename = logs_directory / "points_data.csv" with open(csv_filename, mode="a", newline="") as file: # Use "a" mode for append fieldnames = ["Account", "Earned Points", "Points Difference"] @@ -331,9 +329,7 @@ def export_points_to_csv(points_data): # Define a function to load the previous day's points data from a file in the "logs" folder def load_previous_points_data(): try: - with open( - Utils.getProjectRoot() / "logs" / "previous_points_data.json", "r" - ) as file: + with open(getProjectRoot() / "logs" / "previous_points_data.json", "r") as file: return json.load(file) except FileNotFoundError: return {} @@ -341,7 +337,7 @@ def load_previous_points_data(): # Define a function to save the current day's points data for the next day in the "logs" folder def save_previous_points_data(data): - logs_directory = Utils.getProjectRoot() / "logs" + logs_directory = getProjectRoot() / "logs" with open(logs_directory / "previous_points_data.json", "w") as file: json.dump(data, file, indent=4) @@ -351,6 +347,6 @@ def save_previous_points_data(data): main() except Exception as e: logging.exception("") - Utils.sendNotification( + sendNotification( "⚠️ Error occurred, please check the log", traceback.format_exc(), e ) diff --git a/src/activities.py b/src/activities.py index 6d1a7d12..6119cb7f 100644 --- a/src/activities.py +++ b/src/activities.py @@ -9,7 +9,7 @@ from src.browser import Browser from src.constants import REWARDS_URL -from src.utils import CONFIG, Utils +from src.utils import CONFIG, sendNotification # todo These are US-English specific, maybe there's a good way to internationalize ACTIVITY_TITLE_TO_SEARCH = { @@ -174,7 +174,7 @@ def getAnswerAndCode(self, answerId: str) -> tuple[WebElement, str]: answerTitle = answer.get_attribute("data-option") return ( answer, - self.browser.utils.getAnswerCode(answerEncodeKey, answerTitle), + getAnswerCode(answerEncodeKey, answerTitle), ) def doActivity(self, activity: dict, activities: list[dict]) -> None: @@ -265,7 +265,7 @@ def completeActivities(self): ): incompleteActivities.pop("Safeguard your family's info", None) if incompleteActivities: - Utils.sendNotification( + sendNotification( f"We found some incomplete activities for {self.browser.username}", str(incompleteActivities) + "\n" + REWARDS_URL, ) diff --git a/src/browser.py b/src/browser.py index eb4b6a40..7c71db2a 100644 --- a/src/browser.py +++ b/src/browser.py @@ -19,7 +19,7 @@ from src import Account, RemainingSearches from src.userAgentGenerator import GenerateUserAgent -from src.utils import Utils, CONFIG +from src.utils import Utils, CONFIG, saveBrowserConfig, getProjectRoot, getBrowserConfig class Browser: @@ -45,7 +45,7 @@ def __init__( elif account.proxy: self.proxy = account.proxy self.userDataDir = self.setupProfiles() - self.browserConfig = Utils.getBrowserConfig(self.userDataDir) + self.browserConfig = getBrowserConfig(self.userDataDir) ( self.userAgent, self.userAgentMetadata, @@ -53,7 +53,7 @@ def __init__( ) = GenerateUserAgent().userAgent(self.browserConfig, mobile) if newBrowserConfig: self.browserConfig = newBrowserConfig - Utils.saveBrowserConfig(self.userDataDir, self.browserConfig) + saveBrowserConfig(self.userDataDir, self.browserConfig) self.webdriver = self.browserSetup() self.utils = Utils(self.webdriver) logging.debug("out __init__") @@ -151,7 +151,7 @@ def browserSetup( "height": deviceHeight, "width": deviceWidth, } - Utils.saveBrowserConfig(self.userDataDir, self.browserConfig) + saveBrowserConfig(self.userDataDir, self.browserConfig) if self.mobile: screenHeight = deviceHeight + 146 @@ -211,7 +211,7 @@ def setupProfiles(self) -> Path: Returns: Path """ - sessionsDir = Utils.getProjectRoot() / "sessions" + sessionsDir = getProjectRoot() / "sessions" # Concatenate username and browser type for a plain text session ID sessionid = f"{self.username}" diff --git a/src/morePromotions.py b/src/morePromotions.py index 52cfb875..c3dfd849 100644 --- a/src/morePromotions.py +++ b/src/morePromotions.py @@ -9,7 +9,7 @@ from src.browser import Browser from .activities import Activities -from .utils import Utils, CONFIG +from .utils import CONFIG, sendNotification PROMOTION_TITLE_TO_SEARCH = { "Search the lyrics of a song": "black sabbath supernaut lyrics", @@ -110,7 +110,7 @@ def completeMorePromotions(self): (promotion["title"], promotion["promotionType"]) ) if incompletePromotions: - Utils.sendNotification( + sendNotification( f"We found some incomplete promotions for {self.browser.username} to do!", incompletePromotions, ) diff --git a/src/searches.py b/src/searches.py index 410d93a2..a4a52a8b 100644 --- a/src/searches.py +++ b/src/searches.py @@ -13,7 +13,7 @@ from selenium.webdriver.common.by import By from src.browser import Browser -from src.utils import Utils, CONFIG, makeRequestsSession +from src.utils import CONFIG, makeRequestsSession, getProjectRoot class RetriesStrategy(Enum): @@ -47,7 +47,7 @@ def __init__(self, browser: Browser): self.browser = browser self.webdriver = browser.webdriver - dumbDbm = dbm.dumb.open((Utils.getProjectRoot() / "google_trends").__str__()) + dumbDbm = dbm.dumb.open((getProjectRoot() / "google_trends").__str__()) self.googleTrendsShelf: shelve.Shelf = shelve.Shelf(dumbDbm) def __enter__(self): diff --git a/src/utils.py b/src/utils.py index c8b0c27e..3b712907 100644 --- a/src/utils.py +++ b/src/utils.py @@ -69,56 +69,6 @@ def __init__(self, webdriver: WebDriver): # self.config = self.loadConfig() - @staticmethod - def getProjectRoot() -> Path: - return Path(__file__).parent.parent - - @staticmethod - def loadYaml(path: Path) -> dict: - with open(path, "r") as file: - yamlContents = yaml.safe_load(file) - if not yamlContents: - logging.info(f"{yamlContents} is empty") - yamlContents = {} - return yamlContents - - @staticmethod - def loadConfig( - configFilename="config.yaml", defaultConfig=DEFAULT_CONFIG - ) -> MappingProxyType: - configFile = Utils.getProjectRoot() / configFilename - try: - return MappingProxyType(defaultConfig | Utils.loadYaml(configFile)) - except OSError: - logging.info(f"{configFile} doesn't exist, returning defaults") - return defaultConfig - - @staticmethod - def loadPrivateConfig() -> MappingProxyType: - return Utils.loadConfig("config-private.yaml", DEFAULT_PRIVATE_CONFIG) - - @staticmethod - def sendNotification(title, body, e: Exception = None) -> None: - if Utils.args.disable_apprise or ( - e - and not CONFIG.get("apprise") - .get("notify") - .get("uncaught-exception") - .get("enabled") - ): - return - apprise = Apprise() - urls: list[str] = ( - # Utils.loadConfig("config-private.yaml").get("apprise", {}).get("urls", []) - PRIVATE_CONFIG.get("apprise").get("urls") - ) - if not urls: - logging.debug("No urls found, not sending notification") - return - for url in urls: - apprise.add(url) - assert apprise.notify(title=str(title), body=str(body)) - def waitUntilVisible( self, by: str, selector: str, timeToWait: float = 10 ) -> WebElement: @@ -170,12 +120,6 @@ def goToSearch(self) -> None: # self.webdriver.current_url == SEARCH_URL # ), f"{self.webdriver.current_url} {SEARCH_URL}" # need regex: AssertionError: https://www.bing.com/?toWww=1&redig=A5B72363182B49DEBB7465AD7520FDAA https://bing.com/ - @staticmethod - def getAnswerCode(key: str, string: str) -> str: - t = sum(ord(string[i]) for i in range(len(string))) - t += int(key[-2:], 16) - return str(t) - # Prefer getBingInfo if possible def getDashboardData(self) -> dict: urlBefore = self.webdriver.current_url @@ -268,36 +212,91 @@ def closeCurrentTab(self) -> None: self.webdriver.switch_to.window(window_name=self.webdriver.window_handles[0]) time.sleep(0.5) - @staticmethod - def formatNumber(number, num_decimals=2) -> str: - return pylocale.format_string( - f"%10.{num_decimals}f", number, grouping=True - ).strip() - - @staticmethod - def getBrowserConfig(sessionPath: Path) -> dict | None: - configFile = sessionPath / "config.json" - if not configFile.exists(): - return - with open(configFile, "r") as f: - return json.load(f) - - @staticmethod - def saveBrowserConfig(sessionPath: Path, config: dict) -> None: - configFile = sessionPath / "config.json" - with open(configFile, "w") as f: - json.dump(config, f) - def click(self, element: WebElement) -> None: try: - WebDriverWait(self.webdriver, 10).until(expected_conditions.element_to_be_clickable(element)) + WebDriverWait(self.webdriver, 10).until( + expected_conditions.element_to_be_clickable(element) + ) element.click() except (ElementClickInterceptedException, ElementNotInteractableException): self.tryDismissAllMessages() - WebDriverWait(self.webdriver, 10).until(expected_conditions.element_to_be_clickable(element)) + WebDriverWait(self.webdriver, 10).until( + expected_conditions.element_to_be_clickable(element) + ) element.click() +def getProjectRoot() -> Path: + return Path(__file__).parent.parent + + +def loadYaml(path: Path) -> dict: + with open(path, "r") as file: + yamlContents = yaml.safe_load(file) + if not yamlContents: + logging.info(f"{yamlContents} is empty") + yamlContents = {} + return yamlContents + + +def loadConfig( + configFilename="config.yaml", defaultConfig=DEFAULT_CONFIG +) -> MappingProxyType: + configFile = getProjectRoot() / configFilename + try: + return MappingProxyType(defaultConfig | loadYaml(configFile)) + except OSError: + logging.info(f"{configFile} doesn't exist, returning defaults") + return defaultConfig + + +def loadPrivateConfig() -> MappingProxyType: + return loadConfig("config-private.yaml", DEFAULT_PRIVATE_CONFIG) + + +def sendNotification(title: str, body: str, e: Exception = None) -> None: + if Utils.args.disable_apprise or ( + e + and not CONFIG.get("apprise") + .get("notify") + .get("uncaught-exception") + .get("enabled") + ): + return + apprise = Apprise() + urls: list[str] = PRIVATE_CONFIG.get("apprise").get("urls") + if not urls: + logging.debug("No urls found, not sending notification") + return + for url in urls: + apprise.add(url) + assert apprise.notify(title=str(title), body=str(body)) + + +def getAnswerCode(key: str, string: str) -> str: + t = sum(ord(string[i]) for i in range(len(string))) + t += int(key[-2:], 16) + return str(t) + + +def formatNumber(number, num_decimals=2) -> str: + return pylocale.format_string(f"%10.{num_decimals}f", number, grouping=True).strip() + + +def getBrowserConfig(sessionPath: Path) -> dict | None: + configFile = sessionPath / "config.json" + if not configFile.exists(): + return + with open(configFile, "r") as f: + return json.load(f) + + +def saveBrowserConfig(sessionPath: Path, config: dict) -> None: + configFile = sessionPath / "config.json" + with open(configFile, "w") as f: + json.dump(config, f) + + def makeRequestsSession(session: Session = requests.session()) -> Session: retry = Retry( total=5, @@ -318,5 +317,5 @@ def makeRequestsSession(session: Session = requests.session()) -> Session: return session -CONFIG = Utils.loadConfig() -PRIVATE_CONFIG = Utils.loadPrivateConfig() +CONFIG = loadConfig() +PRIVATE_CONFIG = loadPrivateConfig() diff --git a/test/test_utils.py b/test/test_utils.py index 780a76e8..da529a3b 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -1,11 +1,11 @@ from argparse import Namespace from unittest import TestCase -from src.utils import Utils +from src.utils import Utils, sendNotification class TestUtils(TestCase): def test_send_notification(self): Utils.args = Namespace() Utils.args.disable_apprise = False - Utils.sendNotification("title", "body") + sendNotification("title", "body") From ff3b02d39fa02cefd7d10b5dc8860730d3b6c336 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Wed, 23 Oct 2024 13:16:58 -0400 Subject: [PATCH 57/82] Remove staticmethod in favor of function https://stackoverflow.com/a/11788267/4164390 --- src/activities.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/activities.py b/src/activities.py index 6119cb7f..93d07fc6 100644 --- a/src/activities.py +++ b/src/activities.py @@ -9,7 +9,7 @@ from src.browser import Browser from src.constants import REWARDS_URL -from src.utils import CONFIG, sendNotification +from src.utils import CONFIG, sendNotification, getAnswerCode # todo These are US-English specific, maybe there's a good way to internationalize ACTIVITY_TITLE_TO_SEARCH = { From a15f2e5378790d935784af38535b270408e2d5b3 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Sat, 2 Nov 2024 09:36:04 -0400 Subject: [PATCH 58/82] Add fixme --- src/activities.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/activities.py b/src/activities.py index 93d07fc6..0f4c297e 100644 --- a/src/activities.py +++ b/src/activities.py @@ -240,6 +240,7 @@ def completeActivities(self): logging.info("[MORE PROMOS] Done") # todo Send one email for all accounts? + # fixme This is falsely considering some activities incomplete when complete if ( CONFIG.get("apprise") .get("notify") From c9364390522c5b5c8c8c58478d7c1a73d93c7f81 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Tue, 5 Nov 2024 14:04:36 -0500 Subject: [PATCH 59/82] Remove deprecated classes --- src/__init__.py | 4 +- src/dailySet.py | 91 -------------------------------- src/morePromotions.py | 117 ------------------------------------------ 3 files changed, 1 insertion(+), 211 deletions(-) delete mode 100644 src/dailySet.py delete mode 100644 src/morePromotions.py diff --git a/src/__init__.py b/src/__init__.py index 82174ec9..6caf6a60 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -1,9 +1,7 @@ from .account import Account -from .remainingSearches import RemainingSearches from .browser import Browser -from .dailySet import DailySet from .login import Login -from .morePromotions import MorePromotions from .punchCards import PunchCards from .readToEarn import ReadToEarn +from .remainingSearches import RemainingSearches from .searches import Searches diff --git a/src/dailySet.py b/src/dailySet.py deleted file mode 100644 index 187f3213..00000000 --- a/src/dailySet.py +++ /dev/null @@ -1,91 +0,0 @@ -import logging -import urllib.parse -from datetime import datetime - -from typing_extensions import deprecated - -from src.browser import Browser -from .activities import Activities - - -@deprecated("Use Activities") -class DailySet: - def __init__(self, browser: Browser): - self.browser = browser - self.webdriver = browser.webdriver - self.activities = Activities(browser) - - def completeDailySet(self): - # Function to complete the Daily Set - logging.info("[DAILY SET] " + "Trying to complete the Daily Set...") - data = self.browser.utils.getDashboardData()["dailySetPromotions"] - self.browser.utils.goToRewards() - self.activities.dashboardPopUpModalCloseCross() - todayDate = datetime.now().strftime("%m/%d/%Y") - for activity in data.get(todayDate, []): - cardId = int(activity["offerId"][-1:]) - try: - # Open the Daily Set activity - if activity["complete"] is not False: - continue - self.activities.openDailySetActivity(cardId) - if activity["promotionType"] == "urlreward": - logging.info(f"[DAILY SET] Completing search of card {cardId}") - # Complete search for URL reward - self.activities.completeSearch() - if activity["promotionType"] == "quiz": - if ( - activity["pointProgressMax"] == 50 - and activity["pointProgress"] == 0 - ): - logging.info( - "[DAILY SET] " + f"Completing This or That of card {cardId}" - ) - # Complete This or That for a specific point progress max - self.activities.completeThisOrThat() - elif activity["pointProgressMax"] in [40, 30]: - logging.info(f"[DAILY SET] Completing quiz of card {cardId}") - # Complete quiz for specific point progress max - self.activities.completeQuiz() - elif ( - activity["pointProgressMax"] == 10 - and activity["pointProgress"] == 0 - ): - # Extract and parse search URL for additional checks - searchUrl = urllib.parse.unquote( - urllib.parse.parse_qs( - urllib.parse.urlparse(activity["destinationUrl"]).query - )["ru"][0] - ) - searchUrlQueries = urllib.parse.parse_qs( - urllib.parse.urlparse(searchUrl).query - ) - filters = {} - for filterEl in searchUrlQueries["filters"][0].split(" "): - filterEl = filterEl.split(":", 1) - filters[filterEl[0]] = filterEl[1] - if "PollScenarioId" in filters: - logging.info( - f"[DAILY SET] Completing poll of card {cardId}" - ) - # Complete survey for a specific scenario - self.activities.completeSurvey() - else: - logging.info( - f"[DAILY SET] Completing quiz of card {cardId}" - ) - try: - # Try completing ABC activity - self.activities.completeABC() - except Exception: # pylint: disable=broad-except - logging.warning("", exc_info=True) - # Default to completing quiz - self.activities.completeQuiz() - except Exception: # pylint: disable=broad-except - logging.error( - f"[DAILY SET] Error Daily Set of card {cardId}", exc_info=True - ) - # Reset tabs in case of an exception - self.browser.utils.resetTabs() - continue - logging.info("[DAILY SET] Exiting") diff --git a/src/morePromotions.py b/src/morePromotions.py deleted file mode 100644 index c3dfd849..00000000 --- a/src/morePromotions.py +++ /dev/null @@ -1,117 +0,0 @@ -import contextlib -import logging -import random -import time - -from selenium.common import TimeoutException -from selenium.webdriver.common.by import By -from typing_extensions import deprecated - -from src.browser import Browser -from .activities import Activities -from .utils import CONFIG, sendNotification - -PROMOTION_TITLE_TO_SEARCH = { - "Search the lyrics of a song": "black sabbath supernaut lyrics", - "Translate anything": "translate pencil sharpener to spanish", - "Let's watch that movie again!": "aliens movie", - "Discover open job roles": "walmart open job roles", - "Plan a quick getaway": "flights nyc to paris", - "You can track your package": "usps tracking", - "Find somewhere new to explore": "directions to new york", - "Too tired to cook tonight?": "Pizza Hut near me", - "Quickly convert your money": "convert 374 usd to yen", - "Learn to cook a new recipe": "how cook pierogi", - "Find places to stay": "hotels rome italy", - "How's the economy?": "sp 500", - "Who won?": "braves score", - "Gaming time": "vampire survivors video game", - "What time is it?": "china time", - "Houses near you": "apartments manhattan", - "Get your shopping done faster": "chicken tenders", - "Expand your vocabulary": "define polymorphism", - "Stay on top of the elections": "election news latest", - "Prepare for the weather": "weather tomorrow", -} - - -@deprecated("Use Activities") -# todo Rename MoreActivities? -class MorePromotions: - def __init__(self, browser: Browser): - self.browser = browser - self.activities = Activities(browser) - - # todo Refactor so less complex - def completeMorePromotions(self): - # Function to complete More Promotions - logging.info("[MORE PROMOS] " + "Trying to complete More Promotions...") - morePromotions: list[dict] = self.browser.utils.getDashboardData()[ - "morePromotions" - ] - self.browser.utils.goToRewards() - for promotion in morePromotions: - try: - promotionTitle = ( - promotion["title"].replace("\u200b", "").replace("\xa0", " ") - ) - logging.debug(f"promotionTitle={promotionTitle}") - # Open the activity for the promotion - if ( - promotion["complete"] is not False - or promotion["pointProgressMax"] == 0 - ): - logging.debug("Already done, continuing") - continue - self.activities.openMorePromotionsActivity( - morePromotions.index(promotion) - ) - self.browser.webdriver.execute_script("window.scrollTo(0, 1080)") - with contextlib.suppress(TimeoutException): - searchbar = self.browser.utils.waitUntilClickable( - By.ID, "sb_form_q" - ) - self.browser.utils.click(searchbar) - # todo These and following are US-English specific, maybe there's a good way to internationalize - if promotionTitle in PROMOTION_TITLE_TO_SEARCH: - searchbar.send_keys(PROMOTION_TITLE_TO_SEARCH[promotionTitle]) - searchbar.submit() - elif promotion["promotionType"] == "urlreward": - # Complete search for URL reward - self.activities.completeSearch() - elif promotion["promotionType"] == "quiz": - # Complete different types of quizzes based on point progress max - if promotion["pointProgressMax"] == 10: - self.activities.completeABC() - elif promotion["pointProgressMax"] in [30, 40]: - self.activities.completeQuiz() - elif promotion["pointProgressMax"] == 50: - self.activities.completeThisOrThat() - else: - # Default to completing search - self.activities.completeSearch() - self.browser.webdriver.execute_script("window.scrollTo(0, 1080)") - time.sleep(random.randint(5, 10)) - - self.browser.utils.resetTabs() - time.sleep(2) - except Exception: # pylint: disable=broad-except - logging.error("[MORE PROMOS] Error More Promotions", exc_info=True) - # Reset tabs in case of an exception - self.browser.utils.resetTabs() - continue - if CONFIG.get("apprise").get("notify").get("incomplete-activity"): - incompletePromotions: list[tuple[str, str]] = [] - for promotion in self.browser.utils.getDashboardData()[ - "morePromotions" - ]: # Have to refresh - if promotion["pointProgress"] < promotion["pointProgressMax"]: - incompletePromotions.append( - (promotion["title"], promotion["promotionType"]) - ) - if incompletePromotions: - sendNotification( - f"We found some incomplete promotions for {self.browser.username} to do!", - incompletePromotions, - ) - logging.info("[MORE PROMOS] Exiting") From 6bed6fc196f3319635e3ecbdb6c06da73d6ace9e Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Tue, 5 Nov 2024 14:05:01 -0500 Subject: [PATCH 60/82] Up timeToWait and use default --- src/activities.py | 6 +++--- src/punchCards.py | 6 +++--- src/utils.py | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/activities.py b/src/activities.py index 0f4c297e..e9253c1e 100644 --- a/src/activities.py +++ b/src/activities.py @@ -49,7 +49,7 @@ def openDailySetActivity(self, cardId: int): f'//*[@id="daily-sets"]/mee-card-group[1]/div/mee-card[{cardId}]/div/card-content/mee-rewards-daily-set-item-content/div/a', ) self.browser.utils.click(element) - self.browser.utils.switchToNewTab(timeToWait=8) + self.browser.utils.switchToNewTab() def openMorePromotionsActivity(self, cardId: int): cardId += 1 @@ -59,7 +59,7 @@ def openMorePromotionsActivity(self, cardId: int): f"#more-activities > .m-card-group > .ng-scope:nth-child({cardId}) .ds-card-sec", ) self.browser.utils.click(element) - self.browser.utils.switchToNewTab(timeToWait=8) + self.browser.utils.switchToNewTab() def completeSearch(self): # Simulate completing a search activity @@ -192,12 +192,12 @@ def doActivity(self, activity: dict, activities: list[dict]) -> None: else: self.openMorePromotionsActivity(cardId) self.browser.webdriver.execute_script("window.scrollTo(0, 1080)") - sleep(1) with contextlib.suppress(TimeoutException): searchbar = self.browser.utils.waitUntilClickable(By.ID, "sb_form_q") self.browser.utils.click(searchbar) if activityTitle in ACTIVITY_TITLE_TO_SEARCH: searchbar.send_keys(ACTIVITY_TITLE_TO_SEARCH[activityTitle]) + sleep(2) searchbar.submit() elif "poll" in activityTitle: logging.info(f"[ACTIVITY] Completing poll of card {cardId}") diff --git a/src/punchCards.py b/src/punchCards.py index 1561d020..412f5495 100644 --- a/src/punchCards.py +++ b/src/punchCards.py @@ -23,12 +23,12 @@ def completePunchCard(self, url: str, childPromotions: dict): self.webdriver.find_element( By.XPATH, "//a[@class='offer-cta']/div" ).click() - self.browser.utils.switchToNewTab(random.randint(13, 17), True) + self.browser.utils.switchToNewTab(True) if child["promotionType"] == "quiz": self.webdriver.find_element( By.XPATH, "//a[@class='offer-cta']/div" ).click() - self.browser.utils.switchToNewTab(8) + self.browser.utils.switchToNewTab() counter = str( self.webdriver.find_element( By.XPATH, '//*[@id="QuestionPane0"]/div[2]' @@ -99,6 +99,6 @@ def completePromotionalItems(self): self.webdriver.find_element( By.XPATH, '//*[@id="promo-item"]/section/div/div/div/span' ).click() - self.browser.utils.switchToNewTab(8, True) + self.browser.utils.switchToNewTab(True) except Exception: logging.debug("", exc_info=True) diff --git a/src/utils.py b/src/utils.py index 3b712907..1f00118f 100644 --- a/src/utils.py +++ b/src/utils.py @@ -200,7 +200,7 @@ def tryDismissAllMessages(self) -> None: By.TAG_NAME, "button" ).click() - def switchToNewTab(self, timeToWait: float = 0, closeTab: bool = False) -> None: + def switchToNewTab(self, timeToWait: float = 15, closeTab: bool = False) -> None: time.sleep(timeToWait) self.webdriver.switch_to.window(window_name=self.webdriver.window_handles[1]) if closeTab: From 50dc6dcfb10ac3afe1fccadb7dd33738bc69fc66 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Wed, 6 Nov 2024 09:07:20 -0500 Subject: [PATCH 61/82] Fix import order --- src/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/__init__.py b/src/__init__.py index 6caf6a60..6f179194 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -1,7 +1,7 @@ from .account import Account +from .remainingSearches import RemainingSearches from .browser import Browser from .login import Login from .punchCards import PunchCards from .readToEarn import ReadToEarn -from .remainingSearches import RemainingSearches from .searches import Searches From 8f6ec041f784c874dfb692f5f73048c293a71fdd Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Thu, 7 Nov 2024 13:13:45 -0500 Subject: [PATCH 62/82] Increase base_delay_in_seconds --- config.yaml | 2 +- src/utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config.yaml b/config.yaml index 48cf0aaf..ddc1b3b7 100644 --- a/config.yaml +++ b/config.yaml @@ -12,6 +12,6 @@ default: logging: level: INFO # See https://docs.python.org/3/library/logging.html#logging-levels retries: - base_delay_in_seconds: 14.0625 # base_delay_in_seconds * 2^max = 14.0625 * 2^6 = 900 = 15 minutes + base_delay_in_seconds: 120 # base_delay_in_seconds * 2^max = 14.0625 * 2^6 = 900 = 15 minutes max: 4 strategy: EXPONENTIAL diff --git a/src/utils.py b/src/utils.py index 1f00118f..11c9470f 100644 --- a/src/utils.py +++ b/src/utils.py @@ -43,7 +43,7 @@ "default": {"geolocation": "US"}, "logging": {"level": "INFO"}, "retries": { - "base_delay_in_seconds": 14.0625, + "base_delay_in_seconds": 120, "max": 4, "strategy": "EXPONENTIAL", }, From d9d6e7d6a21d8707fde7a975ce6426231e64918b Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Thu, 14 Nov 2024 17:40:23 -0500 Subject: [PATCH 63/82] Sleep 5-10 minutes after success --- src/activities.py | 17 ++++++----------- src/punchCards.py | 1 - src/searches.py | 3 ++- 3 files changed, 8 insertions(+), 13 deletions(-) diff --git a/src/activities.py b/src/activities.py index e9253c1e..c49643d4 100644 --- a/src/activities.py +++ b/src/activities.py @@ -63,16 +63,12 @@ def openMorePromotionsActivity(self, cardId: int): def completeSearch(self): # Simulate completing a search activity - sleep(randint(20, 30)) - # WebDriverWait(self.webdriver, 30).until() - self.browser.utils.closeCurrentTab() + pass def completeSurvey(self): # Simulate completing a survey activity # noinspection SpellCheckingInspection self.webdriver.find_element(By.ID, f"btoption{randint(0, 1)}").click() - sleep(randint(10, 15)) - self.browser.utils.closeCurrentTab() def completeQuiz(self): # Simulate completing a quiz activity @@ -121,7 +117,6 @@ def completeQuiz(self): self.browser.utils.waitUntilQuestionRefresh() break - self.browser.utils.closeCurrentTab() def completeABC(self): # Simulate completing an ABC activity @@ -138,8 +133,6 @@ def completeABC(self): element = self.webdriver.find_element(By.ID, f"nextQuestionbtn{question}") self.browser.utils.click(element) sleep(randint(10, 15)) - sleep(randint(1, 7)) - self.browser.utils.closeCurrentTab() def completeThisOrThat(self): # Simulate completing a This or That activity @@ -164,8 +157,6 @@ def completeThisOrThat(self): self.browser.utils.click(answerToClick) sleep(randint(10, 15)) - sleep(randint(10, 15)) - self.browser.utils.closeCurrentTab() def getAnswerAndCode(self, answerId: str) -> tuple[WebElement, str]: # Helper function to get answer element and its code @@ -184,6 +175,9 @@ def doActivity(self, activity: dict, activities: list[dict]) -> None: if activity["complete"] is True or activity["pointProgressMax"] == 0: logging.debug("Already done, returning") return + if "Safeguard your family's info" == activityTitle: + logging.debug("Skipping Safeguard your family's info") + return # Open the activity for the activity cardId = activities.index(activity) isDailySet = "daily_set_date" in activity["attributes"] @@ -221,8 +215,9 @@ def doActivity(self, activity: dict, activities: list[dict]) -> None: # sleep(randint(5, 10)) except Exception: logging.error(f"[ACTIVITY] Error doing {activityTitle}", exc_info=True) + # todo Make configurable + sleep(randint(300, 600)) self.browser.utils.resetTabs() - # sleep(randint(900, 1200)) def completeActivities(self): logging.info("[DAILY SET] " + "Trying to complete the Daily Set...") diff --git a/src/punchCards.py b/src/punchCards.py index 412f5495..514e0a06 100644 --- a/src/punchCards.py +++ b/src/punchCards.py @@ -50,7 +50,6 @@ def completePunchCard(self, url: str, childPromotions: dict): ).click() time.sleep(random.randint(100, 700) / 100) time.sleep(random.randint(100, 700) / 100) - self.browser.utils.closeCurrentTab() def completePunchCards(self): # Function to complete all punch cards diff --git a/src/searches.py b/src/searches.py index a4a52a8b..0b2cdf00 100644 --- a/src/searches.py +++ b/src/searches.py @@ -182,7 +182,8 @@ def bingSearch(self) -> None: pointsAfter = self.browser.utils.getAccountPoints() if pointsBefore < pointsAfter: - # sleep(randint(900, 1200)) + # todo Make configurable + sleep(randint(300, 600)) return # todo From 68602ed54c873de9183ffd9274320053531e2263 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Thu, 14 Nov 2024 17:40:54 -0500 Subject: [PATCH 64/82] Get points from dashboard since more reliable --- src/browser.py | 9 +++++---- src/utils.py | 4 ++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/browser.py b/src/browser.py index 7c71db2a..8efa9488 100644 --- a/src/browser.py +++ b/src/browser.py @@ -277,11 +277,12 @@ def getChromeVersion() -> str: def getRemainingSearches( self, desktopAndMobile: bool = False ) -> RemainingSearches | int: - bingInfo = self.utils.getBingInfo() + # bingInfo = self.utils.getBingInfo() + bingInfo = self.utils.getDashboardData() searchPoints = 1 - counters = bingInfo["flyoutResult"]["userStatus"]["counters"] - pcSearch: dict = counters["PCSearch"][0] - mobileSearch: dict = counters["MobileSearch"][0] + counters = bingInfo["userStatus"]["counters"] + pcSearch: dict = counters["pcSearch"][0] + mobileSearch: dict = counters["mobileSearch"][0] pointProgressMax: int = pcSearch["pointProgressMax"] searchPoints: int diff --git a/src/utils.py b/src/utils.py index 11c9470f..e9ce9294 100644 --- a/src/utils.py +++ b/src/utils.py @@ -140,6 +140,7 @@ def getDailySetPromotions(self) -> list[dict]: def getMorePromotions(self) -> list[dict]: return self.getDashboardData()["morePromotions"] + # Not reliable def getBingInfo(self) -> Any: session = makeRequestsSession() @@ -154,7 +155,6 @@ def getBingInfo(self) -> Any: return response.json() def isLoggedIn(self) -> bool: - # return self.getBingInfo()["isRewardsUser"] # todo For some reason doesn't work, but doesn't involve changing url so preferred if self.getBingInfo()["isRewardsUser"]: # faster, if it works return True self.webdriver.get( @@ -168,7 +168,7 @@ def isLoggedIn(self) -> bool: return False def getAccountPoints(self) -> int: - return self.getBingInfo()["userInfo"]["balance"] + return self.getDashboardData()["userStatus"]["availablePoints"] def getGoalPoints(self) -> int: return self.getDashboardData()["userStatus"]["redeemGoal"]["price"] From ef098d9824790d6299ec08b1ae6dd555adb76481 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Thu, 14 Nov 2024 17:41:03 -0500 Subject: [PATCH 65/82] Remove scrolls --- src/activities.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/activities.py b/src/activities.py index c49643d4..9bf7aa6c 100644 --- a/src/activities.py +++ b/src/activities.py @@ -185,7 +185,6 @@ def doActivity(self, activity: dict, activities: list[dict]) -> None: self.openDailySetActivity(cardId) else: self.openMorePromotionsActivity(cardId) - self.browser.webdriver.execute_script("window.scrollTo(0, 1080)") with contextlib.suppress(TimeoutException): searchbar = self.browser.utils.waitUntilClickable(By.ID, "sb_form_q") self.browser.utils.click(searchbar) @@ -211,8 +210,6 @@ def doActivity(self, activity: dict, activities: list[dict]) -> None: else: # Default to completing search self.completeSearch() - self.browser.webdriver.execute_script("window.scrollTo(0, 1080)") - # sleep(randint(5, 10)) except Exception: logging.error(f"[ACTIVITY] Error doing {activityTitle}", exc_info=True) # todo Make configurable From 245d3b1436d3dbb3ebfee5e396c8c584cb84a844 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Fri, 15 Nov 2024 11:32:16 -0500 Subject: [PATCH 66/82] Remove comment --- src/activities.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/activities.py b/src/activities.py index 9bf7aa6c..9d4b8275 100644 --- a/src/activities.py +++ b/src/activities.py @@ -75,7 +75,6 @@ def completeQuiz(self): with contextlib.suppress(TimeoutException): startQuiz = self.browser.utils.waitUntilQuizLoads() self.browser.utils.click(startQuiz) - # this is bugged on Chrome for some reason self.browser.utils.waitUntilVisible(By.ID, "overlayPanel", 5) currentQuestionNumber: int = self.webdriver.execute_script( "return _w.rewardsQuizRenderInfo.currentQuestionNumber" From ef42d254f294c93b73f1e99db6e1a6454d1b4177 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Sun, 17 Nov 2024 17:23:54 -0500 Subject: [PATCH 67/82] Log incomplete activities --- src/activities.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/activities.py b/src/activities.py index 9d4b8275..1530f2aa 100644 --- a/src/activities.py +++ b/src/activities.py @@ -257,6 +257,7 @@ def completeActivities(self): ): incompleteActivities.pop("Safeguard your family's info", None) if incompleteActivities: + logging.info(f"incompleteActivities: {incompleteActivities}") sendNotification( f"We found some incomplete activities for {self.browser.username}", str(incompleteActivities) + "\n" + REWARDS_URL, From 81afb3279c5666e079105c13bb86a7554e2bdc1e Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Sun, 17 Nov 2024 17:25:37 -0500 Subject: [PATCH 68/82] Check for level before getting remaining mobile searches --- src/browser.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/src/browser.py b/src/browser.py index 8efa9488..cc332b21 100644 --- a/src/browser.py +++ b/src/browser.py @@ -282,7 +282,6 @@ def getRemainingSearches( searchPoints = 1 counters = bingInfo["userStatus"]["counters"] pcSearch: dict = counters["pcSearch"][0] - mobileSearch: dict = counters["mobileSearch"][0] pointProgressMax: int = pcSearch["pointProgressMax"] searchPoints: int @@ -293,11 +292,21 @@ def getRemainingSearches( pcPointsRemaining = pcSearch["pointProgressMax"] - pcSearch["pointProgress"] assert pcPointsRemaining % searchPoints == 0 remainingDesktopSearches: int = int(pcPointsRemaining / searchPoints) - mobilePointsRemaining = ( - mobileSearch["pointProgressMax"] - mobileSearch["pointProgress"] - ) - assert mobilePointsRemaining % searchPoints == 0 - remainingMobileSearches: int = int(mobilePointsRemaining / searchPoints) + + activeLevel = bingInfo["userStatus"]["levelInfo"]["activeLevel"] + remainingMobileSearches: int = 0 + if activeLevel == "Level2": + mobileSearch: dict = counters["mobileSearch"][0] + mobilePointsRemaining = ( + mobileSearch["pointProgressMax"] - mobileSearch["pointProgress"] + ) + assert mobilePointsRemaining % searchPoints == 0 + remainingMobileSearches = int(mobilePointsRemaining / searchPoints) + elif activeLevel == "Level1": + pass + else: + raise AssertionError(f"Unknown activeLevel: {activeLevel}") + if desktopAndMobile: return RemainingSearches( desktop=remainingDesktopSearches, mobile=remainingMobileSearches From 6511198664ab7fca1489cdabf46e14f873f75b1b Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Mon, 18 Nov 2024 17:17:39 -0500 Subject: [PATCH 69/82] Fix Discover open job roles --- src/activities.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/activities.py b/src/activities.py index 1530f2aa..3cb8ffde 100644 --- a/src/activities.py +++ b/src/activities.py @@ -13,7 +13,7 @@ # todo These are US-English specific, maybe there's a good way to internationalize ACTIVITY_TITLE_TO_SEARCH = { - "Discover open job roles": "walmart open job roles", + "Discover open job roles": "jobs at microsoft", "Expand your vocabulary": "define demure", "Find places to stay": "hotels rome italy", "Find somewhere new to explore": "directions to new york", From 2e2540cea1b0f202c9efaf44aea80e665529eba3 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Mon, 25 Nov 2024 16:14:04 -0500 Subject: [PATCH 70/82] Add new activities --- src/activities.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/activities.py b/src/activities.py index 3cb8ffde..7c018f38 100644 --- a/src/activities.py +++ b/src/activities.py @@ -13,6 +13,7 @@ # todo These are US-English specific, maybe there's a good way to internationalize ACTIVITY_TITLE_TO_SEARCH = { + "Black Friday shopping": "black friday deals", "Discover open job roles": "jobs at microsoft", "Expand your vocabulary": "define demure", "Find places to stay": "hotels rome italy", @@ -31,6 +32,7 @@ "Too tired to cook tonight?": "Pizza Hut near me", "Translate anything": "translate pencil sharpener to spanish", "What time is it?": "china time", + "What's for Thanksgiving dinner?": "pumpkin pie recipe", "Who won?": "braves score", "You can track your package": "usps tracking", } From 815001ab53ec4be6a5661a308fe416db604b390f Mon Sep 17 00:00:00 2001 From: Kyrela Date: Fri, 29 Nov 2024 18:54:49 +0100 Subject: [PATCH 71/82] Add apprise notification on login with phone code --- src/login.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/login.py b/src/login.py index 01978342..1724e7ff 100644 --- a/src/login.py +++ b/src/login.py @@ -13,6 +13,7 @@ from undetected_chromedriver import Chrome from src.browser import Browser +from src.utils import sendNotification class Login: @@ -101,6 +102,8 @@ def execute_login(self) -> None: "[LOGIN] Confirm your login with code %s on your phone (you have one minute)!\a", codeField.text, ) + sendNotification( + f"Confirm your login on your phone", f"Code: {codeField.text} (expires in 1 minute)") self.utils.waitUntilVisible(By.NAME, "kmsiForm", 60) logging.info("[LOGIN] Successfully verified!") else: @@ -140,6 +143,8 @@ def execute_login(self) -> None: " one minute)!\a", codeField.text, ) + sendNotification( + f"Confirm your login on your phone", f"Code: {codeField.text} (expires in 1 minute)") self.utils.waitUntilVisible(By.NAME, "kmsiForm", 60) logging.info("[LOGIN] Successfully verified!") From b6ee600c0c405480a5ced0c2107b5b7928100e88 Mon Sep 17 00:00:00 2001 From: Kyrela Date: Sun, 1 Dec 2024 03:06:15 +0100 Subject: [PATCH 72/82] Add login notifications configuration in config.yaml --- config.yaml | 2 ++ src/login.py | 22 +++++++++++++++++----- src/utils.py | 1 + 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/config.yaml b/config.yaml index ddc1b3b7..b835582b 100644 --- a/config.yaml +++ b/config.yaml @@ -6,6 +6,8 @@ apprise: ignore-safeguard-info: True uncaught-exception: enabled: True # True or False + login-code: + enabled: True # True or False summary: ON_ERROR default: geolocation: US # Replace with your country code https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2 diff --git a/src/login.py b/src/login.py index 1724e7ff..edf8556e 100644 --- a/src/login.py +++ b/src/login.py @@ -13,7 +13,7 @@ from undetected_chromedriver import Chrome from src.browser import Browser -from src.utils import sendNotification +from src.utils import sendNotification, CONFIG class Login: @@ -102,8 +102,14 @@ def execute_login(self) -> None: "[LOGIN] Confirm your login with code %s on your phone (you have one minute)!\a", codeField.text, ) - sendNotification( - f"Confirm your login on your phone", f"Code: {codeField.text} (expires in 1 minute)") + if ( + CONFIG.get("apprise") + .get("notify") + .get("login-code") + .get("enabled") + ): + sendNotification( + f"Confirm your login on your phone", f"Code: {codeField.text} (expires in 1 minute)") self.utils.waitUntilVisible(By.NAME, "kmsiForm", 60) logging.info("[LOGIN] Successfully verified!") else: @@ -143,8 +149,14 @@ def execute_login(self) -> None: " one minute)!\a", codeField.text, ) - sendNotification( - f"Confirm your login on your phone", f"Code: {codeField.text} (expires in 1 minute)") + if ( + CONFIG.get("apprise") + .get("notify") + .get("login-code") + .get("enabled") + ): + sendNotification( + f"Confirm your login on your phone", f"Code: {codeField.text} (expires in 1 minute)") self.utils.waitUntilVisible(By.NAME, "kmsiForm", 60) logging.info("[LOGIN] Successfully verified!") diff --git a/src/utils.py b/src/utils.py index e9ce9294..87eceac7 100644 --- a/src/utils.py +++ b/src/utils.py @@ -37,6 +37,7 @@ "notify": { "incomplete-activity": {"enabled": True, "ignore-safeguard-info": True}, "uncaught-exception": {"enabled": True}, + "login-code": {"enabled": True}, }, "summary": "ON_ERROR", }, From 0785ba2155740bc3912bc24a7cc86ae72b2c1260 Mon Sep 17 00:00:00 2001 From: jdeath <17914369+jdeath@users.noreply.github.com> Date: Tue, 3 Dec 2024 06:44:53 -0500 Subject: [PATCH 73/82] Update userAgentGenerator.py --- src/userAgentGenerator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/userAgentGenerator.py b/src/userAgentGenerator.py index 854de78e..039b4f91 100644 --- a/src/userAgentGenerator.py +++ b/src/userAgentGenerator.py @@ -141,7 +141,7 @@ def getEdgeVersions(self) -> tuple[str, str]: ) data = response.json() if stableProduct := next( - (product for product in data if product["Product"] == "Stable"), + (product for product in data if product["product"] == "Stable"), None, ): releases = stableProduct["Releases"] From 68e4c7f3b48714d48ed68b797a42751d2c055697 Mon Sep 17 00:00:00 2001 From: jdeath <17914369+jdeath@users.noreply.github.com> Date: Tue, 3 Dec 2024 06:50:10 -0500 Subject: [PATCH 74/82] Update userAgentGenerator.py --- src/userAgentGenerator.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/userAgentGenerator.py b/src/userAgentGenerator.py index 039b4f91..7936a9f4 100644 --- a/src/userAgentGenerator.py +++ b/src/userAgentGenerator.py @@ -144,9 +144,9 @@ def getEdgeVersions(self) -> tuple[str, str]: (product for product in data if product["product"] == "Stable"), None, ): - releases = stableProduct["Releases"] + releases = stableProduct["releases"] androidRelease = next( - (release for release in releases if release["Platform"] == "Android"), + (release for release in releases if release["platform"] == "Android"), None, ) windowsRelease = next( @@ -160,8 +160,8 @@ def getEdgeVersions(self) -> tuple[str, str]: ) if androidRelease and windowsRelease: return ( - windowsRelease["ProductVersion"], - androidRelease["ProductVersion"], + windowsRelease["productVersion"], + androidRelease["productVersion"], ) raise HTTPError("Failed to get Edge versions.") From a1352d841646e050a409ca7cface8536f2953e69 Mon Sep 17 00:00:00 2001 From: jdeath <17914369+jdeath@users.noreply.github.com> Date: Tue, 3 Dec 2024 06:53:55 -0500 Subject: [PATCH 75/82] Update userAgentGenerator.py --- src/userAgentGenerator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/userAgentGenerator.py b/src/userAgentGenerator.py index 7936a9f4..0f8a44a2 100644 --- a/src/userAgentGenerator.py +++ b/src/userAgentGenerator.py @@ -153,8 +153,8 @@ def getEdgeVersions(self) -> tuple[str, str]: ( release for release in releases - if release["Platform"] == "Windows" - and release["Architecture"] == "x64" + if release["platform"] == "Windows" + and release["architecture"] == "x64" ), None, ) From 7974152e806c45445069200495dd4a5ee9016cd1 Mon Sep 17 00:00:00 2001 From: Steve_Tryndamere Date: Sun, 8 Dec 2024 08:58:06 -0500 Subject: [PATCH 76/82] fix: resolve KeyError in getEdgeVersions and bug in getLanguageCountry --- .gitignore | 1 + activities.py | 141 ++++++++++++++++++++++++++++++++++++++ config.yaml | 1 + src/browser.py | 13 ++-- src/userAgentGenerator.py | 30 ++++++-- src/utils.py | 9 ++- 6 files changed, 178 insertions(+), 17 deletions(-) create mode 100644 activities.py diff --git a/.gitignore b/.gitignore index 7302770d..03073b38 100644 --- a/.gitignore +++ b/.gitignore @@ -187,3 +187,4 @@ runbot.bat /google_trends.dir /google_trends.bak /config-private.yaml +mypy.ini diff --git a/activities.py b/activities.py new file mode 100644 index 00000000..bebb0a8c --- /dev/null +++ b/activities.py @@ -0,0 +1,141 @@ +import contextlib +import random +import time + +from selenium.common import TimeoutException +from selenium.webdriver.common.by import By +from selenium.webdriver.remote.webelement import WebElement + +from src.browser import Browser + + +class Activities: + def __init__(self, browser: Browser): + self.browser = browser + self.webdriver = browser.webdriver + + def openDailySetActivity(self, cardId: int): + # Open the Daily Set activity for the given cardId + element = self.webdriver.find_element(By.XPATH, + f'//*[@id="daily-sets"]/mee-card-group[1]/div/mee-card[{cardId}]/div/card-content/mee-rewards-daily-set-item-content/div/a', ) + self.browser.utils.click(element) + self.browser.utils.switchToNewTab(timeToWait=8) + + def openMorePromotionsActivity(self, cardId: int): + # Open the More Promotions activity for the given cardId + element = self.webdriver.find_element(By.CSS_SELECTOR, + f"#more-activities > .m-card-group > .ng-scope:nth-child({cardId + 1}) .ds-card-sec") + self.browser.utils.click(element) + self.browser.utils.switchToNewTab(timeToWait=5) + + def completeSearch(self): + # Simulate completing a search activity + time.sleep(random.randint(10, 15)) + self.browser.utils.closeCurrentTab() + + def completeSurvey(self): + # Simulate completing a survey activity + # noinspection SpellCheckingInspection + self.webdriver.find_element(By.ID, f"btoption{random.randint(0, 1)}").click() + time.sleep(random.randint(10, 15)) + self.browser.utils.closeCurrentTab() + + def completeQuiz(self): + # Simulate completing a quiz activity + with contextlib.suppress(TimeoutException): + startQuiz = self.browser.utils.waitUntilQuizLoads() + self.browser.utils.click(startQuiz) + self.browser.utils.waitUntilVisible( + By.ID, "overlayPanel", 5 + ) + currentQuestionNumber: int = self.webdriver.execute_script( + "return _w.rewardsQuizRenderInfo.currentQuestionNumber" + ) + maxQuestions = self.webdriver.execute_script( + "return _w.rewardsQuizRenderInfo.maxQuestions" + ) + numberOfOptions = self.webdriver.execute_script( + "return _w.rewardsQuizRenderInfo.numberOfOptions" + ) + for _ in range(currentQuestionNumber, maxQuestions + 1): + if numberOfOptions == 8: + answers = [] + for i in range(numberOfOptions): + isCorrectOption = self.webdriver.find_element( + By.ID, f"rqAnswerOption{i}" + ).get_attribute("iscorrectoption") + if isCorrectOption and isCorrectOption.lower() == "true": + answers.append(f"rqAnswerOption{i}") + for answer in answers: + element = self.webdriver.find_element(By.ID, answer) + self.browser.utils.click(element) + self.browser.utils.waitUntilQuestionRefresh() + elif numberOfOptions in [2, 3, 4]: + correctOption = self.webdriver.execute_script( + "return _w.rewardsQuizRenderInfo.correctAnswer" + ) + for i in range(numberOfOptions): + if ( + self.webdriver.find_element( + By.ID, f"rqAnswerOption{i}" + ).get_attribute("data-option") + == correctOption + ): + element = self.webdriver.find_element(By.ID, f"rqAnswerOption{i}") + self.browser.utils.click(element) + + self.browser.utils.waitUntilQuestionRefresh() + break + self.browser.utils.closeCurrentTab() + + def completeABC(self): + # Simulate completing an ABC activity + counter = self.webdriver.find_element( + By.XPATH, '//*[@id="QuestionPane0"]/div[2]' + ).text[:-1][1:] + numberOfQuestions = max(int(s) for s in counter.split() if s.isdigit()) + for question in range(numberOfQuestions): + element = self.webdriver.find_element(By.ID, f"questionOptionChoice{question}{random.randint(0, 2)}") + self.browser.utils.click(element) + time.sleep(random.randint(10, 15)) + element = self.webdriver.find_element(By.ID, f"nextQuestionbtn{question}") + self.browser.utils.click(element) + time.sleep(random.randint(10, 15)) + time.sleep(random.randint(1, 7)) + self.browser.utils.closeCurrentTab() + + def completeThisOrThat(self): + # Simulate completing a This or That activity + startQuiz = self.browser.utils.waitUntilQuizLoads() + self.browser.utils.click(startQuiz) + self.browser.utils.waitUntilVisible( + By.XPATH, '//*[@id="currentQuestionContainer"]/div/div[1]', 10 + ) + time.sleep(random.randint(10, 15)) + for _ in range(10): + correctAnswerCode = self.webdriver.execute_script( + "return _w.rewardsQuizRenderInfo.correctAnswer" + ) + answer1, answer1Code = self.getAnswerAndCode("rqAnswerOption0") + answer2, answer2Code = self.getAnswerAndCode("rqAnswerOption1") + answerToClick: WebElement + if answer1Code == correctAnswerCode: + answerToClick = answer1 + elif answer2Code == correctAnswerCode: + answerToClick = answer2 + + self.browser.utils.click(answerToClick) + time.sleep(random.randint(10, 15)) + + time.sleep(random.randint(10, 15)) + self.browser.utils.closeCurrentTab() + + def getAnswerAndCode(self, answerId: str) -> tuple[WebElement, str]: + # Helper function to get answer element and its code + answerEncodeKey = self.webdriver.execute_script("return _G.IG") + answer = self.webdriver.find_element(By.ID, answerId) + answerTitle = answer.get_attribute("data-option") + return ( + answer, + self.browser.utils.getAnswerCode(answerEncodeKey, answerTitle), + ) diff --git a/config.yaml b/config.yaml index b835582b..9bc98b96 100644 --- a/config.yaml +++ b/config.yaml @@ -11,6 +11,7 @@ apprise: summary: ON_ERROR default: geolocation: US # Replace with your country code https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2 + language: en # Replace with your language code https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes logging: level: INFO # See https://docs.python.org/3/library/logging.html#logging-levels retries: diff --git a/src/browser.py b/src/browser.py index cc332b21..0ee94e3c 100644 --- a/src/browser.py +++ b/src/browser.py @@ -19,7 +19,7 @@ from src import Account, RemainingSearches from src.userAgentGenerator import GenerateUserAgent -from src.utils import Utils, CONFIG, saveBrowserConfig, getProjectRoot, getBrowserConfig +from src.utils import CONFIG, Utils, getBrowserConfig, getProjectRoot, saveBrowserConfig class Browser: @@ -102,7 +102,7 @@ def browserSetup( options.add_argument("--disable-features=PrivacySandboxSettings4") options.add_argument("--disable-http2") options.add_argument("--disable-search-engine-choice-screen") # 153 - options.page_load_strategy = 'eager' + options.page_load_strategy = "eager" seleniumwireOptions: dict[str, Any] = {"verify_ssl": False} @@ -223,19 +223,22 @@ def setupProfiles(self) -> Path: @staticmethod def getLanguageCountry(language: str, country: str) -> tuple[str, str]: if not country: - country = CONFIG.get("default").get("location") + country = CONFIG.get("default").get("geolocation") + + if not language: + country = CONFIG.get("default").get("language") if not language or not country: currentLocale = locale.getlocale() if not language: with contextlib.suppress(ValueError): language = pycountry.languages.get( - name=currentLocale[0].split("_")[0] + alpha_2=currentLocale[0].split("_")[0] ).alpha_2 if not country: with contextlib.suppress(ValueError): country = pycountry.countries.get( - name=currentLocale[0].split("_")[1] + alpha_2=currentLocale[0].split("_")[1] ).alpha_2 if not language or not country: diff --git a/src/userAgentGenerator.py b/src/userAgentGenerator.py index 0f8a44a2..1696657a 100644 --- a/src/userAgentGenerator.py +++ b/src/userAgentGenerator.py @@ -139,29 +139,45 @@ def getEdgeVersions(self) -> tuple[str, str]: response = self.getWebdriverPage( "https://edgeupdates.microsoft.com/api/products" ) + + def get_value_ignore_case(data: dict, key: str) -> Any: + """Get the value from a dictionary ignoring the case of the first letter of the key.""" + for k, v in data.items(): + if k.lower() == key.lower(): + return v + return None + data = response.json() if stableProduct := next( - (product for product in data if product["product"] == "Stable"), + ( + product + for product in data + if get_value_ignore_case(product, "product") == "Stable" + ), None, ): - releases = stableProduct["releases"] + releases = get_value_ignore_case(stableProduct, "releases") androidRelease = next( - (release for release in releases if release["platform"] == "Android"), + ( + release + for release in releases + if get_value_ignore_case(release, "platform") == "Android" + ), None, ) windowsRelease = next( ( release for release in releases - if release["platform"] == "Windows" - and release["architecture"] == "x64" + if get_value_ignore_case(release, "platform") == "Windows" + and get_value_ignore_case(release, "architecture") == "x64" ), None, ) if androidRelease and windowsRelease: return ( - windowsRelease["productVersion"], - androidRelease["productVersion"], + get_value_ignore_case(windowsRelease, "productVersion"), + get_value_ignore_case(androidRelease, "productVersion"), ) raise HTTPError("Failed to get Edge versions.") diff --git a/src/utils.py b/src/utils.py index 87eceac7..6b8781a1 100644 --- a/src/utils.py +++ b/src/utils.py @@ -16,10 +16,10 @@ from requests import Session from requests.adapters import HTTPAdapter from selenium.common import ( - NoSuchElementException, - TimeoutException, ElementClickInterceptedException, ElementNotInteractableException, + NoSuchElementException, + TimeoutException, ) from selenium.webdriver.chrome.webdriver import WebDriver from selenium.webdriver.common.by import By @@ -28,8 +28,7 @@ from selenium.webdriver.support.wait import WebDriverWait from urllib3 import Retry -from .constants import REWARDS_URL -from .constants import SEARCH_URL +from .constants import REWARDS_URL, SEARCH_URL DEFAULT_CONFIG: MappingProxyType = MappingProxyType( { @@ -271,7 +270,7 @@ def sendNotification(title: str, body: str, e: Exception = None) -> None: return for url in urls: apprise.add(url) - assert apprise.notify(title=str(title), body=str(body)) + # assert apprise.notify(title=str(title), body=str(body)) # not work for telegram def getAnswerCode(key: str, string: str) -> str: From 6d23c75abf3cc449e3723724df6e6a121463a158 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Sun, 8 Dec 2024 21:09:07 -0500 Subject: [PATCH 77/82] Handle when daily_set_date exists but is empty --- src/activities.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/activities.py b/src/activities.py index 7c018f38..7788d806 100644 --- a/src/activities.py +++ b/src/activities.py @@ -181,7 +181,7 @@ def doActivity(self, activity: dict, activities: list[dict]) -> None: return # Open the activity for the activity cardId = activities.index(activity) - isDailySet = "daily_set_date" in activity["attributes"] + isDailySet = "daily_set_date" in activity["attributes"] and activity["attributes"]["daily_set_date"] if isDailySet: self.openDailySetActivity(cardId) else: From 2bf51d38a57af2f802e1f7ac7977fef3726c9f4a Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Sun, 8 Dec 2024 21:11:35 -0500 Subject: [PATCH 78/82] Add ability to more generically ignore activities --- config.yaml | 4 +++- src/activities.py | 18 +++++++++++------- src/utils.py | 8 +++++++- 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/config.yaml b/config.yaml index b835582b..2ae47202 100644 --- a/config.yaml +++ b/config.yaml @@ -3,7 +3,9 @@ apprise: notify: incomplete-activity: enabled: True # True or False - ignore-safeguard-info: True + ignore: + - Get 50 entries plus 1000 points! + - Safeguard your family's info uncaught-exception: enabled: True # True or False login-code: diff --git a/src/activities.py b/src/activities.py index 7788d806..91a36a70 100644 --- a/src/activities.py +++ b/src/activities.py @@ -158,7 +158,6 @@ def completeThisOrThat(self): self.browser.utils.click(answerToClick) sleep(randint(10, 15)) - def getAnswerAndCode(self, answerId: str) -> tuple[WebElement, str]: # Helper function to get answer element and its code answerEncodeKey = self.webdriver.execute_script("return _G.IG") @@ -176,12 +175,17 @@ def doActivity(self, activity: dict, activities: list[dict]) -> None: if activity["complete"] is True or activity["pointProgressMax"] == 0: logging.debug("Already done, returning") return - if "Safeguard your family's info" == activityTitle: - logging.debug("Skipping Safeguard your family's info") + if activityTitle in CONFIG.get("apprise").get("notify").get( + "incomplete-activity" + ).get("ignore"): + logging.debug(f"Ignoring {activityTitle}") return # Open the activity for the activity cardId = activities.index(activity) - isDailySet = "daily_set_date" in activity["attributes"] and activity["attributes"]["daily_set_date"] + isDailySet = ( + "daily_set_date" in activity["attributes"] + and activity["attributes"]["daily_set_date"] + ) if isDailySet: self.openDailySetActivity(cardId) else: @@ -251,13 +255,13 @@ def completeActivities(self): activity["pointProgress"], activity["pointProgressMax"], ) - if ( + for incompleteActivityToIgnore in ( CONFIG.get("apprise") .get("notify") .get("incomplete-activity") - .get("ignore-safeguard-info") + .get("ignore") ): - incompleteActivities.pop("Safeguard your family's info", None) + incompleteActivities.pop(incompleteActivityToIgnore, None) if incompleteActivities: logging.info(f"incompleteActivities: {incompleteActivities}") sendNotification( diff --git a/src/utils.py b/src/utils.py index 87eceac7..1c01f13b 100644 --- a/src/utils.py +++ b/src/utils.py @@ -35,7 +35,13 @@ { "apprise": { "notify": { - "incomplete-activity": {"enabled": True, "ignore-safeguard-info": True}, + "incomplete-activity": { + "enabled": True, + "ignore": [ + "Get 50 entries plus 1000 points!", + "Safeguard your family's info", + ], + }, "uncaught-exception": {"enabled": True}, "login-code": {"enabled": True}, }, From c6d58899f1be0933d371a6fff12e4c9b582e890a Mon Sep 17 00:00:00 2001 From: Kyrela Date: Sun, 15 Dec 2024 00:26:37 +0100 Subject: [PATCH 79/82] Exit 1 on exception --- main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/main.py b/main.py index 60053a0a..cfc57c83 100644 --- a/main.py +++ b/main.py @@ -350,3 +350,4 @@ def save_previous_points_data(data): sendNotification( "⚠️ Error occurred, please check the log", traceback.format_exc(), e ) + exit(1) From 63c6b3200aacba5d38aedbaa8d46aba7619d2a65 Mon Sep 17 00:00:00 2001 From: Kyrela Date: Tue, 17 Dec 2024 23:14:46 +0100 Subject: [PATCH 80/82] Better, simpler and unified config Replaces the old static configuration system with a dynamic `Config` class supporting nested structures, defaults, YAML serialization and direct access as attributes. Introduces enhanced CLI options for better usability and removes deprecated configuration files and structures. --- .github/ISSUE_TEMPLATE/bug_report.yml | 3 +- .gitignore | 1 + .template-config-private.yaml | 4 - README.md | 158 ++++++--- accounts.json.sample | 14 - config.yaml | 22 -- main.py | 129 ++------ src/__init__.py | 1 - src/account.py | 9 - src/activities.py | 53 +-- src/browser.py | 30 +- src/login.py | 26 +- src/readToEarn.py | 2 +- src/searches.py | 9 +- src/utils.py | 459 +++++++++++++++++++++++--- test/test_main.py | 8 +- test/test_utils.py | 6 +- 17 files changed, 586 insertions(+), 348 deletions(-) delete mode 100644 .template-config-private.yaml delete mode 100644 accounts.json.sample delete mode 100644 config.yaml delete mode 100644 src/account.py diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 16db6461..da2d09ce 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -46,7 +46,8 @@ body: attributes: label: Copy and paste your error description: | - From the terminal that was running the farmer, copy and paste the error/bug here. + Run the program with -d argument (ex: python main.py -d) and, from the terminal that was running the farmer, + copy and paste the log/error/bug here. validations: required: true - type: textarea diff --git a/.gitignore b/.gitignore index 03073b38..fa083af2 100644 --- a/.gitignore +++ b/.gitignore @@ -188,3 +188,4 @@ runbot.bat /google_trends.bak /config-private.yaml mypy.ini +config.yaml diff --git a/.template-config-private.yaml b/.template-config-private.yaml deleted file mode 100644 index 2c8b441a..00000000 --- a/.template-config-private.yaml +++ /dev/null @@ -1,4 +0,0 @@ -# config-private.yaml -apprise: - urls: - - "discord://{WebhookID}/{WebhookToken}" # Replace with your actual Apprise service URLs diff --git a/README.md b/README.md index ddd72413..57610876 100644 --- a/README.md +++ b/README.md @@ -46,10 +46,12 @@ this [link](https://learn.microsoft.com/en-GB/cpp/windows/latest-supported-vc-redist?view=msvc-170) and reboot your computer -4. Edit the `.template-config-private.yaml` accordingly and rename it to `config-private.yaml`. +4. Run the script with the following arguments: + ``` + python main.py -C + ``` -5. Edit the `accounts.json.sample` with your accounts credentials and rename it by removing - `.sample` at the end. +5. Open the generated `config.yaml` file and edit it with your information. The "totp" field is not mandatory, only enter your TOTP key if you use it for 2FA (if ommitting, don't keep it as an empty string, remove the line completely). @@ -57,24 +59,9 @@ The "proxy" field is not mandatory, you can omit it if you don't want to use proxy (don't keep it as an empty string, remove the line completely). - - If you want to add more than one account, the syntax is the following: - - ```json - [ - { - "username": "Your Email 1", - "password": "Your Password 1", - "totp": "0123 4567 89ab cdef", - "proxy": "http://user:pass@host1:port" - }, - { - "username": "Your Email 2", - "password": "Your Password 2", - "totp": "0123 4567 89ab cdef", - "proxy": "http://user:pass@host2:port" - } - ] - ``` + You can add or remove accounts according to your will. + + the "apprise.urls" field is not mandatory, you can remove it if you don't want to get notifications. 6. Run the script: @@ -90,22 +77,119 @@ To import the XML file into Task Scheduler, see [this guide](https://superuser.com/a/485565/709704). -## Launch arguments - -- `-v/--visible` to disable headless -- `-l/--lang` to force a language (ex: en) see https://serpapi.com/google-languages for options -- `-g/--geo` to force a searching geolocation (ex: US) - see https://serpapi.com/google-trends-locations for options - `https://trends.google.com/trends/ for proper geolocation abbreviation for your choice. These MUST be uppercase!!!` -- `-p/--proxy` to add a proxy to the whole program, supports http/https/socks4/socks5 ( - overrides per-account proxy in accounts.json) - `(ex: http://user:pass@host:port)` -- `-cv/--chromeversion` to use a specific version of chrome - `(ex: 118)` -- `-da/--disable-apprise` disables Apprise notifications for the session, - overriding [config.yaml](config.yaml). - Useful when running manually as opposed to on a schedule. -- `-t/--searchtype` to only do `desktop` or `mobile` searches, `(ex: --searchtype=mobile)` +## Configuration file + +All the variable listed here can be added to you `config.yaml` configuration file, and the values represented here show +the default ones, if not said otherwise. + +```yaml +# config.yaml +apprise: # 'apprise' is the name of the service used for notifications https://github.com/caronc/apprise + enabled: true # set it to false to disable apprise globally + notify: + incomplete-activity: true # set it to false to disable notifications for incomplete activities + uncaught-exception: true # set it to false to disable notifications for uncaught exceptions + login-code: true # set it to false to disable notifications for the temporary M$ Authenticator login code + summary: ON_ERROR # set it to ALWAYS to always receive a summary about your points progression or errors, or to + # NEVER to never receive a summary, even in case of an error. + urls: # add apprise urls here to receive notifications on the specified services : + # https://github.com/caronc/apprise#supported-notifications + # Empty by default. + - discord://{WebhookID}/{WebhookToken} # Exemple url +browser: + geolocation: US # Replace with your country code https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2 + language: en # Replace with your language code https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes + visible: false # set it to true to show the browser window + proxy: null # set the global proxy using the 'http://user:pass@host:port' syntax. + # Override per-account proxies. +activities: + ignore: # list of activities to ignore, like activities that can't be completed + - Get 50 entries plus 1000 points! + - Safeguard your family's info + search: # list of searches to do for search-based activities + "Black Friday shopping": black friday deals + "Discover open job roles": jobs at microsoft + "Expand your vocabulary": define demure + "Find places to stay": hotels rome italy + "Find somewhere new to explore": directions to new york + "Gaming time": vampire survivors video game + "Get your shopping done faster": new iphone + "Houses near you": apartments manhattan + "How's the economy?": sp 500 + "Learn to cook a new recipe": how cook pierogi + "Let's watch that movie again!": aliens movie + "Plan a quick getaway": flights nyc to paris + "Prepare for the weather": weather tomorrow + "Quickly convert your money": convert 374 usd to yen + "Search the lyrics of a song": black sabbath supernaut lyrics + "Stay on top of the elections": election news latest + "Too tired to cook tonight?": Pizza Hut near me + "Translate anything": translate pencil sharpener to spanish + "What time is it?": china time + "What's for Thanksgiving dinner?": pumpkin pie recipe + "Who won?": braves score + "You can track your package": usps tracking +logging: + level: INFO # Set to DEBUG, WARNING, ERROR or CRITICAL to change the level of displayed information in the terminal + # See https://docs.python.org/3/library/logging.html#logging-levels +retries: + base_delay_in_seconds: 120 # The base wait time between each retries. Multiplied by two each try. + max: 4 # The maximal number of retries to do + strategy: EXPONENTIAL # Set it to CONSTANT to use the same delay between each retries. + # Else, increase it exponentially each time. +cooldown: + min: 300 # The minimal wait time between two searches/activities + max: 600 # The maximal wait time between two searches/activities +search: + type: both # Set it to 'mobile' or 'desktop' to only complete searches on one plateform +accounts: # The accounts to use. You can put zero, one or an infinite number of accounts here. + # Empty by default. + - email: Your Email 1 # replace with your email + password: Your Password 1 # replace with your password + totp: 0123 4567 89ab cdef # replace with your totp, or remove it + proxy: http://user:pass@host1:port # replace with your account proxy, or remove it + - email: Your Email 2 # replace with your email + password: Your Password 2 # replace with your password + totp: 0123 4567 89ab cdef # replace with your totp, or remove it + proxy: http://user:pass@host2:port # replace with your account proxy, or remove it +``` + +## Usage + +``` +usage: main.py [-h] [-c CONFIG] [-C] [-v] [-l LANG] [-g GEO] [-em EMAIL] [-pw PASSWORD] + [-p PROXY] [-t {desktop,mobile,both}] [-da] [-d] + +A simple bot that uses Selenium to farm M$ Rewards in Python + +options: + -h, --help show this help message and exit + -c CONFIG, --config CONFIG + Specify the configuration file path + -C, --create-config Create a fillable configuration file with basic settings and given + ones if none exists + -v, --visible Visible browser (Disable headless mode) + -l LANG, --lang LANG Language (ex: en) see https://serpapi.com/google-languages for options + -g GEO, --geo GEO Searching geolocation (ex: US) see https://serpapi.com/google-trends- + locations for options (should be uppercase) + -em EMAIL, --email EMAIL + Email address of the account to run. Only used if a password is given. + -pw PASSWORD, --password PASSWORD + Password of the account to run. Only used if an email is given. + -p PROXY, --proxy PROXY + Global Proxy, supports http/https/socks4/socks5 (overrides config per- + account proxies) `(ex: http://user:pass@host:port)` + -t {desktop,mobile,both}, --searchtype {desktop,mobile,both} + Set to search in either desktop, mobile or both (default: both) + -da, --disable-apprise + Disable Apprise notifications, useful when developing + -d, --debug Set the logging level to DEBUG + +At least one account should be specified, either using command line arguments or a +configuration file. All specified arguments will override the configuration file values +``` + +You can display this message at any moment using `python main.py -h`. ## Features diff --git a/accounts.json.sample b/accounts.json.sample deleted file mode 100644 index 9aceda60..00000000 --- a/accounts.json.sample +++ /dev/null @@ -1,14 +0,0 @@ -[ - { - "username": "Your Email 1", - "password": "Your Password 1", - "totp": "0123 4567 89ab cdef", - "proxy": "http://user:pass@host1:port" - }, - { - "username": "Your Email 2", - "password": "Your Password 2", - "totp": "0123 4567 89ab cdef", - "proxy": "http://user:pass@host2:port" - } -] diff --git a/config.yaml b/config.yaml deleted file mode 100644 index d02856b6..00000000 --- a/config.yaml +++ /dev/null @@ -1,22 +0,0 @@ -# config.yaml -apprise: - notify: - incomplete-activity: - enabled: True # True or False - ignore: - - Get 50 entries plus 1000 points! - - Safeguard your family's info - uncaught-exception: - enabled: True # True or False - login-code: - enabled: True # True or False - summary: ON_ERROR -default: - geolocation: US # Replace with your country code https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2 - language: en # Replace with your language code https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes -logging: - level: INFO # See https://docs.python.org/3/library/logging.html#logging-levels -retries: - base_delay_in_seconds: 120 # base_delay_in_seconds * 2^max = 14.0625 * 2^6 = 900 = 15 minutes - max: 4 - strategy: EXPONENTIAL diff --git a/main.py b/main.py index 60053a0a..b05467be 100644 --- a/main.py +++ b/main.py @@ -1,11 +1,8 @@ -import argparse import csv import json import logging import logging.config import logging.handlers as handlers -import random -import re import sys import traceback from datetime import datetime @@ -17,35 +14,31 @@ PunchCards, Searches, ReadToEarn, - Account, ) from src.activities import Activities from src.browser import RemainingSearches from src.loggingColoredFormatter import ColoredFormatter -from src.utils import Utils, CONFIG, sendNotification, getProjectRoot, formatNumber +from src.utils import CONFIG, sendNotification, getProjectRoot, formatNumber def main(): - args = argumentParser() - Utils.args = args setupLogging() - loadedAccounts = setupAccounts() # Load previous day's points data previous_points_data = load_previous_points_data() - for currentAccount in loadedAccounts: + for currentAccount in CONFIG.accounts: try: - earned_points = executeBot(currentAccount, args) + earned_points = executeBot(currentAccount) except Exception as e1: logging.error("", exc_info=True) sendNotification( - f"⚠️ Error executing {currentAccount.username}, please check the log", + f"⚠️ Error executing {currentAccount.email}, please check the log", traceback.format_exc(), e1, ) continue - previous_points = previous_points_data.get(currentAccount.username, 0) + previous_points = previous_points_data.get(currentAccount.email, 0) # Calculate the difference in points from the prior day points_difference = earned_points - previous_points @@ -54,10 +47,10 @@ def main(): log_daily_points_to_csv(earned_points, points_difference) # Update the previous day's points data - previous_points_data[currentAccount.username] = earned_points + previous_points_data[currentAccount.email] = earned_points logging.info( - f"[POINTS] Data for '{currentAccount.username}' appended to the file." + f"[POINTS] Data for '{currentAccount.email}' appended to the file." ) # Save the current day's points data for the next day in the "logs" folder @@ -90,7 +83,7 @@ def log_daily_points_to_csv(earned_points, points_difference): def setupLogging(): - _format = "%(asctime)s [%(levelname)s] %(message)s" + _format = CONFIG.logging.format terminalHandler = logging.StreamHandler(sys.stdout) terminalHandler.setFormatter(ColoredFormatter(_format)) @@ -105,7 +98,7 @@ def setupLogging(): } ) logging.basicConfig( - level=logging.getLevelName(CONFIG.get("logging").get("level").upper()), + level=logging.getLevelName(CONFIG.logging.level.upper()), format=_format, handlers=[ handlers.TimedRotatingFileHandler( @@ -120,88 +113,6 @@ def setupLogging(): ) -def argumentParser() -> argparse.Namespace: - parser = argparse.ArgumentParser(description="MS Rewards Farmer") - parser.add_argument( - "-v", "--visible", action="store_true", help="Optional: Visible browser" - ) - parser.add_argument( - "-l", "--lang", type=str, default=None, help="Optional: Language (ex: en)" - ) - parser.add_argument( - "-g", "--geo", type=str, default=None, help="Optional: Geolocation (ex: US)" - ) - parser.add_argument( - "-p", - "--proxy", - type=str, - default=None, - help="Optional: Global Proxy (ex: http://user:pass@host:port)", - ) - parser.add_argument( - "-vn", - "--verbosenotifs", - action="store_true", - help="Optional: Send all the logs to the notification service", - ) - parser.add_argument( - "-cv", - "--chromeversion", - type=int, - default=None, - help="Optional: Set fixed Chrome version (ex. 118)", - ) - parser.add_argument( - "-da", - "--disable-apprise", - action="store_true", - help="Optional: Disable Apprise, overrides config.yaml, useful when developing", - ) - parser.add_argument( - "-t", - "--searchtype", - type=str, - default=None, - help="Optional: Set to only search in either desktop or mobile (ex: 'desktop' or 'mobile')", - ) - return parser.parse_args() - - -def setupAccounts() -> list[Account]: - """Sets up and validates a list of accounts loaded from 'accounts.json'.""" - - def validEmail(email: str) -> bool: - """Validate Email.""" - pattern = r"^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$" - return bool(re.match(pattern, email)) - - accountPath = getProjectRoot() / "accounts.json" - if not accountPath.exists(): - accountPath.write_text( - json.dumps( - [{"username": "Your Email", "password": "Your Password"}], indent=4 - ), - encoding="utf-8", - ) - noAccountsNotice = """ - [ACCOUNT] Accounts credential file "accounts.json" not found. - [ACCOUNT] A new file has been created, please edit with your credentials and save. - """ - logging.warning(noAccountsNotice) - exit(1) - loadedAccounts: list[Account] = [] - for rawAccount in json.loads(accountPath.read_text(encoding="utf-8")): - account: Account = Account(**rawAccount) - if not validEmail(account.username): - logging.warning( - f"[CREDENTIALS] Invalid email: {account.username}, skipping this account" - ) - continue - loadedAccounts.append(account) - random.shuffle(loadedAccounts) - return loadedAccounts - - class AppriseSummary(Enum): """ configures how results are summarized via Apprise @@ -221,8 +132,8 @@ class AppriseSummary(Enum): """ -def executeBot(currentAccount: Account, args: argparse.Namespace): - logging.info(f"********************{currentAccount.username}********************") +def executeBot(currentAccount): + logging.info(f"********************{currentAccount.email}********************") startingPoints: int | None = None accountPoints: int @@ -230,10 +141,10 @@ def executeBot(currentAccount: Account, args: argparse.Namespace): goalTitle: str goalPoints: int - if args.searchtype in ("desktop", None): - with Browser(mobile=False, account=currentAccount, args=args) as desktopBrowser: + if CONFIG.search.type in ("desktop", "both", None): + with Browser(mobile=False, account=currentAccount) as desktopBrowser: utils = desktopBrowser.utils - Login(desktopBrowser, args).login() + Login(desktopBrowser).login() startingPoints = utils.getAccountPoints() logging.info( f"[POINTS] You have {formatNumber(startingPoints)} points on your account" @@ -253,10 +164,10 @@ def executeBot(currentAccount: Account, args: argparse.Namespace): ) accountPoints = utils.getAccountPoints() - if args.searchtype in ("mobile", None): - with Browser(mobile=True, account=currentAccount, args=args) as mobileBrowser: + if CONFIG.search.type in ("mobile", "both", None): + with Browser(mobile=True, account=currentAccount) as mobileBrowser: utils = mobileBrowser.utils - Login(mobileBrowser, args).login() + Login(mobileBrowser).login() if startingPoints is None: startingPoints = utils.getAccountPoints() ReadToEarn(mobileBrowser).completeReadToEarn() @@ -275,7 +186,7 @@ def executeBot(currentAccount: Account, args: argparse.Namespace): f"[POINTS] You have earned {formatNumber(accountPoints - startingPoints)} points this run !" ) logging.info(f"[POINTS] You are now at {formatNumber(accountPoints)} points !") - appriseSummary = AppriseSummary[CONFIG.get("apprise").get("summary")] + appriseSummary = AppriseSummary[CONFIG.apprise.summary] if appriseSummary == AppriseSummary.ALWAYS: goalStatus = "" if goalPoints > 0: @@ -292,7 +203,7 @@ def executeBot(currentAccount: Account, args: argparse.Namespace): "Daily Points Update", "\n".join( [ - f"👤 Account: {currentAccount.username}", + f"👤 Account: {currentAccount.email}", f"⭐️ Points earned today: {formatNumber(accountPoints - startingPoints)}", f"💰 Total points: {formatNumber(accountPoints)}", goalStatus, @@ -303,7 +214,7 @@ def executeBot(currentAccount: Account, args: argparse.Namespace): if remainingSearches.getTotal() > 0: sendNotification( "Error: remaining searches", - f"account username: {currentAccount.username}, {remainingSearches}", + f"account email: {currentAccount.email}, {remainingSearches}", ) elif appriseSummary == AppriseSummary.NEVER: pass diff --git a/src/__init__.py b/src/__init__.py index 6f179194..9d6a2fe8 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -1,4 +1,3 @@ -from .account import Account from .remainingSearches import RemainingSearches from .browser import Browser from .login import Login diff --git a/src/account.py b/src/account.py deleted file mode 100644 index 3ec30515..00000000 --- a/src/account.py +++ /dev/null @@ -1,9 +0,0 @@ -from dataclasses import dataclass - - -@dataclass -class Account: - username: str - password: str - totp: str | None = None - proxy: str | None = None diff --git a/src/activities.py b/src/activities.py index 91a36a70..1a7d5453 100644 --- a/src/activities.py +++ b/src/activities.py @@ -11,32 +11,6 @@ from src.constants import REWARDS_URL from src.utils import CONFIG, sendNotification, getAnswerCode -# todo These are US-English specific, maybe there's a good way to internationalize -ACTIVITY_TITLE_TO_SEARCH = { - "Black Friday shopping": "black friday deals", - "Discover open job roles": "jobs at microsoft", - "Expand your vocabulary": "define demure", - "Find places to stay": "hotels rome italy", - "Find somewhere new to explore": "directions to new york", - "Gaming time": "vampire survivors video game", - "Get your shopping done faster": "new iphone", - "Houses near you": "apartments manhattan", - "How's the economy?": "sp 500", - "Learn to cook a new recipe": "how cook pierogi", - "Let's watch that movie again!": "aliens movie", - "Plan a quick getaway": "flights nyc to paris", - "Prepare for the weather": "weather tomorrow", - "Quickly convert your money": "convert 374 usd to yen", - "Search the lyrics of a song": "black sabbath supernaut lyrics", - "Stay on top of the elections": "election news latest", - "Too tired to cook tonight?": "Pizza Hut near me", - "Translate anything": "translate pencil sharpener to spanish", - "What time is it?": "china time", - "What's for Thanksgiving dinner?": "pumpkin pie recipe", - "Who won?": "braves score", - "You can track your package": "usps tracking", -} - class Activities: def __init__(self, browser: Browser): @@ -175,9 +149,7 @@ def doActivity(self, activity: dict, activities: list[dict]) -> None: if activity["complete"] is True or activity["pointProgressMax"] == 0: logging.debug("Already done, returning") return - if activityTitle in CONFIG.get("apprise").get("notify").get( - "incomplete-activity" - ).get("ignore"): + if activityTitle in CONFIG.activities.ignore: logging.debug(f"Ignoring {activityTitle}") return # Open the activity for the activity @@ -193,8 +165,8 @@ def doActivity(self, activity: dict, activities: list[dict]) -> None: with contextlib.suppress(TimeoutException): searchbar = self.browser.utils.waitUntilClickable(By.ID, "sb_form_q") self.browser.utils.click(searchbar) - if activityTitle in ACTIVITY_TITLE_TO_SEARCH: - searchbar.send_keys(ACTIVITY_TITLE_TO_SEARCH[activityTitle]) + if activityTitle in CONFIG.activities.search: + searchbar.send_keys(CONFIG.activities.search[activityTitle]) sleep(2) searchbar.submit() elif "poll" in activityTitle: @@ -217,8 +189,7 @@ def doActivity(self, activity: dict, activities: list[dict]) -> None: self.completeSearch() except Exception: logging.error(f"[ACTIVITY] Error doing {activityTitle}", exc_info=True) - # todo Make configurable - sleep(randint(300, 600)) + sleep(randint(CONFIG.cooldown.min, CONFIG.cooldown.max)) self.browser.utils.resetTabs() def completeActivities(self): @@ -238,12 +209,7 @@ def completeActivities(self): # todo Send one email for all accounts? # fixme This is falsely considering some activities incomplete when complete - if ( - CONFIG.get("apprise") - .get("notify") - .get("incomplete-activity") - .get("enabled") - ): + if CONFIG.get('apprise.notify.incomplete-activity'): incompleteActivities: dict[str, tuple[str, str, str]] = {} for activity in ( self.browser.utils.getDailySetPromotions() @@ -255,17 +221,12 @@ def completeActivities(self): activity["pointProgress"], activity["pointProgressMax"], ) - for incompleteActivityToIgnore in ( - CONFIG.get("apprise") - .get("notify") - .get("incomplete-activity") - .get("ignore") - ): + for incompleteActivityToIgnore in CONFIG.activities.ignore: incompleteActivities.pop(incompleteActivityToIgnore, None) if incompleteActivities: logging.info(f"incompleteActivities: {incompleteActivities}") sendNotification( - f"We found some incomplete activities for {self.browser.username}", + f"We found some incomplete activities for {self.browser.email}", str(incompleteActivities) + "\n" + REWARDS_URL, ) diff --git a/src/browser.py b/src/browser.py index 0ee94e3c..32000736 100644 --- a/src/browser.py +++ b/src/browser.py @@ -1,4 +1,3 @@ -import argparse import contextlib import locale import logging @@ -15,9 +14,8 @@ from ipapi.exceptions import RateLimited from selenium.webdriver import ChromeOptions from selenium.webdriver.chrome.webdriver import WebDriver -from selenium.webdriver.common.by import By -from src import Account, RemainingSearches +from src import RemainingSearches from src.userAgentGenerator import GenerateUserAgent from src.utils import CONFIG, Utils, getBrowserConfig, getProjectRoot, saveBrowserConfig @@ -28,21 +26,19 @@ class Browser: webdriver: undetected_chromedriver.Chrome def __init__( - self, mobile: bool, account: Account, args: argparse.Namespace + self, mobile: bool, account ) -> None: # Initialize browser instance logging.debug("in __init__") self.mobile = mobile self.browserType = "mobile" if mobile else "desktop" - self.headless = not args.visible - self.username = account.username + self.headless = not CONFIG.browser.visible + self.email = account.email self.password = account.password - self.totp = account.totp - self.localeLang, self.localeGeo = self.getLanguageCountry(args.lang, args.geo) - self.proxy = None - if args.proxy: - self.proxy = args.proxy - elif account.proxy: + self.totp = account.get('totp') + self.localeLang, self.localeGeo = self.getLanguageCountry(CONFIG.browser.language, CONFIG.browser.geolocation) + self.proxy = CONFIG.browser.proxy + if not self.proxy and account.get('proxy'): self.proxy = account.proxy self.userDataDir = self.setupProfiles() self.browserConfig = getBrowserConfig(self.userDataDir) @@ -206,15 +202,15 @@ def browserSetup( def setupProfiles(self) -> Path: """ Sets up the sessions profile for the chrome browser. - Uses the username to create a unique profile for the session. + Uses the email to create a unique profile for the session. Returns: Path """ sessionsDir = getProjectRoot() / "sessions" - # Concatenate username and browser type for a plain text session ID - sessionid = f"{self.username}" + # Concatenate email and browser type for a plain text session ID + sessionid = f"{self.email}" sessionsDir = sessionsDir / sessionid sessionsDir.mkdir(parents=True, exist_ok=True) @@ -223,10 +219,10 @@ def setupProfiles(self) -> Path: @staticmethod def getLanguageCountry(language: str, country: str) -> tuple[str, str]: if not country: - country = CONFIG.get("default").get("geolocation") + country = CONFIG.browser.geolocation if not language: - country = CONFIG.get("default").get("language") + country = CONFIG.browser.language if not language or not country: currentLocale = locale.getlocale() diff --git a/src/login.py b/src/login.py index edf8556e..adf48366 100644 --- a/src/login.py +++ b/src/login.py @@ -18,14 +18,12 @@ class Login: browser: Browser - args: Namespace webdriver: Chrome - def __init__(self, browser: Browser, args: argparse.Namespace): + def __init__(self, browser: Browser): self.browser = browser self.webdriver = browser.webdriver self.utils = browser.utils - self.args = args def check_locked_user(self): try: @@ -84,8 +82,8 @@ def execute_login(self) -> None: emailField = self.utils.waitUntilVisible(By.ID, "i0116") logging.info("[LOGIN] Entering email...") emailField.click() - emailField.send_keys(self.browser.username) - assert emailField.get_attribute("value") == self.browser.username + emailField.send_keys(self.browser.email) + assert emailField.get_attribute("value") == self.browser.email self.utils.waitUntilClickable(By.ID, "idSIButton9").click() # Passwordless check @@ -102,12 +100,7 @@ def execute_login(self) -> None: "[LOGIN] Confirm your login with code %s on your phone (you have one minute)!\a", codeField.text, ) - if ( - CONFIG.get("apprise") - .get("notify") - .get("login-code") - .get("enabled") - ): + if CONFIG.get("apprise.notify.login-code"): sendNotification( f"Confirm your login on your phone", f"Code: {codeField.text} (expires in 1 minute)") self.utils.waitUntilVisible(By.NAME, "kmsiForm", 60) @@ -149,12 +142,7 @@ def execute_login(self) -> None: " one minute)!\a", codeField.text, ) - if ( - CONFIG.get("apprise") - .get("notify") - .get("login-code") - .get("enabled") - ): + if CONFIG.get("apprise.notify.login-code"): sendNotification( f"Confirm your login on your phone", f"Code: {codeField.text} (expires in 1 minute)") self.utils.waitUntilVisible(By.NAME, "kmsiForm", 60) @@ -176,7 +164,7 @@ def execute_login(self) -> None: ).click() else: # TOTP token not provided, manual intervention required - assert self.args.visible, ( + assert CONFIG.browser.visible, ( "[LOGIN] 2FA detected, provide token in accounts.json or or run in" "[LOGIN] 2FA detected, provide token in accounts.json or handle manually." " visible mode to handle login." @@ -202,7 +190,7 @@ def execute_login(self) -> None: if isAskingToProtect: assert ( - self.args.visible + CONFIG.browser.visible ), "Account protection detected, run in visible mode to handle login" print( "Account protection detected, handle prompts and press enter when on rewards page" diff --git a/src/readToEarn.py b/src/readToEarn.py index 67979f6e..3f17d663 100644 --- a/src/readToEarn.py +++ b/src/readToEarn.py @@ -27,7 +27,7 @@ def completeReadToEarn(self): logging.info("[READ TO EARN] " + "Trying to complete Read to Earn...") - accountName = self.browser.username + accountName = self.browser.email # Should Really Cache Token and load it in. # To Save token diff --git a/src/searches.py b/src/searches.py index 0b2cdf00..d583ae93 100644 --- a/src/searches.py +++ b/src/searches.py @@ -32,16 +32,16 @@ class RetriesStrategy(Enum): class Searches: - maxRetries: Final[int] = CONFIG.get("retries").get("max") + maxRetries: Final[int] = CONFIG.retries.max """ the max amount of retries to attempt """ - baseDelay: Final[float] = CONFIG.get("retries").get("base_delay_in_seconds") + baseDelay: Final[float] = CONFIG.get("retries.base_delay_in_seconds") """ how many seconds to delay """ # retriesStrategy = Final[ # todo Figure why doesn't work with equality below - retriesStrategy = RetriesStrategy[CONFIG.get("retries").get("strategy")] + retriesStrategy = RetriesStrategy[CONFIG.retries.strategy] def __init__(self, browser: Browser): self.browser = browser @@ -182,8 +182,7 @@ def bingSearch(self) -> None: pointsAfter = self.browser.utils.getAccountPoints() if pointsBefore < pointsAfter: - # todo Make configurable - sleep(randint(300, 600)) + sleep(randint(CONFIG.cooldown.min, CONFIG.cooldown.max)) return # todo diff --git a/src/utils.py b/src/utils.py index c4cb4835..56283be5 100644 --- a/src/utils.py +++ b/src/utils.py @@ -4,11 +4,12 @@ import logging import re import time -from argparse import Namespace +from argparse import Namespace, ArgumentParser from datetime import date from pathlib import Path -from types import MappingProxyType -from typing import Any +import random +from typing import Any, Self +from copy import deepcopy import requests import yaml @@ -30,42 +31,202 @@ from .constants import REWARDS_URL, SEARCH_URL -DEFAULT_CONFIG: MappingProxyType = MappingProxyType( +class Config(dict): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + for key, value in self.items(): + if isinstance(value, dict): + self[key] = self.__class__(value) + if isinstance(value, list): + for i, v in enumerate(value): + if isinstance(v, dict): + value[i] = self.__class__(v) + + def __or__(self, other): + new = deepcopy(self) + for key in other: + if key in new: + if isinstance(new[key], dict) and isinstance(other[key], dict): + new[key] = new[key] | other[key] + continue + if isinstance(other[key], dict): + new[key] = self.__class__(other[key]) + continue + if isinstance(other[key], list): + new[key] = self.configifyList(other[key]) + continue + new[key] = other[key] + return new + + + def __getattribute__(self, item): + if item in self: + return self[item] + return super().__getattribute__(item) + + def __setattr__(self, key, value): + if type(value) is dict: + value = self.__class__(value) + if type(value) is list: + value = self.configifyList(value) + self[key] = value + + + def __getitem__(self, item): + if type(item) is not str or not '.' in item: + return super().__getitem__(item) + item: str + items = item.split(".") + found = super().__getitem__(items[0]) + for item in items[1:]: + found = found.__getitem__(item) + return found + + def __setitem__(self, key, value): + if type(value) is dict: + value = self.__class__(value) + if type(value) is list: + value = self.configifyList(value) + if type(key) is not str or not '.' in key: + return super().__setitem__(key, value) + item: str + items = key.split(".") + found = super().__getitem__(items[0]) + for item in items[1:-1]: + found = found.__getitem__(item) + found.__setitem__(items[-1], value) + + @classmethod + def fromYaml(cls, path: Path) -> Self: + if not path.exists() or not path.is_file(): + return cls() + with open(path, encoding="utf-8") as f: + yamlContents = yaml.safe_load(f) + if not yamlContents: + return cls() + return cls(yamlContents) + + + @classmethod + def configifyList(cls, listToConvert: list) -> list: + new = [None] * len(listToConvert) + for index, item in enumerate(listToConvert): + if isinstance(item, dict): + new[index] = cls(item) + continue + if isinstance(item, list): + new[index] = cls.configifyList(item) + continue + new[index] = item + return new + + @classmethod + def dictifyList(cls, listToConvert: list) -> list: + new = [None] * len(listToConvert) + for index, item in enumerate(listToConvert): + if isinstance(item, cls): + new[index] = item.toDict() + continue + if isinstance(item, list): + new[index] = cls.dictifyList(item) + continue + new[index] = item + return new + + + def get(self, key, default=None): + if type(key) is not str or not '.' in key: + return super().get(key, default) + item: str + keys = key.split(".") + found = super().get(keys[0], default) + for key in keys[1:]: + found = found.get(key, default) + return found + + def toDict(self) -> dict: + new = {} + for key, value in self.items(): + if isinstance(value, self.__class__): + new[key] = value.toDict() + continue + if isinstance(value, list): + new[key] = self.dictifyList(value) + continue + new[key] = value + return new + + +DEFAULT_CONFIG: Config = Config( { - "apprise": { - "notify": { - "incomplete-activity": { - "enabled": True, - "ignore": [ - "Get 50 entries plus 1000 points!", - "Safeguard your family's info", - ], - }, - "uncaught-exception": {"enabled": True}, - "login-code": {"enabled": True}, + 'apprise': { + 'enabled': True, + 'notify': { + 'incomplete-activity': True, + 'uncaught-exception': True, + 'login-code': True }, - "summary": "ON_ERROR", + 'summary': 'ON_ERROR', + 'urls': [] }, - "default": {"geolocation": "US"}, - "logging": {"level": "INFO"}, - "retries": { - "base_delay_in_seconds": 120, - "max": 4, - "strategy": "EXPONENTIAL", + 'browser': { + 'geolocation': 'US', + 'language': 'en', + 'visible': False, + 'proxy': None }, - } -) -DEFAULT_PRIVATE_CONFIG: MappingProxyType = MappingProxyType( - { - "apprise": { - "urls": [], + 'activities': { + 'ignore': [ + 'Get 50 entries plus 1000 points!', + "Safeguard your family's info" + ], + 'search': { + 'Black Friday shopping': 'black friday deals', + 'Discover open job roles': 'jobs at microsoft', + 'Expand your vocabulary': 'define demure', + 'Find places to stay': 'hotels rome italy', + 'Find somewhere new to explore': 'directions to new york', + 'Gaming time': 'vampire survivors video game', + 'Get your shopping done faster': 'new iphone', + 'Houses near you': 'apartments manhattan', + "How's the economy?": 'sp 500', + 'Learn to cook a new recipe': 'how cook pierogi', + "Let's watch that movie again!": 'aliens movie', + 'Plan a quick getaway': 'flights nyc to paris', + 'Prepare for the weather': 'weather tomorrow', + 'Quickly convert your money': 'convert 374 usd to yen', + 'Search the lyrics of a song': 'black sabbath supernaut lyrics', + 'Stay on top of the elections': 'election news latest', + 'Too tired to cook tonight?': 'Pizza Hut near me', + 'Translate anything': 'translate pencil sharpener to spanish', + 'What time is it?': 'china time', + "What's for Thanksgiving dinner?": 'pumpkin pie recipe', + 'Who won?': 'braves score', + 'You can track your package': 'usps tracking' + } + }, + 'logging': { + 'format': '%(asctime)s [%(levelname)s] %(message)s', + 'level': 'INFO' + }, + 'retries': { + 'base_delay_in_seconds': 120, + 'max': 4, + 'strategy': 'EXPONENTIAL' + }, + 'cooldown': { + 'min': 300, + 'max': 600 + }, + 'search': { + 'type': 'both' }, + 'accounts': [] } ) class Utils: - args: Namespace def __init__(self, webdriver: WebDriver): self.webdriver = webdriver @@ -232,45 +393,234 @@ def click(self, element: WebElement) -> None: element.click() +def argumentParser() -> Namespace: + parser = ArgumentParser( + description="A simple bot that uses Selenium to farm M$ Rewards in Python", + epilog="At least one account should be specified, either using command line arguments or a configuration file." + "\nAll specified arguments will override the configuration file values." + ) + parser.add_argument( + "-c", + "--config", + type=str, + default=None, + help="Specify the configuration file path", + ) + parser.add_argument( + "-C", + "--create-config", + action="store_true", + help="Create a fillable configuration file with basic settings and given ones if none exists", + ) + parser.add_argument( + "-v", + "--visible", + action="store_true", + help="Visible browser (Disable headless mode)", + ) + parser.add_argument( + "-l", + "--lang", + type=str, + default=None, + help="Language (ex: en)" + "\nsee https://serpapi.com/google-languages for options" + ) + parser.add_argument( + "-g", + "--geo", + type=str, + default=None, + help="Searching geolocation (ex: US)" + "\nsee https://serpapi.com/google-trends-locations for options (should be uppercase)" + ) + parser.add_argument( + "-em", + "--email", + type=str, + default=None, + help="Email address of the account to run. Only used if a password is given.", + ) + parser.add_argument( + "-pw", + "--password", + type=str, + default=None, + help="Password of the account to run. Only used if an email is given.", + ) + parser.add_argument( + "-p", + "--proxy", + type=str, + default=None, + help="Global Proxy, supports http/https/socks4/socks5 (overrides config per-account proxies)" + "\n`(ex: http://user:pass@host:port)`", + ) + parser.add_argument( + "-t", + "--searchtype", + choices=['desktop', 'mobile', 'both'], + default=None, + help="Set to search in either desktop, mobile or both (default: both)", + ) + parser.add_argument( + "-da", + "--disable-apprise", + action="store_true", + help="Disable Apprise notifications, useful when developing", + ) + parser.add_argument( + "-d", + "--debug", + action="store_true", + help="Set the logging level to DEBUG", + ) + return parser.parse_args() + + def getProjectRoot() -> Path: return Path(__file__).parent.parent -def loadYaml(path: Path) -> dict: - with open(path, "r") as file: - yamlContents = yaml.safe_load(file) - if not yamlContents: - logging.info(f"{yamlContents} is empty") - yamlContents = {} - return yamlContents +def commandLineArgumentsAsConfig(args: Namespace) -> Config: + config = Config() + if args.visible: + config.browser = Config() + config.browser.visible = True + if args.lang: + if not 'browser' in config: + config.browser = Config() + config.browser.language = args.lang + if args.geo: + if not 'browser' in config: + config.browser = Config() + config.browser.geolocation = args.geo + if args.proxy: + if not 'browser' in config: + config.browser = Config() + config.browser.proxy = args.proxy + if args.disable_apprise: + config.apprise = Config() + config.apprise.enabled = False + if args.debug: + config.logging = Config() + config.logging.level = 'DEBUG' + if args.searchtype: + config.search = Config() + config.search.type = args.searchtype + if args.email and args.password: + config.accounts = [Config( + email=args.email, + password=args.password, + )] + + return config + + +def setupAccounts(config: Config) -> Config: + def validEmail(email: str) -> bool: + """Validate Email.""" + pattern = r"^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$" + return bool(re.match(pattern, email)) + + loadedAccounts = [] + for account in config.accounts: + if ( + not 'email' in account + or not isinstance(account.email, str) + or not validEmail(account.email) + ): + logging.warning( + f"[CREDENTIALS] Invalid email '{account.get('email', 'No email provided')}'," + f" skipping this account" + ) + continue + if not 'password' in account or not isinstance(account['password'], str): + logging.warning( + f"[CREDENTIALS] Invalid password '{account.get('password', 'No password provided')}'," + f" skipping this account" + ) + loadedAccounts.append(account) + + if not loadedAccounts: + noAccountsNotice = """ + [ACCOUNT] No valid account provided. + [ACCOUNT] Please provide a valid account, either using command line arguments or a configuration file. + [ACCOUNT] For command line, please use the following arguments (change the email and password): + [ACCOUNT] `--email youremail@domain.com --password yourpassword` + [ACCOUNT] For configuration file, please generate a configuration file using the `-C` argument, + [ACCOUNT] then edit the generated file by replacing the email and password using yours. + """ + logging.error(noAccountsNotice) + exit(1) + + random.shuffle(loadedAccounts) + config.accounts = loadedAccounts + return config + +def createEmptyConfig(configPath: Path, config: Config) -> None: + if configPath.is_file(): + logging.error( + f"[CONFIG] A file already exists at '{configPath}'" + ) + exit(1) + + emptyConfig = Config( + { + 'apprise': { + 'urls': ['discord://{WebhookID}/{WebhookToken}'] + }, + 'accounts': [ + { + 'email': 'Your Email 1', + 'password': 'Your Password 1', + 'totp': '0123 4567 89ab cdef', + 'proxy': 'http://user:pass@host1:port' + }, + { + 'email': 'Your Email 2', + 'password': 'Your Password 2', + 'totp': '0123 4567 89ab cdef', + 'proxy': 'http://user:pass@host2:port' + } + ] + } + ) + with open(configPath, "w", encoding="utf-8") as configFile: + yaml.dump((emptyConfig | config).toDict(), configFile) + logging.info( + f"[CONFIG] A configuration file was created at '{configPath}'" + ) + exit(0) def loadConfig( configFilename="config.yaml", defaultConfig=DEFAULT_CONFIG -) -> MappingProxyType: - configFile = getProjectRoot() / configFilename - try: - return MappingProxyType(defaultConfig | loadYaml(configFile)) - except OSError: - logging.info(f"{configFile} doesn't exist, returning defaults") - return defaultConfig +) -> Config: + args = argumentParser() + if args.config: + configFile = Path(args.config) + else: + configFile = getProjectRoot() / configFilename + + args_config = commandLineArgumentsAsConfig(args) + + if args.create_config: + createEmptyConfig(configFile, args_config) + config = defaultConfig | Config.fromYaml(configFile) | args_config + config = setupAccounts(config) -def loadPrivateConfig() -> MappingProxyType: - return loadConfig("config-private.yaml", DEFAULT_PRIVATE_CONFIG) + return config def sendNotification(title: str, body: str, e: Exception = None) -> None: - if Utils.args.disable_apprise or ( - e - and not CONFIG.get("apprise") - .get("notify") - .get("uncaught-exception") - .get("enabled") + if not CONFIG.apprise.enabled or ( + e and not CONFIG.get("apprise.notify.uncaught-exception") ): return apprise = Apprise() - urls: list[str] = PRIVATE_CONFIG.get("apprise").get("urls") + urls: list[str] = CONFIG.apprise.urls if not urls: logging.debug("No urls found, not sending notification") return @@ -305,14 +655,14 @@ def saveBrowserConfig(sessionPath: Path, config: dict) -> None: def makeRequestsSession(session: Session = requests.session()) -> Session: retry = Retry( - total=5, + total=CONFIG.retries.max, backoff_factor=1, status_forcelist=[ 500, 502, 503, 504, - ], # todo Use global retries from config + ], ) session.mount( "https://", HTTPAdapter(max_retries=retry) @@ -324,4 +674,3 @@ def makeRequestsSession(session: Session = requests.session()) -> Session: CONFIG = loadConfig() -PRIVATE_CONFIG = loadPrivateConfig() diff --git a/test/test_main.py b/test/test_main.py index 5081dbd7..5df13c49 100644 --- a/test/test_main.py +++ b/test/test_main.py @@ -2,6 +2,8 @@ from unittest.mock import patch, MagicMock import main +from src import utils +from src.utils import Config, CONFIG class TestMain(unittest.TestCase): @@ -9,18 +11,16 @@ class TestMain(unittest.TestCase): # noinspection PyUnusedLocal @patch.object(main, "save_previous_points_data") @patch.object(main, "setupLogging") - @patch.object(main, "setupAccounts") @patch.object(main, "executeBot") - # @patch.object(Utils, "send_notification") + # @patch.object(utils, "sendNotification") def test_send_notification_when_exception( self, # mock_send_notification: MagicMock, mock_executeBot: MagicMock, - mock_setupAccounts: MagicMock, mock_setupLogging: MagicMock, mock_save_previous_points_data: MagicMock, ): - mock_setupAccounts.return_value = [{"password": "foo", "username": "bar"}] + CONFIG.accounts = [Config({"password": "foo", "email": "bar"})] mock_executeBot.side_effect = Exception main.main() diff --git a/test/test_utils.py b/test/test_utils.py index da529a3b..a359a437 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -1,11 +1,9 @@ -from argparse import Namespace from unittest import TestCase -from src.utils import Utils, sendNotification +from src.utils import CONFIG, sendNotification class TestUtils(TestCase): def test_send_notification(self): - Utils.args = Namespace() - Utils.args.disable_apprise = False + CONFIG.apprise.enabled = True sendNotification("title", "body") From a3d2aac17a6497da7bde30521a4eda71948b77c0 Mon Sep 17 00:00:00 2001 From: Kyrela Date: Wed, 25 Dec 2024 11:15:19 +0100 Subject: [PATCH 81/82] Enhance README.md with command-line override details Updated the configuration documentation to clarify that several settings can now be overridden through command-line arguments. This improves flexibility and provides clearer guidance for users to customize behavior. --- README.md | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 57610876..e9b43b43 100644 --- a/README.md +++ b/README.md @@ -85,23 +85,25 @@ the default ones, if not said otherwise. ```yaml # config.yaml apprise: # 'apprise' is the name of the service used for notifications https://github.com/caronc/apprise - enabled: true # set it to false to disable apprise globally + enabled: true # set it to false to disable apprise globally, can be overridden with command-line arguments. notify: incomplete-activity: true # set it to false to disable notifications for incomplete activities uncaught-exception: true # set it to false to disable notifications for uncaught exceptions login-code: true # set it to false to disable notifications for the temporary M$ Authenticator login code summary: ON_ERROR # set it to ALWAYS to always receive a summary about your points progression or errors, or to - # NEVER to never receive a summary, even in case of an error. + # NEVER to never receive a summary, even in case of an error. urls: # add apprise urls here to receive notifications on the specified services : - # https://github.com/caronc/apprise#supported-notifications - # Empty by default. + # https://github.com/caronc/apprise#supported-notifications + # Empty by default. - discord://{WebhookID}/{WebhookToken} # Exemple url browser: - geolocation: US # Replace with your country code https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2 - language: en # Replace with your language code https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes - visible: false # set it to true to show the browser window + geolocation: US # Replace with your country code https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2. + # Can be overridden with command-line arguments. + language: en # Replace with your language code https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes. + # Can be overridden with command-line arguments. + visible: false # set it to true to show the browser window, can be overridden with command-line arguments. proxy: null # set the global proxy using the 'http://user:pass@host:port' syntax. - # Override per-account proxies. + # Override per-account proxies. Can be overridden with command-line arguments. activities: ignore: # list of activities to ignore, like activities that can't be completed - Get 50 entries plus 1000 points! @@ -131,19 +133,20 @@ activities: "You can track your package": usps tracking logging: level: INFO # Set to DEBUG, WARNING, ERROR or CRITICAL to change the level of displayed information in the terminal - # See https://docs.python.org/3/library/logging.html#logging-levels + # See https://docs.python.org/3/library/logging.html#logging-levels. Can be overridden with command-line arguments. retries: base_delay_in_seconds: 120 # The base wait time between each retries. Multiplied by two each try. max: 4 # The maximal number of retries to do strategy: EXPONENTIAL # Set it to CONSTANT to use the same delay between each retries. - # Else, increase it exponentially each time. + # Else, increase it exponentially each time. cooldown: min: 300 # The minimal wait time between two searches/activities max: 600 # The maximal wait time between two searches/activities search: - type: both # Set it to 'mobile' or 'desktop' to only complete searches on one plateform + type: both # Set it to 'mobile' or 'desktop' to only complete searches on one plateform, + # can be overridden with command-line arguments. accounts: # The accounts to use. You can put zero, one or an infinite number of accounts here. - # Empty by default. + # Empty by default, can be overridden with command-line arguments. - email: Your Email 1 # replace with your email password: Your Password 1 # replace with your password totp: 0123 4567 89ab cdef # replace with your totp, or remove it From e88f6eedb22bbac7ab0aa109f9d9713f1a7b1483 Mon Sep 17 00:00:00 2001 From: Kyrela Date: Wed, 22 Jan 2025 14:05:26 +0100 Subject: [PATCH 82/82] Fix unwanted changed on #237 - Apprise notifications disable reverted - `get_value_ignore_case` renamed to `getValueIgnoreCase` to preserve consistency --- src/userAgentGenerator.py | 16 ++++++++-------- src/utils.py | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/userAgentGenerator.py b/src/userAgentGenerator.py index 1696657a..31895a0f 100644 --- a/src/userAgentGenerator.py +++ b/src/userAgentGenerator.py @@ -140,7 +140,7 @@ def getEdgeVersions(self) -> tuple[str, str]: "https://edgeupdates.microsoft.com/api/products" ) - def get_value_ignore_case(data: dict, key: str) -> Any: + def getValueIgnoreCase(data: dict, key: str) -> Any: """Get the value from a dictionary ignoring the case of the first letter of the key.""" for k, v in data.items(): if k.lower() == key.lower(): @@ -152,16 +152,16 @@ def get_value_ignore_case(data: dict, key: str) -> Any: ( product for product in data - if get_value_ignore_case(product, "product") == "Stable" + if getValueIgnoreCase(product, "product") == "Stable" ), None, ): - releases = get_value_ignore_case(stableProduct, "releases") + releases = getValueIgnoreCase(stableProduct, "releases") androidRelease = next( ( release for release in releases - if get_value_ignore_case(release, "platform") == "Android" + if getValueIgnoreCase(release, "platform") == "Android" ), None, ) @@ -169,15 +169,15 @@ def get_value_ignore_case(data: dict, key: str) -> Any: ( release for release in releases - if get_value_ignore_case(release, "platform") == "Windows" - and get_value_ignore_case(release, "architecture") == "x64" + if getValueIgnoreCase(release, "platform") == "Windows" + and getValueIgnoreCase(release, "architecture") == "x64" ), None, ) if androidRelease and windowsRelease: return ( - get_value_ignore_case(windowsRelease, "productVersion"), - get_value_ignore_case(androidRelease, "productVersion"), + getValueIgnoreCase(windowsRelease, "productVersion"), + getValueIgnoreCase(androidRelease, "productVersion"), ) raise HTTPError("Failed to get Edge versions.") diff --git a/src/utils.py b/src/utils.py index 56283be5..b0f5cab8 100644 --- a/src/utils.py +++ b/src/utils.py @@ -626,7 +626,7 @@ def sendNotification(title: str, body: str, e: Exception = None) -> None: return for url in urls: apprise.add(url) - # assert apprise.notify(title=str(title), body=str(body)) # not work for telegram + assert apprise.notify(title=str(title), body=str(body)) def getAnswerCode(key: str, string: str) -> str: