From d61ea534e283b3d450cf0c89b0b8465f76e77a47 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Fri, 7 Jun 2024 22:28:55 -0400 Subject: [PATCH 01/74] Lots of refactoring and fixes --- config.yaml | 5 ++ main.py | 184 +++++++++++++++++++++++------------------- requirements.txt | 2 +- src/__init__.py | 7 +- src/account.py | 9 +++ src/activities.py | 30 +++---- src/browser.py | 24 +++--- src/dailySet.py | 3 +- src/login.py | 8 +- src/morePromotions.py | 2 +- src/punchCards.py | 2 +- src/searches.py | 156 ++++++++++++++++++++--------------- src/utils.py | 69 +++++++++------- test/__init__.py | 0 test/test_main.py | 32 ++++++++ 15 files changed, 316 insertions(+), 217 deletions(-) create mode 100644 src/account.py create mode 100644 test/__init__.py create mode 100644 test/test_main.py diff --git a/config.yaml b/config.yaml index 508b63e9..88a273ba 100644 --- a/config.yaml +++ b/config.yaml @@ -1,4 +1,9 @@ # config.yaml apprise: + summary: on_error urls: - 'discord://WebhookID/WebhookToken' # Replace with your actual Apprise service URLs +attempts: + base_delay_in_seconds: 60 + max: 6 + strategy: exponential diff --git a/main.py b/main.py index b9ec158f..3cc97d9d 100644 --- a/main.py +++ b/main.py @@ -3,28 +3,27 @@ import csv import json import logging +import logging.config import logging.handlers as handlers import random import re import sys -import time from datetime import datetime -from pathlib import Path +from enum import Enum, auto import psutil from src import ( Browser, - DailySet, Login, MorePromotions, PunchCards, Searches, + DailySet, + Account, ) from src.loggingColoredFormatter import ColoredFormatter -from src.utils import Utils - -POINTS_COUNTER = 0 +from src.utils import Utils, RemainingSearches def main(): @@ -40,22 +39,28 @@ def main(): for currentAccount in loadedAccounts: try: earned_points = executeBot(currentAccount, args) - account_name = currentAccount.get("username", "") - previous_points = previous_points_data.get(account_name, 0) + previous_points = previous_points_data.get(currentAccount.username, 0) # Calculate the difference in points from the prior day points_difference = earned_points - previous_points # Append the daily points and points difference to CSV and Excel - log_daily_points_to_csv(account_name, earned_points, points_difference) + log_daily_points_to_csv( + currentAccount.username, earned_points, points_difference + ) # Update the previous day's points data - previous_points_data[account_name] = earned_points + previous_points_data[currentAccount.username] = earned_points - logging.info(f"[POINTS] Data for '{account_name}' appended to the file.") + logging.info( + f"[POINTS] Data for '{currentAccount.username}' appended to the file." + ) except Exception as e: - Utils.send_notification("⚠️ Error occurred, please check the log", str(e)) + Utils.sendNotification( + "⚠️ Error occurred, please check the log", f"{e}\n{e.__traceback__}" + ) logging.exception(f"{e.__class__.__name__}: {e}") + exit(1) # Save the current day's points data for the next day in the "logs" folder save_previous_points_data(previous_points_data) @@ -63,7 +68,7 @@ def main(): def log_daily_points_to_csv(date, earned_points, points_difference): - logs_directory = Path(__file__).resolve().parent / "logs" + logs_directory = Utils.getProjectRoot() / "logs" csv_filename = logs_directory / "points_data.csv" # Create a new row with the date, daily points, and points difference @@ -87,16 +92,24 @@ def log_daily_points_to_csv(date, earned_points, points_difference): def setupLogging(): - format = "%(asctime)s [%(levelname)s] %(message)s" + _format = "%(asctime)s [%(levelname)s] %(message)s" terminalHandler = logging.StreamHandler(sys.stdout) - terminalHandler.setFormatter(ColoredFormatter(format)) + terminalHandler.setFormatter(ColoredFormatter(_format)) - logs_directory = Path(__file__).resolve().parent / "logs" + logs_directory = Utils.getProjectRoot() / "logs" logs_directory.mkdir(parents=True, exist_ok=True) + # so only our code is logged if level=logging.DEBUG or finer + # if not working see https://stackoverflow.com/a/48891485/4164390 + logging.config.dictConfig( + { + "version": 1, + "disable_existing_loggers": True, + } + ) logging.basicConfig( - level=logging.INFO, - format=format, + level=logging.DEBUG, + format=_format, handlers=[ handlers.TimedRotatingFileHandler( logs_directory / "activity.log", @@ -116,7 +129,7 @@ def cleanupChromeProcesses(): if process.info["name"] == "chrome.exe": try: psutil.Process(process.info["pid"]).terminate() - except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess): + except (psutil.NoSuchProcess, psutil.AccessDenied): pass @@ -154,7 +167,7 @@ def argumentParser() -> argparse.Namespace: return parser.parse_args() -def setupAccounts() -> list: +def setupAccounts() -> list[Account]: """Sets up and validates a list of accounts loaded from 'accounts.json'.""" def validEmail(email: str) -> bool: @@ -162,7 +175,7 @@ def validEmail(email: str) -> bool: pattern = r"^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$" return bool(re.match(pattern, email)) - accountPath = Path(__file__).resolve().parent / "accounts.json" + accountPath = Utils.getProjectRoot() / "accounts.json" if not accountPath.exists(): accountPath.write_text( json.dumps( @@ -175,24 +188,30 @@ def validEmail(email: str) -> bool: [ACCOUNT] A new file has been created, please edit with your credentials and save. """ logging.warning(noAccountsNotice) - exit() - loadedAccounts = json.loads(accountPath.read_text(encoding="utf-8")) - for account in loadedAccounts: - if not validEmail(account["username"]): - logging.error(f"[CREDENTIALS] Wrong Email Address: '{account['username']}'") - exit() + exit(1) + loadedAccounts: list[Account] = [] + for rawAccount in json.loads(accountPath.read_text(encoding="utf-8")): + account: Account = Account(**rawAccount) + if not validEmail(account.username): + logging.warning( + f"[CREDENTIALS] Invalid email: {account.username}, skipping this account" + ) + continue + loadedAccounts.append(account) random.shuffle(loadedAccounts) return loadedAccounts -def executeBot(currentAccount, args: argparse.Namespace): - logging.info( - f"********************{currentAccount.get('username', '')}********************" - ) - +class AppriseSummary(Enum): + always = auto() + on_error = auto() + + +def executeBot(currentAccount: Account, args: argparse.Namespace): + logging.info(f"********************{currentAccount.username}********************") + accountPointsCounter = 0 - remainingSearches = 0 - remainingSearchesM = 0 + remainingSearches: RemainingSearches startingPoints = 0 with Browser(mobile=False, account=currentAccount, args=args) as desktopBrowser: @@ -200,88 +219,85 @@ def executeBot(currentAccount, args: argparse.Namespace): accountPointsCounter = Login(desktopBrowser).login() startingPoints = accountPointsCounter if startingPoints == "Locked": - Utils.send_notification("🚫 Account is Locked", currentAccount["username"]) + Utils.sendNotification("🚫 Account is Locked", currentAccount.username) return 0 if startingPoints == "Verify": - Utils.send_notification("❗️ Account needs to be verified", currentAccount["username"]) + Utils.sendNotification( + "❗️ Account needs to be verified", currentAccount.username + ) return 0 logging.info( f"[POINTS] You have {utils.formatNumber(accountPointsCounter)} points on your account" ) + # todo - make quicker if done DailySet(desktopBrowser).completeDailySet() PunchCards(desktopBrowser).completePunchCards() MorePromotions(desktopBrowser).completeMorePromotions() # VersusGame(desktopBrowser).completeVersusGame() - ( - remainingSearches, - remainingSearchesM, - ) = utils.getRemainingSearches() - - # Introduce random pauses before and after searches - pause_before_search = random.uniform( - 11.0, 15.0 - ) # Random pause between 11 to 15 seconds - time.sleep(pause_before_search) - - if remainingSearches != 0: - accountPointsCounter = Searches(desktopBrowser).bingSearches( - remainingSearches - ) + utils.goHome() + remainingSearches = utils.getRemainingSearches() - pause_after_search = random.uniform( - 11.0, 15.0 - ) # Random pause between 11 to 15 seconds - time.sleep(pause_after_search) + if remainingSearches.desktop != 0: + accountPointsCounter = Searches( + desktopBrowser, remainingSearches + ).bingSearches(remainingSearches.desktop) utils.goHome() goalPoints = utils.getGoalPoints() goalTitle = utils.getGoalTitle() - desktopBrowser.closeBrowser() - if remainingSearchesM != 0: - desktopBrowser.closeBrowser() + if remainingSearches.mobile != 0: with Browser(mobile=True, account=currentAccount, args=args) as mobileBrowser: utils = mobileBrowser.utils - accountPointsCounter = Login(mobileBrowser).login() - accountPointsCounter = Searches(mobileBrowser).bingSearches( - remainingSearchesM - ) + Login(mobileBrowser).login() + accountPointsCounter = Searches( + mobileBrowser, remainingSearches + ).bingSearches(remainingSearches.mobile) utils.goHome() goalPoints = utils.getGoalPoints() goalTitle = utils.getGoalTitle() - mobileBrowser.closeBrowser() + + remainingSearches = utils.getRemainingSearches() logging.info( - f"[POINTS] You have earned {utils.formatNumber(accountPointsCounter - startingPoints)} points today !" + f"[POINTS] You have earned {utils.formatNumber(accountPointsCounter - startingPoints)} points this run !" ) logging.info( f"[POINTS] You are now at {utils.formatNumber(accountPointsCounter)} points !" ) - goalNotifier = "" - if goalPoints > 0: - logging.info( - f"[POINTS] You are now at {(utils.formatNumber((accountPointsCounter / goalPoints) * 100))}% of your goal ({goalTitle}) !\n" + appriseSummary = AppriseSummary[utils.config["apprise"]["summary"]] + if appriseSummary == AppriseSummary.always: + goalNotifier = "" + if goalPoints > 0: + logging.info( + f"[POINTS] You are now at {(utils.formatNumber((accountPointsCounter / goalPoints) * 100))}% of your goal ({goalTitle}) !\n" + ) + goalNotifier = f"🎯 Goal reached: {(utils.formatNumber((accountPointsCounter / goalPoints) * 100))}% ({goalTitle})" + + Utils.sendNotification( + "Daily Points Update", + "\n".join( + [ + f"👤 Account: {currentAccount.username}", + f"⭐️ Points earned today: {utils.formatNumber(accountPointsCounter - startingPoints)}", + f"💰 Total points: {utils.formatNumber(accountPointsCounter)}", + goalNotifier, + ] + ), ) - goalNotifier = f"🎯 Goal reached: {(utils.formatNumber((accountPointsCounter / goalPoints) * 100))}% ({goalTitle})" - - Utils.send_notification( - "Daily Points Update", - "\n".join( - [ - f"👤 Account: {currentAccount.get('username')}", - f"⭐️ Points earned today: {utils.formatNumber(accountPointsCounter - startingPoints)}", - f"💰 Total points: {utils.formatNumber(accountPointsCounter)}", - goalNotifier, - ] - ), - ) + elif appriseSummary == AppriseSummary.on_error: + if remainingSearches.desktop > 0 or remainingSearches.mobile > 0: + Utils.sendNotification( + "Error: remaining searches", + f"account username: {currentAccount.username}, {remainingSearches}", + ) return accountPointsCounter def export_points_to_csv(points_data): - logs_directory = Path(__file__).resolve().parent / "logs" + logs_directory = Utils.getProjectRoot() / "logs" csv_filename = logs_directory / "points_data.csv" with open(csv_filename, mode="a", newline="") as file: # Use "a" mode for append fieldnames = ["Account", "Earned Points", "Points Difference"] @@ -297,7 +313,7 @@ def export_points_to_csv(points_data): # Define a function to load the previous day's points data from a file in the "logs" folder def load_previous_points_data(): - logs_directory = Path(__file__).resolve().parent / "logs" + logs_directory = Utils.getProjectRoot() / "logs" try: with open(logs_directory / "previous_points_data.json", "r") as file: return json.load(file) @@ -307,7 +323,7 @@ def load_previous_points_data(): # Define a function to save the current day's points data for the next day in the "logs" folder def save_previous_points_data(data): - logs_directory = Path(__file__).resolve().parent / "logs" + logs_directory = Utils.getProjectRoot() / "logs" with open(logs_directory / "previous_points_data.json", "w") as file: json.dump(data, file, indent=4) diff --git a/requirements.txt b/requirements.txt index 042c60a5..65b37b60 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,6 +6,6 @@ selenium-wire numpy>=1.22.2 # not directly required, pinned by Snyk to avoid a vulnerability setuptools>=69.0.2 # not directly required, pinned by Snyk to avoid a vulnerability psutil -blinker==1.7.0 #prevents issues on newer versions +blinker==1.7.0 # prevents issues on newer versions apprise pyyaml diff --git a/src/__init__.py b/src/__init__.py index 095110f2..d3f5a12f 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -1,6 +1 @@ -from .browser import Browser -from .dailySet import DailySet -from .login import Login -from .morePromotions import MorePromotions -from .punchCards import PunchCards -from .searches import Searches + diff --git a/src/account.py b/src/account.py new file mode 100644 index 00000000..9889ee61 --- /dev/null +++ b/src/account.py @@ -0,0 +1,9 @@ +from dataclasses import dataclass +from typing import Optional + + +@dataclass +class Account: + username: str + password: str + proxy: Optional[str] = None diff --git a/src/activities.py b/src/activities.py index 47bc14e8..bb1343c8 100644 --- a/src/activities.py +++ b/src/activities.py @@ -4,7 +4,6 @@ from selenium.webdriver.common.by import By from src.browser import Browser -from src.utils import Utils class Activities: @@ -30,13 +29,14 @@ def openMorePromotionsActivity(self, cardId: int): def completeSearch(self): # Simulate completing a search activity - time.sleep(Utils.randomSeconds(10, 15)) + time.sleep(random.randint(10, 15)) self.browser.utils.closeCurrentTab() def completeSurvey(self): # Simulate completing a survey activity + # noinspection SpellCheckingInspection self.webdriver.find_element(By.ID, f"btoption{random.randint(0, 1)}").click() - time.sleep(Utils.randomSeconds(10, 15)) + time.sleep(random.randint(10, 15)) self.browser.utils.closeCurrentTab() def completeQuiz(self): @@ -48,7 +48,7 @@ def completeQuiz(self): self.browser.utils.waitUntilVisible( By.XPATH, '//*[@id="currentQuestionContainer"]/div/div[1]', 5 ) - time.sleep(Utils.randomSeconds(10, 15)) + time.sleep(random.randint(10, 15)) numberOfQuestions = self.webdriver.execute_script( "return _w.rewardsQuizRenderInfo.maxQuestions" ) @@ -66,7 +66,7 @@ def completeQuiz(self): answers.append(f"rqAnswerOption{i}") for answer in answers: self.webdriver.find_element(By.ID, answer).click() - time.sleep(Utils.randomSeconds(10, 15)) + time.sleep(random.randint(10, 15)) if not self.browser.utils.waitUntilQuestionRefresh(): self.browser.utils.resetTabs() return @@ -82,14 +82,14 @@ def completeQuiz(self): == correctOption ): self.webdriver.find_element(By.ID, f"rqAnswerOption{i}").click() - time.sleep(Utils.randomSeconds(10, 15)) + time.sleep(random.randint(10, 15)) if not self.browser.utils.waitUntilQuestionRefresh(): self.browser.utils.resetTabs() return break if question + 1 != numberOfQuestions: - time.sleep(Utils.randomSeconds(10, 15)) - time.sleep(Utils.randomSeconds(10, 15)) + time.sleep(random.randint(10, 15)) + time.sleep(random.randint(10, 15)) self.browser.utils.closeCurrentTab() def completeABC(self): @@ -102,10 +102,10 @@ def completeABC(self): self.webdriver.find_element( By.ID, f"questionOptionChoice{question}{random.randint(0, 2)}" ).click() - time.sleep(Utils.randomSeconds(10, 15)) + time.sleep(random.randint(10, 15)) self.webdriver.find_element(By.ID, f"nextQuestionbtn{question}").click() - time.sleep(Utils.randomSeconds(10, 15)) - time.sleep(Utils.randomSeconds(1, 7)) + time.sleep(random.randint(10, 15)) + time.sleep(random.randint(1, 7)) self.browser.utils.closeCurrentTab() def completeThisOrThat(self): @@ -117,7 +117,7 @@ def completeThisOrThat(self): self.browser.utils.waitUntilVisible( By.XPATH, '//*[@id="currentQuestionContainer"]/div/div[1]', 10 ) - time.sleep(Utils.randomSeconds(10, 15)) + time.sleep(random.randint(10, 15)) for _ in range(10): correctAnswerCode = self.webdriver.execute_script( "return _w.rewardsQuizRenderInfo.correctAnswer" @@ -126,12 +126,12 @@ def completeThisOrThat(self): answer2, answer2Code = self.getAnswerAndCode("rqAnswerOption1") if answer1Code == correctAnswerCode: answer1.click() - time.sleep(Utils.randomSeconds(10, 15)) + time.sleep(random.randint(10, 15)) elif answer2Code == correctAnswerCode: answer2.click() - time.sleep(Utils.randomSeconds(10, 15)) + time.sleep(random.randint(10, 15)) - time.sleep(Utils.randomSeconds(10, 15)) + time.sleep(random.randint(10, 15)) self.browser.utils.closeCurrentTab() def getAnswerAndCode(self, answerId: str) -> tuple: diff --git a/src/browser.py b/src/browser.py index fa838c1e..de9d9593 100644 --- a/src/browser.py +++ b/src/browser.py @@ -9,6 +9,7 @@ from selenium.webdriver import ChromeOptions from selenium.webdriver.chrome.webdriver import WebDriver +from src import Account from src.userAgentGenerator import GenerateUserAgent from src.utils import Utils @@ -16,19 +17,19 @@ class Browser: """WebDriver wrapper class.""" - def __init__(self, mobile: bool, account, args: Any) -> None: + def __init__(self, mobile: bool, account: Account, args: Any) -> None: # Initialize browser instance self.mobile = mobile self.browserType = "mobile" if mobile else "desktop" self.headless = not args.visible - self.username = account["username"] - self.password = account["password"] + self.username = account.username + self.password = account.password self.localeLang, self.localeGeo = self.getCCodeLang(args.lang, args.geo) self.proxy = None if args.proxy: self.proxy = args.proxy - elif account.get("proxy"): - self.proxy = account["proxy"] + elif account.proxy: + self.proxy = account.proxy self.userDataDir = self.setupProfiles() self.browserConfig = Utils.getBrowserConfig(self.userDataDir) ( @@ -54,7 +55,7 @@ def closeBrowser(self) -> None: # Close the web browser with contextlib.suppress(Exception): self.webdriver.close() - + def browserSetup( self, ) -> WebDriver: @@ -173,9 +174,7 @@ def setupProfiles(self) -> Path: Returns: Path """ - currentPath = Path(__file__) - parent = currentPath.parent.parent - sessionsDir = parent / "sessions" + sessionsDir = Utils.getProjectRoot() / "sessions" # Concatenate username and browser type for a plain text session ID sessionid = f"{self.username}" @@ -184,7 +183,8 @@ def setupProfiles(self) -> Path: sessionsDir.mkdir(parents=True, exist_ok=True) return sessionsDir - def getCCodeLang(self, lang: str, geo: str) -> tuple: + @staticmethod + def getCCodeLang(lang: str, geo: str) -> tuple: if lang is None or geo is None: try: nfo = ipapi.location() @@ -194,8 +194,8 @@ def getCCodeLang(self, lang: str, geo: str) -> tuple: if geo is None: geo = nfo["country"] except Exception: # pylint: disable=broad-except - return ("en", "US") - return (lang, geo) + return "en", "US" + return lang, geo def getChromeVersion(self) -> str: chrome_options = ChromeOptions() diff --git a/src/dailySet.py b/src/dailySet.py index 6ec6fd37..b8f15272 100644 --- a/src/dailySet.py +++ b/src/dailySet.py @@ -3,7 +3,6 @@ from datetime import datetime from src.browser import Browser - from .activities import Activities @@ -82,9 +81,11 @@ def completeDailySet(self): # Try completing ABC activity self.activities.completeABC() except Exception: # pylint: disable=broad-except + logging.exception(Exception) # Default to completing quiz self.activities.completeQuiz() except Exception: # pylint: disable=broad-except + logging.exception(Exception) # Reset tabs in case of an exception self.browser.utils.resetTabs() logging.info("[DAILY SET] Completed the Daily Set successfully !") diff --git a/src/login.py b/src/login.py index 4dde1d6e..e611990c 100644 --- a/src/login.py +++ b/src/login.py @@ -14,7 +14,7 @@ def __init__(self, browser: Browser): self.webdriver = browser.webdriver self.utils = browser.utils - def login(self): + def login(self) -> int: logging.info("[LOGIN] " + "Logging-in...") self.webdriver.get( "https://rewards.bing.com/Signin/" @@ -28,10 +28,12 @@ def login(self): alreadyLoggedIn = True break except Exception: # pylint: disable=broad-except + logging.exception(Exception) try: self.utils.waitUntilVisible(By.ID, "i0116", 10) break except Exception: # pylint: disable=broad-except + logging.exception(Exception) if self.utils.tryDismissAllMessages(): continue @@ -69,12 +71,12 @@ def executeLogin(self): try: self.enterPassword(self.browser.password) except Exception: # pylint: disable=broad-except - logging.error("[LOGIN] " + "2FA Code required !") + logging.info("[LOGIN] " + "2FA Code required !") with contextlib.suppress(Exception): code = self.webdriver.find_element( By.ID, "idRemoteNGC_DisplaySign" ).get_attribute("innerHTML") - logging.error(f"[LOGIN] 2FA code: {code}") + logging.info(f"[LOGIN] 2FA code: {code}") logging.info("[LOGIN] Press enter when confirmed on your device...") input() diff --git a/src/morePromotions.py b/src/morePromotions.py index c676ae08..0787ebe6 100644 --- a/src/morePromotions.py +++ b/src/morePromotions.py @@ -1,7 +1,6 @@ import logging from src.browser import Browser - from .activities import Activities @@ -43,6 +42,7 @@ def completeMorePromotions(self): # Default to completing search self.activities.completeSearch() except Exception: # pylint: disable=broad-except + logging.exception(Exception) # Reset tabs in case of an exception self.browser.utils.resetTabs() logging.info("[MORE PROMOS] Completed More Promotions successfully !") diff --git a/src/punchCards.py b/src/punchCards.py index 24e192b9..eb63ac8b 100644 --- a/src/punchCards.py +++ b/src/punchCards.py @@ -7,7 +7,6 @@ from selenium.webdriver.common.by import By from src.browser import Browser - from .constants import BASE_URL @@ -73,6 +72,7 @@ def completePunchCards(self): punchCard["childPromotions"], ) except Exception: # pylint: disable=broad-except + logging.exception(Exception) self.browser.utils.resetTabs() logging.info("[PUNCH CARDS] Completed the Punch Cards successfully !") time.sleep(random.randint(100, 700) / 100) diff --git a/src/searches.py b/src/searches.py index 2ed4510e..0e2f3024 100644 --- a/src/searches.py +++ b/src/searches.py @@ -3,22 +3,54 @@ import random import time from datetime import date, timedelta +from enum import Enum, auto +from itertools import cycle import requests from selenium.common.exceptions import TimeoutException from selenium.webdriver.common.by import By -from selenium.webdriver.common.keys import Keys +from selenium.webdriver.remote.webelement import WebElement from src.browser import Browser -from src.utils import Utils +from src.utils import Utils, RemainingSearches + + +class AttemptsStrategy(Enum): + exponential = auto() + constant = auto() + + +DEFAULT_ATTEMPTS_MAX = 3 +DEFAULT_BASE_DELAY = 900 +DEFAULT_ATTEMPTS_STRATEGY = AttemptsStrategy.exponential.name class Searches: - def __init__(self, browser: Browser): + config = Utils.loadConfig() + # todo get rid of duplication, if possible + maxAttempts: int = config.get("attempts", DEFAULT_ATTEMPTS_MAX).get( + "max", DEFAULT_ATTEMPTS_MAX + ) + baseDelay: int = config.get("attempts", DEFAULT_BASE_DELAY).get( + "base_delay_in_seconds", DEFAULT_BASE_DELAY + ) + attemptsStrategy = AttemptsStrategy[ + config.get("attempts", DEFAULT_ATTEMPTS_STRATEGY).get( + "strategy", DEFAULT_ATTEMPTS_STRATEGY + ) + ] + searchTerms: list[str] = None + + def __init__(self, browser: Browser, searches: RemainingSearches): self.browser = browser self.webdriver = browser.webdriver + if Searches.searchTerms is None: + Searches.searchTerms = self.getGoogleTrends( + searches.desktop + searches.mobile + ) + random.shuffle(Searches.searchTerms) - def getGoogleTrends(self, wordsCount: int) -> list: + def getGoogleTrends(self, wordsCount: int) -> list[str]: # Function to retrieve Google Trends search terms searchTerms: list[str] = [] i = 0 @@ -41,7 +73,7 @@ def getGoogleTrends(self, wordsCount: int) -> list: del searchTerms[wordsCount : (len(searchTerms) + 1)] return searchTerms - def getRelatedTerms(self, word: str) -> list: + def getRelatedTerms(self, word: str) -> list[str]: # Function to retrieve related terms from Bing API try: r = requests.get( @@ -50,6 +82,7 @@ def getRelatedTerms(self, word: str) -> list: ) return r.json()[1] except Exception: # pylint: disable=broad-except + logging.warn(Exception) return [] def bingSearches(self, numberOfSearches: int, pointsCounter: int = 0): @@ -58,74 +91,71 @@ def bingSearches(self, numberOfSearches: int, pointsCounter: int = 0): f"[BING] Starting {self.browser.browserType.capitalize()} Edge Bing searches..." ) - search_terms = self.getGoogleTrends(numberOfSearches) self.webdriver.get("https://bing.com") - i = 0 - attempt = 0 - for word in search_terms: - i += 1 - logging.info(f"[BING] {i}/{numberOfSearches}") - points = self.bingSearch(word) - if points <= pointsCounter: - relatedTerms = self.getRelatedTerms(word)[:2] - for term in relatedTerms: - points = self.bingSearch(term) - if not points <= pointsCounter: - break - if points > 0: - pointsCounter = points - else: - break - - if points <= pointsCounter: - attempt += 1 - if attempt == 2: - logging.warning( - "[BING] Possible blockage. Refreshing the page." - ) - self.webdriver.refresh() - attempt = 0 + for searchCount in range(1, numberOfSearches + 1): + logging.info(f"[BING] {searchCount}/{numberOfSearches}") + searchTerm = Searches.searchTerms[0] + pointsCounter = self.bingSearch(searchTerm) + Searches.searchTerms.remove(searchTerm) + if not Utils.isDebuggerAttached(): + time.sleep(random.randint(10, 15)) + logging.info( f"[BING] Finished {self.browser.browserType.capitalize()} Edge Bing searches !" ) return pointsCounter - def bingSearch(self, word: str): + def bingSearch(self, word: str) -> int: # Function to perform a single Bing search - i = 0 + bingAccountPointsBefore: int = self.browser.utils.getBingAccountPoints() + + wordsCycle: cycle[str] = cycle(self.getRelatedTerms(word)) + baseDelay = Searches.baseDelay - while True: + for i in range(self.maxAttempts): try: - self.browser.utils.waitUntilClickable(By.ID, "sb_form_q") - searchbar = self.webdriver.find_element(By.ID, "sb_form_q") - searchbar.clear() - searchbar.send_keys(word) + searchbar: WebElement + for _ in range(100): + self.browser.utils.waitUntilClickable(By.ID, "sb_form_q") + searchbar = self.webdriver.find_element(By.ID, "sb_form_q") + searchbar.clear() + word = next(wordsCycle) + logging.debug(f"word={word}") + searchbar.send_keys(word) + typed_word = searchbar.get_attribute("value") + if typed_word == word: + break + logging.debug(f"typed_word != word, {typed_word} != {word}") + self.browser.webdriver.refresh() + else: + raise Exception("Problem sending words to searchbar") + searchbar.submit() - time.sleep(Utils.randomSeconds(100, 180)) - - # Scroll down after the search (adjust the number of scrolls as needed) - for _ in range(3): # Scroll down 3 times - self.webdriver.execute_script( - "window.scrollTo(0, document.body.scrollHeight);" - ) - time.sleep( - Utils.randomSeconds(7, 10) - ) # Random wait between scrolls - - return self.browser.utils.getBingAccountPoints() + time.sleep(random.randint(5, 15)) # wait a bit for search to complete + + bingAccountPointsNow: int = self.browser.utils.getBingAccountPoints() + if bingAccountPointsNow > bingAccountPointsBefore: + return bingAccountPointsNow + + raise TimeoutException + except TimeoutException: - if i == 5: - logging.info("[BING] " + "TIMED OUT GETTING NEW PROXY") - self.webdriver.proxy = self.browser.giveMeProxy() - elif i == 10: - logging.error( - "[BING] " - + "Cancelling mobile searches due to too many retries." - ) - return self.browser.utils.getBingAccountPoints() + # todo + # if i == (maxAttempts / 2): + # logging.info("[BING] " + "TIMED OUT GETTING NEW PROXY") + # self.webdriver.proxy = self.browser.giveMeProxy() self.browser.utils.tryDismissAllMessages() - logging.error("[BING] " + "Timeout, retrying in 5~ seconds...") - time.sleep(Utils.randomSeconds(7, 15)) - i += 1 - continue + + baseDelay += random.randint(1, 10) # add some jitter + logging.debug( + f"[BING] Search attempt failed {i + 1}/{Searches.maxAttempts}, retrying after sleeping {baseDelay}" + f" seconds..." + ) + if not Utils.isDebuggerAttached(): + time.sleep(baseDelay) + + if Searches.attemptsStrategy == AttemptsStrategy.exponential: + baseDelay *= 2 + logging.error("[BING] Reached max search attempt retries") + return bingAccountPointsBefore diff --git a/src/utils.py b/src/utils.py index 776745ed..87592c86 100644 --- a/src/utils.py +++ b/src/utils.py @@ -1,43 +1,52 @@ import contextlib import json import locale as pylocale -import random +import sys import time import urllib.parse from pathlib import Path +from typing import NamedTuple import requests +import yaml +from apprise import Apprise from selenium.webdriver.chrome.webdriver import WebDriver from selenium.webdriver.common.by import By from selenium.webdriver.support import expected_conditions as ec from selenium.webdriver.support.wait import WebDriverWait -import apprise -import yaml - from .constants import BASE_URL +class RemainingSearches(NamedTuple): + desktop: int + mobile: int + + class Utils: - def __init__(self, webdriver: WebDriver, config_file='config.yaml'): + def __init__(self, webdriver: WebDriver): self.webdriver = webdriver with contextlib.suppress(Exception): locale = pylocale.getdefaultlocale()[0] pylocale.setlocale(pylocale.LC_NUMERIC, locale) - - self.config = self.load_config(config_file) + + self.config = self.loadConfig() + + @staticmethod + def getProjectRoot() -> Path: + return Path(__file__).parent.parent @staticmethod - def load_config(config_file): - with open(config_file, 'r') as file: + def loadConfig(config_file=getProjectRoot() / "config.yaml"): + with open(config_file, "r") as file: return yaml.safe_load(file) @staticmethod - def send_notification(title, body, config_file='config.yaml'): - apobj = apprise.Apprise() - for url in Utils.load_config(config_file)['apprise']['urls']: - apobj.add(url) - apobj.notify(body=body, title=title) + def sendNotification(title, body): + apprise = Apprise() + for url in Utils.loadConfig()["apprise"]["urls"]: + apprise.add(url) + apprise.notify(body=body, title=title) def waitUntilVisible(self, by: str, selector: str, timeToWait: float = 10): WebDriverWait(self.webdriver, timeToWait).until( @@ -51,7 +60,7 @@ def waitUntilClickable(self, by: str, selector: str, timeToWait: float = 10): def waitForMSRewardElement(self, by: str, selector: str): loadingTimeAllowed = 5 - refreshsAllowed = 5 + refreshesAllowed = 5 checkingInterval = 0.5 checks = loadingTimeAllowed / checkingInterval @@ -66,7 +75,7 @@ def waitForMSRewardElement(self, by: str, selector: str): if tries < checks: tries += 1 time.sleep(checkingInterval) - elif refreshCount < refreshsAllowed: + elif refreshCount < refreshesAllowed: self.webdriver.refresh() refreshCount += 1 tries = 0 @@ -82,7 +91,7 @@ def waitUntilQuizLoads(self): def waitUntilJS(self, jsSrc: str): loadingTimeAllowed = 5 - refreshsAllowed = 5 + refreshesAllowed = 5 checkingInterval = 0.5 checks = loadingTimeAllowed / checkingInterval @@ -97,7 +106,7 @@ def waitUntilJS(self, jsSrc: str): if tries < checks: tries += 1 time.sleep(checkingInterval) - elif refreshCount < refreshsAllowed: + elif refreshCount < refreshesAllowed: self.webdriver.refresh() refreshCount += 1 tries = 0 @@ -152,12 +161,14 @@ def goHome(self): if reloads >= reloadThreshold: break - def getAnswerCode(self, key: str, string: str) -> str: + @staticmethod + def getAnswerCode(key: str, string: str) -> str: t = sum(ord(string[i]) for i in range(len(string))) t += int(key[-2:], 16) return str(t) def getDashboardData(self) -> dict: + self.goHome() return self.webdriver.execute_script("return dashboard") def getBingInfo(self): @@ -202,7 +213,7 @@ def tryDismissAllMessages(self): (By.ID, "idSIButton9"), (By.CSS_SELECTOR, ".ms-Button.ms-Button--primary"), (By.ID, "bnp_btn_accept"), - (By.ID, "acceptButton") + (By.ID, "acceptButton"), ] result = False for button in buttons: @@ -246,14 +257,11 @@ def visitNewTab(self, timeToWait: int = 0): self.switchToNewTab(timeToWait) self.closeCurrentTab() - def getRemainingSearches(self): + def getRemainingSearches(self) -> RemainingSearches: dashboard = self.getDashboardData() searchPoints = 1 counters = dashboard["userStatus"]["counters"] - if "pcSearch" not in counters: - return 0, 0 - progressDesktop = counters["pcSearch"][0]["pointProgress"] targetDesktop = counters["pcSearch"][0]["pointProgressMax"] if len(counters["pcSearch"]) >= 2: @@ -269,17 +277,14 @@ def getRemainingSearches(self): progressMobile = counters["mobileSearch"][0]["pointProgress"] targetMobile = counters["mobileSearch"][0]["pointProgressMax"] remainingMobile = int((targetMobile - progressMobile) / searchPoints) - return remainingDesktop, remainingMobile + return RemainingSearches(desktop=remainingDesktop, mobile=remainingMobile) - def formatNumber(self, number, num_decimals=2): + @staticmethod + def formatNumber(number, num_decimals=2): return pylocale.format_string( f"%10.{num_decimals}f", number, grouping=True ).strip() - def randomSeconds(self, max_value): - random_number = random.uniform(self, max_value) - return round(random_number, 3) - @staticmethod def getBrowserConfig(sessionPath: Path) -> dict: configFile = sessionPath.joinpath("config.json") @@ -294,3 +299,7 @@ def saveBrowserConfig(sessionPath: Path, config: dict): configFile = sessionPath.joinpath("config.json") with open(configFile, "w") as f: json.dump(config, f) + + @staticmethod + def isDebuggerAttached() -> bool: + return sys.gettrace() is not None \ No newline at end of file diff --git a/test/__init__.py b/test/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/test_main.py b/test/test_main.py new file mode 100644 index 00000000..5081dbd7 --- /dev/null +++ b/test/test_main.py @@ -0,0 +1,32 @@ +import unittest +from unittest.mock import patch, MagicMock + +import main + + +class TestMain(unittest.TestCase): + + # noinspection PyUnusedLocal + @patch.object(main, "save_previous_points_data") + @patch.object(main, "setupLogging") + @patch.object(main, "setupAccounts") + @patch.object(main, "executeBot") + # @patch.object(Utils, "send_notification") + def test_send_notification_when_exception( + self, + # mock_send_notification: MagicMock, + mock_executeBot: MagicMock, + mock_setupAccounts: MagicMock, + mock_setupLogging: MagicMock, + mock_save_previous_points_data: MagicMock, + ): + mock_setupAccounts.return_value = [{"password": "foo", "username": "bar"}] + mock_executeBot.side_effect = Exception + + main.main() + + # mock_send_notification.assert_called() + + +if __name__ == "__main__": + unittest.main() From bcfb74dcff6f6acc19f94c576171507157638939 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Fri, 7 Jun 2024 22:36:08 -0400 Subject: [PATCH 02/74] Fix double random sleep --- src/searches.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/searches.py b/src/searches.py index 0e2f3024..53e5660d 100644 --- a/src/searches.py +++ b/src/searches.py @@ -132,7 +132,7 @@ def bingSearch(self, word: str) -> int: raise Exception("Problem sending words to searchbar") searchbar.submit() - time.sleep(random.randint(5, 15)) # wait a bit for search to complete + time.sleep(2) # wait a bit for search to complete bingAccountPointsNow: int = self.browser.utils.getBingAccountPoints() if bingAccountPointsNow > bingAccountPointsBefore: From 6a8aa33a21442a2be5e71618054426d4717b0224 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Sat, 8 Jun 2024 09:47:02 -0400 Subject: [PATCH 03/74] Add todo --- src/searches.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/searches.py b/src/searches.py index 53e5660d..5944b718 100644 --- a/src/searches.py +++ b/src/searches.py @@ -116,7 +116,7 @@ def bingSearch(self, word: str) -> int: for i in range(self.maxAttempts): try: searchbar: WebElement - for _ in range(100): + for _ in range(100): # todo make configurable self.browser.utils.waitUntilClickable(By.ID, "sb_form_q") searchbar = self.webdriver.find_element(By.ID, "sb_form_q") searchbar.clear() From 7e55b7195a9a2215e1bc644101ad692fa32e3f61 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Sat, 8 Jun 2024 10:22:21 -0400 Subject: [PATCH 04/74] Fix __init__.py --- src/__init__.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/__init__.py b/src/__init__.py index d3f5a12f..057991c6 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -1 +1,7 @@ - +from .account import Account +from .browser import Browser +from .dailySet import DailySet +from .login import Login +from .morePromotions import MorePromotions +from .punchCards import PunchCards +from .searches import Searches From 9c04e0a1ab334c1c9ec52a7a8663b166ad5b2821 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Sat, 8 Jun 2024 10:24:48 -0400 Subject: [PATCH 05/74] Share some useful config --- .idea/inspectionProfiles/Project_Default.xml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 .idea/inspectionProfiles/Project_Default.xml diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml new file mode 100644 index 00000000..debf80d0 --- /dev/null +++ b/.idea/inspectionProfiles/Project_Default.xml @@ -0,0 +1,14 @@ + + + + \ No newline at end of file From 3922471e4a68863a4fbb7573258f8f75d05f31e2 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Sat, 8 Jun 2024 10:26:36 -0400 Subject: [PATCH 06/74] Default logging back to info --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index 3cc97d9d..ab9f282c 100644 --- a/main.py +++ b/main.py @@ -108,7 +108,7 @@ def setupLogging(): } ) logging.basicConfig( - level=logging.DEBUG, + level=logging.INFO, format=_format, handlers=[ handlers.TimedRotatingFileHandler( From ef87aaeab4a1ef41653d1db63dccb98f7769cb5b Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Sun, 9 Jun 2024 11:31:03 -0400 Subject: [PATCH 07/74] Fix bug opening headless mobile browser, other improvements --- main.py | 669 ++++++++++++++++++++++++------------------------ src/browser.py | 16 +- src/searches.py | 10 +- 3 files changed, 352 insertions(+), 343 deletions(-) diff --git a/main.py b/main.py index ab9f282c..cee42900 100644 --- a/main.py +++ b/main.py @@ -1,332 +1,337 @@ -import argparse -import atexit -import csv -import json -import logging -import logging.config -import logging.handlers as handlers -import random -import re -import sys -from datetime import datetime -from enum import Enum, auto - -import psutil - -from src import ( - Browser, - Login, - MorePromotions, - PunchCards, - Searches, - DailySet, - Account, -) -from src.loggingColoredFormatter import ColoredFormatter -from src.utils import Utils, RemainingSearches - - -def main(): - args = argumentParser() - setupLogging() - loadedAccounts = setupAccounts() - # Register the cleanup function to be called on script exit - atexit.register(cleanupChromeProcesses) - - # Load previous day's points data - previous_points_data = load_previous_points_data() - - for currentAccount in loadedAccounts: - try: - earned_points = executeBot(currentAccount, args) - previous_points = previous_points_data.get(currentAccount.username, 0) - - # Calculate the difference in points from the prior day - points_difference = earned_points - previous_points - - # Append the daily points and points difference to CSV and Excel - log_daily_points_to_csv( - currentAccount.username, earned_points, points_difference - ) - - # Update the previous day's points data - previous_points_data[currentAccount.username] = earned_points - - logging.info( - f"[POINTS] Data for '{currentAccount.username}' appended to the file." - ) - except Exception as e: - Utils.sendNotification( - "⚠️ Error occurred, please check the log", f"{e}\n{e.__traceback__}" - ) - logging.exception(f"{e.__class__.__name__}: {e}") - exit(1) - - # Save the current day's points data for the next day in the "logs" folder - save_previous_points_data(previous_points_data) - logging.info("[POINTS] Data saved for the next day.") - - -def log_daily_points_to_csv(date, earned_points, points_difference): - logs_directory = Utils.getProjectRoot() / "logs" - csv_filename = logs_directory / "points_data.csv" - - # Create a new row with the date, daily points, and points difference - date = datetime.now().strftime("%Y-%m-%d") - new_row = { - "Date": date, - "Earned Points": earned_points, - "Points Difference": points_difference, - } - - fieldnames = ["Date", "Earned Points", "Points Difference"] - is_new_file = not csv_filename.exists() - - with open(csv_filename, mode="a", newline="") as file: - writer = csv.DictWriter(file, fieldnames=fieldnames) - - if is_new_file: - writer.writeheader() - - writer.writerow(new_row) - - -def setupLogging(): - _format = "%(asctime)s [%(levelname)s] %(message)s" - terminalHandler = logging.StreamHandler(sys.stdout) - terminalHandler.setFormatter(ColoredFormatter(_format)) - - logs_directory = Utils.getProjectRoot() / "logs" - logs_directory.mkdir(parents=True, exist_ok=True) - - # so only our code is logged if level=logging.DEBUG or finer - # if not working see https://stackoverflow.com/a/48891485/4164390 - logging.config.dictConfig( - { - "version": 1, - "disable_existing_loggers": True, - } - ) - logging.basicConfig( - level=logging.INFO, - format=_format, - handlers=[ - handlers.TimedRotatingFileHandler( - logs_directory / "activity.log", - when="midnight", - interval=1, - backupCount=2, - encoding="utf-8", - ), - terminalHandler, - ], - ) - - -def cleanupChromeProcesses(): - # Use psutil to find and terminate Chrome processes - for process in psutil.process_iter(["pid", "name"]): - if process.info["name"] == "chrome.exe": - try: - psutil.Process(process.info["pid"]).terminate() - except (psutil.NoSuchProcess, psutil.AccessDenied): - pass - - -def argumentParser() -> argparse.Namespace: - parser = argparse.ArgumentParser(description="MS Rewards Farmer") - parser.add_argument( - "-v", "--visible", action="store_true", help="Optional: Visible browser" - ) - parser.add_argument( - "-l", "--lang", type=str, default=None, help="Optional: Language (ex: en)" - ) - parser.add_argument( - "-g", "--geo", type=str, default=None, help="Optional: Geolocation (ex: US)" - ) - parser.add_argument( - "-p", - "--proxy", - type=str, - default=None, - help="Optional: Global Proxy (ex: http://user:pass@host:port)", - ) - parser.add_argument( - "-vn", - "--verbosenotifs", - action="store_true", - help="Optional: Send all the logs to the notification service", - ) - parser.add_argument( - "-cv", - "--chromeversion", - type=int, - default=None, - help="Optional: Set fixed Chrome version (ex. 118)", - ) - return parser.parse_args() - - -def setupAccounts() -> list[Account]: - """Sets up and validates a list of accounts loaded from 'accounts.json'.""" - - def validEmail(email: str) -> bool: - """Validate Email.""" - pattern = r"^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$" - return bool(re.match(pattern, email)) - - accountPath = Utils.getProjectRoot() / "accounts.json" - if not accountPath.exists(): - accountPath.write_text( - json.dumps( - [{"username": "Your Email", "password": "Your Password"}], indent=4 - ), - encoding="utf-8", - ) - noAccountsNotice = """ - [ACCOUNT] Accounts credential file "accounts.json" not found. - [ACCOUNT] A new file has been created, please edit with your credentials and save. - """ - logging.warning(noAccountsNotice) - exit(1) - loadedAccounts: list[Account] = [] - for rawAccount in json.loads(accountPath.read_text(encoding="utf-8")): - account: Account = Account(**rawAccount) - if not validEmail(account.username): - logging.warning( - f"[CREDENTIALS] Invalid email: {account.username}, skipping this account" - ) - continue - loadedAccounts.append(account) - random.shuffle(loadedAccounts) - return loadedAccounts - - -class AppriseSummary(Enum): - always = auto() - on_error = auto() - - -def executeBot(currentAccount: Account, args: argparse.Namespace): - logging.info(f"********************{currentAccount.username}********************") - - accountPointsCounter = 0 - remainingSearches: RemainingSearches - startingPoints = 0 - - with Browser(mobile=False, account=currentAccount, args=args) as desktopBrowser: - utils = desktopBrowser.utils - accountPointsCounter = Login(desktopBrowser).login() - startingPoints = accountPointsCounter - if startingPoints == "Locked": - Utils.sendNotification("🚫 Account is Locked", currentAccount.username) - return 0 - if startingPoints == "Verify": - Utils.sendNotification( - "❗️ Account needs to be verified", currentAccount.username - ) - return 0 - logging.info( - f"[POINTS] You have {utils.formatNumber(accountPointsCounter)} points on your account" - ) - # todo - make quicker if done - DailySet(desktopBrowser).completeDailySet() - PunchCards(desktopBrowser).completePunchCards() - MorePromotions(desktopBrowser).completeMorePromotions() - # VersusGame(desktopBrowser).completeVersusGame() - utils.goHome() - remainingSearches = utils.getRemainingSearches() - - if remainingSearches.desktop != 0: - accountPointsCounter = Searches( - desktopBrowser, remainingSearches - ).bingSearches(remainingSearches.desktop) - - utils.goHome() - goalPoints = utils.getGoalPoints() - goalTitle = utils.getGoalTitle() - - if remainingSearches.mobile != 0: - with Browser(mobile=True, account=currentAccount, args=args) as mobileBrowser: - utils = mobileBrowser.utils - Login(mobileBrowser).login() - accountPointsCounter = Searches( - mobileBrowser, remainingSearches - ).bingSearches(remainingSearches.mobile) - - utils.goHome() - goalPoints = utils.getGoalPoints() - goalTitle = utils.getGoalTitle() - - remainingSearches = utils.getRemainingSearches() - - logging.info( - f"[POINTS] You have earned {utils.formatNumber(accountPointsCounter - startingPoints)} points this run !" - ) - logging.info( - f"[POINTS] You are now at {utils.formatNumber(accountPointsCounter)} points !" - ) - appriseSummary = AppriseSummary[utils.config["apprise"]["summary"]] - if appriseSummary == AppriseSummary.always: - goalNotifier = "" - if goalPoints > 0: - logging.info( - f"[POINTS] You are now at {(utils.formatNumber((accountPointsCounter / goalPoints) * 100))}% of your goal ({goalTitle}) !\n" - ) - goalNotifier = f"🎯 Goal reached: {(utils.formatNumber((accountPointsCounter / goalPoints) * 100))}% ({goalTitle})" - - Utils.sendNotification( - "Daily Points Update", - "\n".join( - [ - f"👤 Account: {currentAccount.username}", - f"⭐️ Points earned today: {utils.formatNumber(accountPointsCounter - startingPoints)}", - f"💰 Total points: {utils.formatNumber(accountPointsCounter)}", - goalNotifier, - ] - ), - ) - elif appriseSummary == AppriseSummary.on_error: - if remainingSearches.desktop > 0 or remainingSearches.mobile > 0: - Utils.sendNotification( - "Error: remaining searches", - f"account username: {currentAccount.username}, {remainingSearches}", - ) - - return accountPointsCounter - - -def export_points_to_csv(points_data): - logs_directory = Utils.getProjectRoot() / "logs" - csv_filename = logs_directory / "points_data.csv" - with open(csv_filename, mode="a", newline="") as file: # Use "a" mode for append - fieldnames = ["Account", "Earned Points", "Points Difference"] - writer = csv.DictWriter(file, fieldnames=fieldnames) - - # Check if the file is empty, and if so, write the header row - if file.tell() == 0: - writer.writeheader() - - for data in points_data: - writer.writerow(data) - - -# Define a function to load the previous day's points data from a file in the "logs" folder -def load_previous_points_data(): - logs_directory = Utils.getProjectRoot() / "logs" - try: - with open(logs_directory / "previous_points_data.json", "r") as file: - return json.load(file) - except FileNotFoundError: - return {} - - -# Define a function to save the current day's points data for the next day in the "logs" folder -def save_previous_points_data(data): - logs_directory = Utils.getProjectRoot() / "logs" - with open(logs_directory / "previous_points_data.json", "w") as file: - json.dump(data, file, indent=4) - - -if __name__ == "__main__": - main() +import argparse +import atexit +import csv +import json +import logging +import logging.config +import logging.handlers as handlers +import random +import re +import sys +import time +from datetime import datetime +from enum import Enum, auto + +import psutil + +from src import ( + Browser, + Login, + MorePromotions, + PunchCards, + Searches, + DailySet, + Account, +) +from src.loggingColoredFormatter import ColoredFormatter +from src.utils import Utils, RemainingSearches + + +def main(): + args = argumentParser() + setupLogging() + loadedAccounts = setupAccounts() + # Register the cleanup function to be called on script exit + atexit.register(cleanupChromeProcesses) + + # Load previous day's points data + previous_points_data = load_previous_points_data() + + for currentAccount in loadedAccounts: + try: + earned_points = executeBot(currentAccount, args) + previous_points = previous_points_data.get(currentAccount.username, 0) + + # Calculate the difference in points from the prior day + points_difference = earned_points - previous_points + + # Append the daily points and points difference to CSV and Excel + log_daily_points_to_csv( + earned_points, points_difference + ) + + # Update the previous day's points data + previous_points_data[currentAccount.username] = earned_points + + logging.info( + f"[POINTS] Data for '{currentAccount.username}' appended to the file." + ) + except Exception as e: + Utils.sendNotification( + "⚠️ Error occurred, please check the log", f"{e}\n{e.__traceback__}" + ) + logging.exception(f"{e.__class__.__name__}: {e}") + exit(1) + + # Save the current day's points data for the next day in the "logs" folder + save_previous_points_data(previous_points_data) + logging.info("[POINTS] Data saved for the next day.") + + +def log_daily_points_to_csv(earned_points, points_difference): + logs_directory = Utils.getProjectRoot() / "logs" + csv_filename = logs_directory / "points_data.csv" + + # Create a new row with the date, daily points, and points difference + date = datetime.now().strftime("%Y-%m-%d") + new_row = { + "Date": date, + "Earned Points": earned_points, + "Points Difference": points_difference, + } + + fieldnames = ["Date", "Earned Points", "Points Difference"] + is_new_file = not csv_filename.exists() + + with open(csv_filename, mode="a", newline="") as file: + writer = csv.DictWriter(file, fieldnames=fieldnames) + + if is_new_file: + writer.writeheader() + + writer.writerow(new_row) + + +def setupLogging(): + _format = "%(asctime)s [%(levelname)s] %(message)s" + terminalHandler = logging.StreamHandler(sys.stdout) + terminalHandler.setFormatter(ColoredFormatter(_format)) + + logs_directory = Utils.getProjectRoot() / "logs" + logs_directory.mkdir(parents=True, exist_ok=True) + + # so only our code is logged if level=logging.DEBUG or finer + # if not working see https://stackoverflow.com/a/48891485/4164390 + logging.config.dictConfig( + { + "version": 1, + "disable_existing_loggers": True, + } + ) + logging.basicConfig( + level=logging.INFO, + format=_format, + handlers=[ + handlers.TimedRotatingFileHandler( + logs_directory / "activity.log", + when="midnight", + interval=1, + backupCount=2, + encoding="utf-8", + ), + terminalHandler, + ], + ) + + +def cleanupChromeProcesses(): + # Use psutil to find and terminate Chrome processes + for process in psutil.process_iter(["pid", "name"]): + if process.info["name"] == "chrome.exe": + try: + psutil.Process(process.info["pid"]).terminate() + except (psutil.NoSuchProcess, psutil.AccessDenied): + pass + + +def argumentParser() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="MS Rewards Farmer") + parser.add_argument( + "-v", "--visible", action="store_true", help="Optional: Visible browser" + ) + parser.add_argument( + "-l", "--lang", type=str, default=None, help="Optional: Language (ex: en)" + ) + parser.add_argument( + "-g", "--geo", type=str, default=None, help="Optional: Geolocation (ex: US)" + ) + parser.add_argument( + "-p", + "--proxy", + type=str, + default=None, + help="Optional: Global Proxy (ex: http://user:pass@host:port)", + ) + parser.add_argument( + "-vn", + "--verbosenotifs", + action="store_true", + help="Optional: Send all the logs to the notification service", + ) + parser.add_argument( + "-cv", + "--chromeversion", + type=int, + default=None, + help="Optional: Set fixed Chrome version (ex. 118)", + ) + return parser.parse_args() + + +def setupAccounts() -> list[Account]: + """Sets up and validates a list of accounts loaded from 'accounts.json'.""" + + def validEmail(email: str) -> bool: + """Validate Email.""" + pattern = r"^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$" + return bool(re.match(pattern, email)) + + accountPath = Utils.getProjectRoot() / "accounts.json" + if not accountPath.exists(): + accountPath.write_text( + json.dumps( + [{"username": "Your Email", "password": "Your Password"}], indent=4 + ), + encoding="utf-8", + ) + noAccountsNotice = """ + [ACCOUNT] Accounts credential file "accounts.json" not found. + [ACCOUNT] A new file has been created, please edit with your credentials and save. + """ + logging.warning(noAccountsNotice) + exit(1) + loadedAccounts: list[Account] = [] + for rawAccount in json.loads(accountPath.read_text(encoding="utf-8")): + account: Account = Account(**rawAccount) + if not validEmail(account.username): + logging.warning( + f"[CREDENTIALS] Invalid email: {account.username}, skipping this account" + ) + continue + loadedAccounts.append(account) + random.shuffle(loadedAccounts) + return loadedAccounts + + +class AppriseSummary(Enum): + always = auto() + on_error = auto() + + +def executeBot(currentAccount: Account, args: argparse.Namespace): + logging.info(f"********************{currentAccount.username}********************") + + accountPointsCounter: int + remainingSearches: RemainingSearches + startingPoints: int + + with Browser(mobile=False, account=currentAccount, args=args) as desktopBrowser: + utils = desktopBrowser.utils + accountPointsCounter = Login(desktopBrowser).login() + startingPoints = accountPointsCounter + if startingPoints == "Locked": + Utils.sendNotification("🚫 Account is Locked", currentAccount.username) + return 0 + if startingPoints == "Verify": + Utils.sendNotification( + "❗️ Account needs to be verified", currentAccount.username + ) + return 0 + logging.info( + f"[POINTS] You have {utils.formatNumber(accountPointsCounter)} points on your account" + ) + # todo - make quicker if done + DailySet(desktopBrowser).completeDailySet() + PunchCards(desktopBrowser).completePunchCards() + MorePromotions(desktopBrowser).completeMorePromotions() + # VersusGame(desktopBrowser).completeVersusGame() + utils.goHome() + remainingSearches = utils.getRemainingSearches() + + if remainingSearches.desktop != 0: + accountPointsCounter = Searches( + desktopBrowser, remainingSearches + ).bingSearches(remainingSearches.desktop) + + utils.goHome() + goalPoints = utils.getGoalPoints() + goalTitle = utils.getGoalTitle() + + time.sleep(60) # give time for browser to close, probably can be less time + + if remainingSearches.mobile != 0: + with Browser(mobile=True, account=currentAccount, args=args) as mobileBrowser: + utils = mobileBrowser.utils + Login(mobileBrowser).login() + accountPointsCounter = Searches( + mobileBrowser, remainingSearches + ).bingSearches(remainingSearches.mobile) + + utils.goHome() + goalPoints = utils.getGoalPoints() + goalTitle = utils.getGoalTitle() + + remainingSearches = utils.getRemainingSearches() + + logging.info( + f"[POINTS] You have earned {utils.formatNumber(accountPointsCounter - startingPoints)} points this run !" + ) + logging.info( + f"[POINTS] You are now at {utils.formatNumber(accountPointsCounter)} points !" + ) + appriseSummary = AppriseSummary[utils.config["apprise"]["summary"]] + if appriseSummary == AppriseSummary.always: + goalNotifier = "" + if goalPoints > 0: + logging.info( + f"[POINTS] You are now at {(utils.formatNumber((accountPointsCounter / goalPoints) * 100))}% of your " + f"goal ({goalTitle}) !" + ) + goalNotifier = (f"🎯 Goal reached: {(utils.formatNumber((accountPointsCounter / goalPoints) * 100))}%" + f" ({goalTitle})") + + Utils.sendNotification( + "Daily Points Update", + "\n".join( + [ + f"👤 Account: {currentAccount.username}", + f"⭐️ Points earned today: {utils.formatNumber(accountPointsCounter - startingPoints)}", + f"💰 Total points: {utils.formatNumber(accountPointsCounter)}", + goalNotifier, + ] + ), + ) + elif appriseSummary == AppriseSummary.on_error: + if remainingSearches.desktop > 0 or remainingSearches.mobile > 0: + Utils.sendNotification( + "Error: remaining searches", + f"account username: {currentAccount.username}, {remainingSearches}", + ) + + return accountPointsCounter + + +def export_points_to_csv(points_data): + logs_directory = Utils.getProjectRoot() / "logs" + csv_filename = logs_directory / "points_data.csv" + with open(csv_filename, mode="a", newline="") as file: # Use "a" mode for append + fieldnames = ["Account", "Earned Points", "Points Difference"] + writer = csv.DictWriter(file, fieldnames=fieldnames) + + # Check if the file is empty, and if so, write the header row + if file.tell() == 0: + writer.writeheader() + + for data in points_data: + writer.writerow(data) + + +# Define a function to load the previous day's points data from a file in the "logs" folder +def load_previous_points_data(): + logs_directory = Utils.getProjectRoot() / "logs" + try: + with open(logs_directory / "previous_points_data.json", "r") as file: + return json.load(file) + except FileNotFoundError: + return {} + + +# Define a function to save the current day's points data for the next day in the "logs" folder +def save_previous_points_data(data): + logs_directory = Utils.getProjectRoot() / "logs" + with open(logs_directory / "previous_points_data.json", "w") as file: + json.dump(data, file, indent=4) + + +if __name__ == "__main__": + main() diff --git a/src/browser.py b/src/browser.py index de9d9593..a234374e 100644 --- a/src/browser.py +++ b/src/browser.py @@ -1,4 +1,3 @@ -import contextlib import logging import random from pathlib import Path @@ -19,6 +18,7 @@ class Browser: def __init__(self, mobile: bool, account: Account, args: Any) -> None: # Initialize browser instance + logging.debug("in __init__") self.mobile = mobile self.browserType = "mobile" if mobile else "desktop" self.headless = not args.visible @@ -42,19 +42,18 @@ def __init__(self, mobile: bool, account: Account, args: Any) -> None: Utils.saveBrowserConfig(self.userDataDir, self.browserConfig) self.webdriver = self.browserSetup() self.utils = Utils(self.webdriver) + logging.debug("out __init__") def __enter__(self) -> "Browser": + logging.debug("in __enter__") return self def __exit__(self, *args: Any) -> None: # Cleanup actions when exiting the browser context - self.closeBrowser() - - def closeBrowser(self) -> None: - """Perform actions to close the browser cleanly.""" - # Close the web browser - with contextlib.suppress(Exception): - self.webdriver.close() + logging.debug("in __exit__") + # self.webdriver.close() # just closes window, doesn't lose driver, see https://stackoverflow.com/a/32447644/4164390 + self.webdriver.quit() + # self.webdriver.__exit__(None, None, None) # doesn't seem to work def browserSetup( self, @@ -194,6 +193,7 @@ def getCCodeLang(lang: str, geo: str) -> tuple: if geo is None: geo = nfo["country"] except Exception: # pylint: disable=broad-except + logging.debug(Exception) return "en", "US" return lang, geo diff --git a/src/searches.py b/src/searches.py index 5944b718..05324cdb 100644 --- a/src/searches.py +++ b/src/searches.py @@ -5,6 +5,7 @@ from datetime import date, timedelta from enum import Enum, auto from itertools import cycle +from typing import Optional import requests from selenium.common.exceptions import TimeoutException @@ -39,15 +40,17 @@ class Searches: "strategy", DEFAULT_ATTEMPTS_STRATEGY ) ] - searchTerms: list[str] = None + searchTerms: Optional[list[str]] = None def __init__(self, browser: Browser, searches: RemainingSearches): self.browser = browser self.webdriver = browser.webdriver + # Share search terms across instances to get rid of duplicates if Searches.searchTerms is None: Searches.searchTerms = self.getGoogleTrends( searches.desktop + searches.mobile ) + # Shuffle in case not only run of the day random.shuffle(Searches.searchTerms) def getGoogleTrends(self, wordsCount: int) -> list[str]: @@ -58,7 +61,8 @@ def getGoogleTrends(self, wordsCount: int) -> list[str]: i += 1 # Fetching daily trends from Google Trends API r = requests.get( - f'https://trends.google.com/trends/api/dailytrends?hl={self.browser.localeLang}&ed={(date.today() - timedelta(days=i)).strftime("%Y%m%d")}&geo={self.browser.localeGeo}&ns=15' + f'https://trends.google.com/trends/api/dailytrends?hl={self.browser.localeLang}' + f'&ed={(date.today() - timedelta(days=i)).strftime("%Y%m%d")}&geo={self.browser.localeGeo}&ns=15' ) trends = json.loads(r.text[6:]) for topic in trends["default"]["trendingSearchesDays"][0][ @@ -70,7 +74,7 @@ def getGoogleTrends(self, wordsCount: int) -> list[str]: for relatedTopic in topic["relatedQueries"] ) searchTerms = list(set(searchTerms)) - del searchTerms[wordsCount : (len(searchTerms) + 1)] + del searchTerms[wordsCount: (len(searchTerms) + 1)] return searchTerms def getRelatedTerms(self, word: str) -> list[str]: From e3ea989a03809f82137be04121e7909752467213 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Wed, 12 Jun 2024 16:04:35 -0400 Subject: [PATCH 08/74] Give time for browser to close, suppress exceptions, prefer | to Optional, add some to-dos --- main.py | 4 ++-- src/account.py | 3 +-- src/browser.py | 16 ++++++++++------ src/searches.py | 6 +++--- 4 files changed, 16 insertions(+), 13 deletions(-) diff --git a/main.py b/main.py index cee42900..d8c20173 100644 --- a/main.py +++ b/main.py @@ -109,7 +109,7 @@ def setupLogging(): } ) logging.basicConfig( - level=logging.INFO, + level=logging.DEBUG, format=_format, handlers=[ handlers.TimedRotatingFileHandler( @@ -247,7 +247,7 @@ def executeBot(currentAccount: Account, args: argparse.Namespace): goalPoints = utils.getGoalPoints() goalTitle = utils.getGoalTitle() - time.sleep(60) # give time for browser to close, probably can be less time + time.sleep(15) # give time for browser to close, probably can be less time if remainingSearches.mobile != 0: with Browser(mobile=True, account=currentAccount, args=args) as mobileBrowser: diff --git a/src/account.py b/src/account.py index 9889ee61..35773dd6 100644 --- a/src/account.py +++ b/src/account.py @@ -1,9 +1,8 @@ from dataclasses import dataclass -from typing import Optional @dataclass class Account: username: str password: str - proxy: Optional[str] = None + proxy: str | None = None diff --git a/src/browser.py b/src/browser.py index a234374e..b1cdf192 100644 --- a/src/browser.py +++ b/src/browser.py @@ -1,7 +1,9 @@ +import contextlib import logging import random from pathlib import Path -from typing import Any +from types import TracebackType +from typing import Any, Type import ipapi import seleniumwire.undetected_chromedriver as webdriver @@ -48,12 +50,14 @@ def __enter__(self) -> "Browser": logging.debug("in __enter__") return self - def __exit__(self, *args: Any) -> None: + def __exit__(self, exc_type: Type[BaseException] | None, exc_value: BaseException | None, + traceback: TracebackType | None) -> None: # Cleanup actions when exiting the browser context - logging.debug("in __exit__") - # self.webdriver.close() # just closes window, doesn't lose driver, see https://stackoverflow.com/a/32447644/4164390 - self.webdriver.quit() - # self.webdriver.__exit__(None, None, None) # doesn't seem to work + logging.debug(f"in __exit__ exc_type={exc_type} exc_value={exc_value} traceback={traceback}") + with contextlib.suppress(Exception): + # self.webdriver.close() # just closes window, doesn't lose driver, see https://stackoverflow.com/a/32447644/4164390 + # self.webdriver.__exit__(None, None, None) # doesn't seem to work # doesn't work + self.webdriver.quit() def browserSetup( self, diff --git a/src/searches.py b/src/searches.py index 05324cdb..af16620c 100644 --- a/src/searches.py +++ b/src/searches.py @@ -5,7 +5,6 @@ from datetime import date, timedelta from enum import Enum, auto from itertools import cycle -from typing import Optional import requests from selenium.common.exceptions import TimeoutException @@ -40,7 +39,7 @@ class Searches: "strategy", DEFAULT_ATTEMPTS_STRATEGY ) ] - searchTerms: Optional[list[str]] = None + searchTerms: list[str] | None = None def __init__(self, browser: Browser, searches: RemainingSearches): self.browser = browser @@ -52,6 +51,7 @@ def __init__(self, browser: Browser, searches: RemainingSearches): ) # Shuffle in case not only run of the day random.shuffle(Searches.searchTerms) + # todo could write shuffled total searches to disk and read to make even more random def getGoogleTrends(self, wordsCount: int) -> list[str]: # Function to retrieve Google Trends search terms @@ -87,7 +87,7 @@ def getRelatedTerms(self, word: str) -> list[str]: return r.json()[1] except Exception: # pylint: disable=broad-except logging.warn(Exception) - return [] + return [word] def bingSearches(self, numberOfSearches: int, pointsCounter: int = 0): # Function to perform Bing searches From b3d1f48d2d2b3175ccd17a475721de36b76a8c2e Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Thu, 13 Jun 2024 00:15:54 -0400 Subject: [PATCH 09/74] Provide default config if not present --- main.py | 5 +- src/searches.py | 22 +- src/utils.py | 608 ++++++++++++++++++++++++------------------------ 3 files changed, 310 insertions(+), 325 deletions(-) diff --git a/main.py b/main.py index d8c20173..1ad2e8f3 100644 --- a/main.py +++ b/main.py @@ -1,5 +1,4 @@ import argparse -import atexit import csv import json import logging @@ -12,8 +11,6 @@ from datetime import datetime from enum import Enum, auto -import psutil - from src import ( Browser, Login, @@ -269,7 +266,7 @@ def executeBot(currentAccount: Account, args: argparse.Namespace): logging.info( f"[POINTS] You are now at {utils.formatNumber(accountPointsCounter)} points !" ) - appriseSummary = AppriseSummary[utils.config["apprise"]["summary"]] + appriseSummary = AppriseSummary[utils.config.get("apprise", {}).get("summary", AppriseSummary.on_error.name)] if appriseSummary == AppriseSummary.always: goalNotifier = "" if goalPoints > 0: diff --git a/src/searches.py b/src/searches.py index af16620c..96fdbe4e 100644 --- a/src/searches.py +++ b/src/searches.py @@ -20,24 +20,12 @@ class AttemptsStrategy(Enum): constant = auto() -DEFAULT_ATTEMPTS_MAX = 3 -DEFAULT_BASE_DELAY = 900 -DEFAULT_ATTEMPTS_STRATEGY = AttemptsStrategy.exponential.name - - class Searches: config = Utils.loadConfig() - # todo get rid of duplication, if possible - maxAttempts: int = config.get("attempts", DEFAULT_ATTEMPTS_MAX).get( - "max", DEFAULT_ATTEMPTS_MAX - ) - baseDelay: int = config.get("attempts", DEFAULT_BASE_DELAY).get( - "base_delay_in_seconds", DEFAULT_BASE_DELAY - ) + maxAttempts: int = config.get("attempts", {}).get("max", 6) + baseDelay: int = config.get("attempts", {}).get("base_delay_in_seconds", 60) attemptsStrategy = AttemptsStrategy[ - config.get("attempts", DEFAULT_ATTEMPTS_STRATEGY).get( - "strategy", DEFAULT_ATTEMPTS_STRATEGY - ) + config.get("attempts", {}).get("strategy", AttemptsStrategy.exponential.name) ] searchTerms: list[str] | None = None @@ -61,7 +49,7 @@ def getGoogleTrends(self, wordsCount: int) -> list[str]: i += 1 # Fetching daily trends from Google Trends API r = requests.get( - f'https://trends.google.com/trends/api/dailytrends?hl={self.browser.localeLang}' + f"https://trends.google.com/trends/api/dailytrends?hl={self.browser.localeLang}" f'&ed={(date.today() - timedelta(days=i)).strftime("%Y%m%d")}&geo={self.browser.localeGeo}&ns=15' ) trends = json.loads(r.text[6:]) @@ -74,7 +62,7 @@ def getGoogleTrends(self, wordsCount: int) -> list[str]: for relatedTopic in topic["relatedQueries"] ) searchTerms = list(set(searchTerms)) - del searchTerms[wordsCount: (len(searchTerms) + 1)] + del searchTerms[wordsCount : (len(searchTerms) + 1)] return searchTerms def getRelatedTerms(self, word: str) -> list[str]: diff --git a/src/utils.py b/src/utils.py index 87592c86..6e1a66fe 100644 --- a/src/utils.py +++ b/src/utils.py @@ -1,305 +1,305 @@ -import contextlib -import json -import locale as pylocale -import sys -import time -import urllib.parse -from pathlib import Path -from typing import NamedTuple - -import requests -import yaml -from apprise import Apprise -from selenium.webdriver.chrome.webdriver import WebDriver -from selenium.webdriver.common.by import By -from selenium.webdriver.support import expected_conditions as ec -from selenium.webdriver.support.wait import WebDriverWait - -from .constants import BASE_URL - - -class RemainingSearches(NamedTuple): - desktop: int - mobile: int - - -class Utils: - def __init__(self, webdriver: WebDriver): - self.webdriver = webdriver - with contextlib.suppress(Exception): - locale = pylocale.getdefaultlocale()[0] - pylocale.setlocale(pylocale.LC_NUMERIC, locale) - - self.config = self.loadConfig() - - @staticmethod - def getProjectRoot() -> Path: - return Path(__file__).parent.parent - - @staticmethod - def loadConfig(config_file=getProjectRoot() / "config.yaml"): - with open(config_file, "r") as file: - return yaml.safe_load(file) - - @staticmethod - def sendNotification(title, body): - apprise = Apprise() - for url in Utils.loadConfig()["apprise"]["urls"]: - apprise.add(url) - apprise.notify(body=body, title=title) - - def waitUntilVisible(self, by: str, selector: str, timeToWait: float = 10): - WebDriverWait(self.webdriver, timeToWait).until( - ec.visibility_of_element_located((by, selector)) - ) - - def waitUntilClickable(self, by: str, selector: str, timeToWait: float = 10): - WebDriverWait(self.webdriver, timeToWait).until( - ec.element_to_be_clickable((by, selector)) - ) - - def waitForMSRewardElement(self, by: str, selector: str): - loadingTimeAllowed = 5 - refreshesAllowed = 5 - - checkingInterval = 0.5 - checks = loadingTimeAllowed / checkingInterval - - tries = 0 - refreshCount = 0 - while True: - try: - self.webdriver.find_element(by, selector) - return True - except Exception: - if tries < checks: - tries += 1 - time.sleep(checkingInterval) - elif refreshCount < refreshesAllowed: - self.webdriver.refresh() - refreshCount += 1 - tries = 0 - time.sleep(5) - else: - return False - - def waitUntilQuestionRefresh(self): - return self.waitForMSRewardElement(By.CLASS_NAME, "rqECredits") - - def waitUntilQuizLoads(self): - return self.waitForMSRewardElement(By.XPATH, '//*[@id="rqStartQuiz"]') - - def waitUntilJS(self, jsSrc: str): - loadingTimeAllowed = 5 - refreshesAllowed = 5 - - checkingInterval = 0.5 - checks = loadingTimeAllowed / checkingInterval - - tries = 0 - refreshCount = 0 - while True: - elem = self.webdriver.execute_script(jsSrc) - if elem: - return elem - - if tries < checks: - tries += 1 - time.sleep(checkingInterval) - elif refreshCount < refreshesAllowed: - self.webdriver.refresh() - refreshCount += 1 - tries = 0 - time.sleep(5) - else: - return elem - - def resetTabs(self): - try: - curr = self.webdriver.current_window_handle - - for handle in self.webdriver.window_handles: - if handle != curr: - self.webdriver.switch_to.window(handle) - time.sleep(0.5) - self.webdriver.close() - time.sleep(0.5) - - self.webdriver.switch_to.window(curr) - time.sleep(0.5) - self.goHome() - except Exception: - self.goHome() - - def goHome(self): - reloadThreshold = 5 - reloadInterval = 10 - targetUrl = urllib.parse.urlparse(BASE_URL) - self.webdriver.get(BASE_URL) - reloads = 0 - interval = 1 - intervalCount = 0 - while True: - self.tryDismissCookieBanner() - with contextlib.suppress(Exception): - self.webdriver.find_element(By.ID, "more-activities") - break - currentUrl = urllib.parse.urlparse(self.webdriver.current_url) - if ( - currentUrl.hostname != targetUrl.hostname - ) and self.tryDismissAllMessages(): - time.sleep(1) - self.webdriver.get(BASE_URL) - time.sleep(interval) - if "proofs" in str(self.webdriver.current_url): - return "Verify" - intervalCount += 1 - if intervalCount >= reloadInterval: - intervalCount = 0 - reloads += 1 - self.webdriver.refresh() - if reloads >= reloadThreshold: - break - - @staticmethod - def getAnswerCode(key: str, string: str) -> str: - t = sum(ord(string[i]) for i in range(len(string))) - t += int(key[-2:], 16) - return str(t) - - def getDashboardData(self) -> dict: - self.goHome() - return self.webdriver.execute_script("return dashboard") - - def getBingInfo(self): - cookieJar = self.webdriver.get_cookies() - cookies = {cookie["name"]: cookie["value"] for cookie in cookieJar} - maxTries = 5 - for _ in range(maxTries): - with contextlib.suppress(Exception): - response = requests.get( - "https://www.bing.com/rewards/panelflyout/getuserinfo", - cookies=cookies, - ) - if response.status_code == requests.codes.ok: - return response.json() - time.sleep(1) - return None - - def checkBingLogin(self): - if data := self.getBingInfo(): - return data["userInfo"]["isRewardsUser"] - else: - return False - - def getAccountPoints(self) -> int: - return self.getDashboardData()["userStatus"]["availablePoints"] - - def getBingAccountPoints(self) -> int: - return data["userInfo"]["balance"] if (data := self.getBingInfo()) else 0 - - def getGoalPoints(self) -> int: - return self.getDashboardData()["userStatus"]["redeemGoal"]["price"] - - def getGoalTitle(self) -> str: - return self.getDashboardData()["userStatus"]["redeemGoal"]["title"] - - def tryDismissAllMessages(self): - buttons = [ - (By.ID, "iLandingViewAction"), - (By.ID, "iShowSkip"), - (By.ID, "iNext"), - (By.ID, "iLooksGood"), - (By.ID, "idSIButton9"), - (By.CSS_SELECTOR, ".ms-Button.ms-Button--primary"), - (By.ID, "bnp_btn_accept"), - (By.ID, "acceptButton"), - ] - result = False - for button in buttons: - try: - elements = self.webdriver.find_elements(button[0], button[1]) - try: - for element in elements: - element.click() - except Exception: - continue - result = True - except Exception: - continue - return result - - def tryDismissCookieBanner(self): - with contextlib.suppress(Exception): - self.webdriver.find_element(By.ID, "cookie-banner").find_element( - By.TAG_NAME, "button" - ).click() - time.sleep(2) - - def tryDismissBingCookieBanner(self): - with contextlib.suppress(Exception): - self.webdriver.find_element(By.ID, "bnp_btn_accept").click() - time.sleep(2) - - def switchToNewTab(self, timeToWait: int = 0): - time.sleep(0.5) - self.webdriver.switch_to.window(window_name=self.webdriver.window_handles[1]) - if timeToWait > 0: - time.sleep(timeToWait) - - def closeCurrentTab(self): - self.webdriver.close() - time.sleep(0.5) - self.webdriver.switch_to.window(window_name=self.webdriver.window_handles[0]) - time.sleep(0.5) - - def visitNewTab(self, timeToWait: int = 0): - self.switchToNewTab(timeToWait) - self.closeCurrentTab() - - def getRemainingSearches(self) -> RemainingSearches: - dashboard = self.getDashboardData() - searchPoints = 1 - counters = dashboard["userStatus"]["counters"] - - progressDesktop = counters["pcSearch"][0]["pointProgress"] - targetDesktop = counters["pcSearch"][0]["pointProgressMax"] - if len(counters["pcSearch"]) >= 2: - progressDesktop = progressDesktop + counters["pcSearch"][1]["pointProgress"] - targetDesktop = targetDesktop + counters["pcSearch"][1]["pointProgressMax"] - if targetDesktop in [30, 90, 102]: - searchPoints = 3 - elif targetDesktop == 50 or targetDesktop >= 170 or targetDesktop == 150: - searchPoints = 5 - remainingDesktop = int((targetDesktop - progressDesktop) / searchPoints) - remainingMobile = 0 - if dashboard["userStatus"]["levelInfo"]["activeLevel"] != "Level1": - progressMobile = counters["mobileSearch"][0]["pointProgress"] - targetMobile = counters["mobileSearch"][0]["pointProgressMax"] - remainingMobile = int((targetMobile - progressMobile) / searchPoints) - return RemainingSearches(desktop=remainingDesktop, mobile=remainingMobile) - - @staticmethod - def formatNumber(number, num_decimals=2): - return pylocale.format_string( - f"%10.{num_decimals}f", number, grouping=True - ).strip() - - @staticmethod - def getBrowserConfig(sessionPath: Path) -> dict: - configFile = sessionPath.joinpath("config.json") - if configFile.exists(): - with open(configFile, "r") as f: - return json.load(f) - else: - return {} - - @staticmethod - def saveBrowserConfig(sessionPath: Path, config: dict): - configFile = sessionPath.joinpath("config.json") - with open(configFile, "w") as f: - json.dump(config, f) - - @staticmethod - def isDebuggerAttached() -> bool: +import contextlib +import json +import locale as pylocale +import time +import urllib.parse +from pathlib import Path +from typing import NamedTuple + +import requests +import yaml +from apprise import Apprise +from selenium.webdriver.chrome.webdriver import WebDriver +from selenium.webdriver.common.by import By +from selenium.webdriver.support import expected_conditions as ec +from selenium.webdriver.support.wait import WebDriverWait + +from .constants import BASE_URL + + +class RemainingSearches(NamedTuple): + desktop: int + mobile: int + + +class Utils: + def __init__(self, webdriver: WebDriver): + self.webdriver = webdriver + with contextlib.suppress(Exception): + locale = pylocale.getdefaultlocale()[0] + pylocale.setlocale(pylocale.LC_NUMERIC, locale) + + self.config = self.loadConfig() + + @staticmethod + def getProjectRoot() -> Path: + return Path(__file__).parent.parent + + @staticmethod + def loadConfig(config_file=getProjectRoot() / "config.yaml") -> dict: + with open(config_file, "r") as file: + return yaml.safe_load(file) + + @staticmethod + def sendNotification(title, body): + apprise = Apprise() + urls: list[str] = Utils.loadConfig().get("apprise", {}).get("urls", []) + for url in urls: + apprise.add(url) + apprise.notify(body=body, title=title) + + def waitUntilVisible(self, by: str, selector: str, timeToWait: float = 10): + WebDriverWait(self.webdriver, timeToWait).until( + ec.visibility_of_element_located((by, selector)) + ) + + def waitUntilClickable(self, by: str, selector: str, timeToWait: float = 10): + WebDriverWait(self.webdriver, timeToWait).until( + ec.element_to_be_clickable((by, selector)) + ) + + def waitForMSRewardElement(self, by: str, selector: str): + loadingTimeAllowed = 5 + refreshesAllowed = 5 + + checkingInterval = 0.5 + checks = loadingTimeAllowed / checkingInterval + + tries = 0 + refreshCount = 0 + while True: + try: + self.webdriver.find_element(by, selector) + return True + except Exception: + if tries < checks: + tries += 1 + time.sleep(checkingInterval) + elif refreshCount < refreshesAllowed: + self.webdriver.refresh() + refreshCount += 1 + tries = 0 + time.sleep(5) + else: + return False + + def waitUntilQuestionRefresh(self): + return self.waitForMSRewardElement(By.CLASS_NAME, "rqECredits") + + def waitUntilQuizLoads(self): + return self.waitForMSRewardElement(By.XPATH, '//*[@id="rqStartQuiz"]') + + def waitUntilJS(self, jsSrc: str): + loadingTimeAllowed = 5 + refreshesAllowed = 5 + + checkingInterval = 0.5 + checks = loadingTimeAllowed / checkingInterval + + tries = 0 + refreshCount = 0 + while True: + elem = self.webdriver.execute_script(jsSrc) + if elem: + return elem + + if tries < checks: + tries += 1 + time.sleep(checkingInterval) + elif refreshCount < refreshesAllowed: + self.webdriver.refresh() + refreshCount += 1 + tries = 0 + time.sleep(5) + else: + return elem + + def resetTabs(self): + try: + curr = self.webdriver.current_window_handle + + for handle in self.webdriver.window_handles: + if handle != curr: + self.webdriver.switch_to.window(handle) + time.sleep(0.5) + self.webdriver.close() + time.sleep(0.5) + + self.webdriver.switch_to.window(curr) + time.sleep(0.5) + self.goHome() + except Exception: + self.goHome() + + def goHome(self): + reloadThreshold = 5 + reloadInterval = 10 + targetUrl = urllib.parse.urlparse(BASE_URL) + self.webdriver.get(BASE_URL) + reloads = 0 + interval = 1 + intervalCount = 0 + while True: + self.tryDismissCookieBanner() + with contextlib.suppress(Exception): + self.webdriver.find_element(By.ID, "more-activities") + break + currentUrl = urllib.parse.urlparse(self.webdriver.current_url) + if ( + currentUrl.hostname != targetUrl.hostname + ) and self.tryDismissAllMessages(): + time.sleep(1) + self.webdriver.get(BASE_URL) + time.sleep(interval) + if "proofs" in str(self.webdriver.current_url): + return "Verify" + intervalCount += 1 + if intervalCount >= reloadInterval: + intervalCount = 0 + reloads += 1 + self.webdriver.refresh() + if reloads >= reloadThreshold: + break + + @staticmethod + def getAnswerCode(key: str, string: str) -> str: + t = sum(ord(string[i]) for i in range(len(string))) + t += int(key[-2:], 16) + return str(t) + + def getDashboardData(self) -> dict: + self.goHome() + return self.webdriver.execute_script("return dashboard") + + def getBingInfo(self): + cookieJar = self.webdriver.get_cookies() + cookies = {cookie["name"]: cookie["value"] for cookie in cookieJar} + maxTries = 5 + for _ in range(maxTries): + with contextlib.suppress(Exception): + response = requests.get( + "https://www.bing.com/rewards/panelflyout/getuserinfo", + cookies=cookies, + ) + if response.status_code == requests.codes.ok: + return response.json() + time.sleep(1) + return None + + def checkBingLogin(self): + if data := self.getBingInfo(): + return data["userInfo"]["isRewardsUser"] + else: + return False + + def getAccountPoints(self) -> int: + return self.getDashboardData()["userStatus"]["availablePoints"] + + def getBingAccountPoints(self) -> int: + return data["userInfo"]["balance"] if (data := self.getBingInfo()) else 0 + + def getGoalPoints(self) -> int: + return self.getDashboardData()["userStatus"]["redeemGoal"]["price"] + + def getGoalTitle(self) -> str: + return self.getDashboardData()["userStatus"]["redeemGoal"]["title"] + + def tryDismissAllMessages(self): + buttons = [ + (By.ID, "iLandingViewAction"), + (By.ID, "iShowSkip"), + (By.ID, "iNext"), + (By.ID, "iLooksGood"), + (By.ID, "idSIButton9"), + (By.CSS_SELECTOR, ".ms-Button.ms-Button--primary"), + (By.ID, "bnp_btn_accept"), + (By.ID, "acceptButton"), + ] + result = False + for button in buttons: + try: + elements = self.webdriver.find_elements(button[0], button[1]) + try: + for element in elements: + element.click() + except Exception: + continue + result = True + except Exception: + continue + return result + + def tryDismissCookieBanner(self): + with contextlib.suppress(Exception): + self.webdriver.find_element(By.ID, "cookie-banner").find_element( + By.TAG_NAME, "button" + ).click() + time.sleep(2) + + def tryDismissBingCookieBanner(self): + with contextlib.suppress(Exception): + self.webdriver.find_element(By.ID, "bnp_btn_accept").click() + time.sleep(2) + + def switchToNewTab(self, timeToWait: int = 0): + time.sleep(0.5) + self.webdriver.switch_to.window(window_name=self.webdriver.window_handles[1]) + if timeToWait > 0: + time.sleep(timeToWait) + + def closeCurrentTab(self): + self.webdriver.close() + time.sleep(0.5) + self.webdriver.switch_to.window(window_name=self.webdriver.window_handles[0]) + time.sleep(0.5) + + def visitNewTab(self, timeToWait: int = 0): + self.switchToNewTab(timeToWait) + self.closeCurrentTab() + + def getRemainingSearches(self) -> RemainingSearches: + dashboard = self.getDashboardData() + searchPoints = 1 + counters = dashboard["userStatus"]["counters"] + + progressDesktop = counters["pcSearch"][0]["pointProgress"] + targetDesktop = counters["pcSearch"][0]["pointProgressMax"] + if len(counters["pcSearch"]) >= 2: + progressDesktop = progressDesktop + counters["pcSearch"][1]["pointProgress"] + targetDesktop = targetDesktop + counters["pcSearch"][1]["pointProgressMax"] + if targetDesktop in [30, 90, 102]: + searchPoints = 3 + elif targetDesktop == 50 or targetDesktop >= 170 or targetDesktop == 150: + searchPoints = 5 + remainingDesktop = int((targetDesktop - progressDesktop) / searchPoints) + remainingMobile = 0 + if dashboard["userStatus"]["levelInfo"]["activeLevel"] != "Level1": + progressMobile = counters["mobileSearch"][0]["pointProgress"] + targetMobile = counters["mobileSearch"][0]["pointProgressMax"] + remainingMobile = int((targetMobile - progressMobile) / searchPoints) + return RemainingSearches(desktop=remainingDesktop, mobile=remainingMobile) + + @staticmethod + def formatNumber(number, num_decimals=2): + return pylocale.format_string( + f"%10.{num_decimals}f", number, grouping=True + ).strip() + + @staticmethod + def getBrowserConfig(sessionPath: Path) -> dict: + configFile = sessionPath.joinpath("config.json") + if configFile.exists(): + with open(configFile, "r") as f: + return json.load(f) + else: + return {} + + @staticmethod + def saveBrowserConfig(sessionPath: Path, config: dict): + configFile = sessionPath.joinpath("config.json") + with open(configFile, "w") as f: + json.dump(config, f) + + @staticmethod + def isDebuggerAttached() -> bool: return sys.gettrace() is not None \ No newline at end of file From ca8c7323feb8373182d4383b4702de89ba9c9146 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Thu, 13 Jun 2024 19:50:40 -0400 Subject: [PATCH 10/74] Adjust sleep between browsers --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index 1ad2e8f3..453a2caf 100644 --- a/main.py +++ b/main.py @@ -244,7 +244,7 @@ def executeBot(currentAccount: Account, args: argparse.Namespace): goalPoints = utils.getGoalPoints() goalTitle = utils.getGoalTitle() - time.sleep(15) # give time for browser to close, probably can be less time + time.sleep(7.5) # give time for browser to close, probably can be more fine-tuned if remainingSearches.mobile != 0: with Browser(mobile=True, account=currentAccount, args=args) as mobileBrowser: From a61f6b2d728b2ef9f9cf61a68c6c90d98ebcf299 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Thu, 13 Jun 2024 19:51:03 -0400 Subject: [PATCH 11/74] Restore default log level --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index 453a2caf..0dd7a729 100644 --- a/main.py +++ b/main.py @@ -106,7 +106,7 @@ def setupLogging(): } ) logging.basicConfig( - level=logging.DEBUG, + level=logging.INFO, format=_format, handlers=[ handlers.TimedRotatingFileHandler( From 4a598fc70709903c61f6da3989d1034f483ce520 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Thu, 13 Jun 2024 19:51:53 -0400 Subject: [PATCH 12/74] Remove redundant cleanup, handled already by __exit__ --- main.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/main.py b/main.py index 0dd7a729..8e959568 100644 --- a/main.py +++ b/main.py @@ -28,8 +28,6 @@ def main(): args = argumentParser() setupLogging() loadedAccounts = setupAccounts() - # Register the cleanup function to be called on script exit - atexit.register(cleanupChromeProcesses) # Load previous day's points data previous_points_data = load_previous_points_data() @@ -121,16 +119,6 @@ def setupLogging(): ) -def cleanupChromeProcesses(): - # Use psutil to find and terminate Chrome processes - for process in psutil.process_iter(["pid", "name"]): - if process.info["name"] == "chrome.exe": - try: - psutil.Process(process.info["pid"]).terminate() - except (psutil.NoSuchProcess, psutil.AccessDenied): - pass - - def argumentParser() -> argparse.Namespace: parser = argparse.ArgumentParser(description="MS Rewards Farmer") parser.add_argument( From c93285acae325c9869b06d5a6d1416f6a1bfea0a Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Thu, 13 Jun 2024 19:52:47 -0400 Subject: [PATCH 13/74] Remove exception suppression since handled by quit already --- src/browser.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/browser.py b/src/browser.py index b1cdf192..102304ab 100644 --- a/src/browser.py +++ b/src/browser.py @@ -1,4 +1,3 @@ -import contextlib import logging import random from pathlib import Path @@ -54,10 +53,9 @@ def __exit__(self, exc_type: Type[BaseException] | None, exc_value: BaseExceptio traceback: TracebackType | None) -> None: # Cleanup actions when exiting the browser context logging.debug(f"in __exit__ exc_type={exc_type} exc_value={exc_value} traceback={traceback}") - with contextlib.suppress(Exception): - # self.webdriver.close() # just closes window, doesn't lose driver, see https://stackoverflow.com/a/32447644/4164390 - # self.webdriver.__exit__(None, None, None) # doesn't seem to work # doesn't work - self.webdriver.quit() + # self.webdriver.close() # just closes window, doesn't lose driver, see https://stackoverflow.com/a/32447644/4164390 + # self.webdriver.__exit__(None, None, None) # doesn't seem to work # doesn't work + self.webdriver.quit() def browserSetup( self, From e153d95356fe4b1c1a0895c2b8f0a9a3701e19f0 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Thu, 13 Jun 2024 20:00:40 -0400 Subject: [PATCH 14/74] Adjust todos --- src/searches.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/searches.py b/src/searches.py index 96fdbe4e..deedc30a 100644 --- a/src/searches.py +++ b/src/searches.py @@ -39,7 +39,7 @@ def __init__(self, browser: Browser, searches: RemainingSearches): ) # Shuffle in case not only run of the day random.shuffle(Searches.searchTerms) - # todo could write shuffled total searches to disk and read to make even more random + # todo write shuffled searchTerms to disk to better emulate actual searches def getGoogleTrends(self, wordsCount: int) -> list[str]: # Function to retrieve Google Trends search terms @@ -149,5 +149,6 @@ def bingSearch(self, word: str) -> int: if Searches.attemptsStrategy == AttemptsStrategy.exponential: baseDelay *= 2 + # todo debug why we get to this point occasionally even though searches complete logging.error("[BING] Reached max search attempt retries") return bingAccountPointsBefore From a9cb94983c20af49b20561cf5cf255b93d498dfa Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Thu, 13 Jun 2024 20:27:14 -0400 Subject: [PATCH 15/74] Restore original defaults --- main.py | 2 +- src/searches.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/main.py b/main.py index 8e959568..4eb71da8 100644 --- a/main.py +++ b/main.py @@ -254,7 +254,7 @@ def executeBot(currentAccount: Account, args: argparse.Namespace): logging.info( f"[POINTS] You are now at {utils.formatNumber(accountPointsCounter)} points !" ) - appriseSummary = AppriseSummary[utils.config.get("apprise", {}).get("summary", AppriseSummary.on_error.name)] + appriseSummary = AppriseSummary[utils.config.get("apprise", {}).get("summary", AppriseSummary.always.name)] if appriseSummary == AppriseSummary.always: goalNotifier = "" if goalPoints > 0: diff --git a/src/searches.py b/src/searches.py index deedc30a..2cd7606b 100644 --- a/src/searches.py +++ b/src/searches.py @@ -25,7 +25,7 @@ class Searches: maxAttempts: int = config.get("attempts", {}).get("max", 6) baseDelay: int = config.get("attempts", {}).get("base_delay_in_seconds", 60) attemptsStrategy = AttemptsStrategy[ - config.get("attempts", {}).get("strategy", AttemptsStrategy.exponential.name) + config.get("attempts", {}).get("strategy", AttemptsStrategy.constant.name) ] searchTerms: list[str] | None = None From c76d24f8f4069d19af43ea5b6391af8e3c047d03 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Thu, 13 Jun 2024 20:32:08 -0400 Subject: [PATCH 16/74] Fix exception logging --- src/dailySet.py | 8 ++++---- src/login.py | 8 ++++---- src/morePromotions.py | 4 ++-- src/punchCards.py | 4 ++-- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/dailySet.py b/src/dailySet.py index b8f15272..f0dda44b 100644 --- a/src/dailySet.py +++ b/src/dailySet.py @@ -80,12 +80,12 @@ def completeDailySet(self): try: # Try completing ABC activity self.activities.completeABC() - except Exception: # pylint: disable=broad-except - logging.exception(Exception) + except Exception as e: # pylint: disable=broad-except + logging.warning(e) # Default to completing quiz self.activities.completeQuiz() - except Exception: # pylint: disable=broad-except - logging.exception(Exception) + except Exception as e: # pylint: disable=broad-except + logging.warning(e) # Reset tabs in case of an exception self.browser.utils.resetTabs() logging.info("[DAILY SET] Completed the Daily Set successfully !") diff --git a/src/login.py b/src/login.py index e611990c..40995b1f 100644 --- a/src/login.py +++ b/src/login.py @@ -27,13 +27,13 @@ def login(self) -> int: ) alreadyLoggedIn = True break - except Exception: # pylint: disable=broad-except - logging.exception(Exception) + except Exception as e: # pylint: disable=broad-except + logging.warning(e) try: self.utils.waitUntilVisible(By.ID, "i0116", 10) break - except Exception: # pylint: disable=broad-except - logging.exception(Exception) + except Exception as e: # pylint: disable=broad-except + logging.warning(e) if self.utils.tryDismissAllMessages(): continue diff --git a/src/morePromotions.py b/src/morePromotions.py index 0787ebe6..f73366ae 100644 --- a/src/morePromotions.py +++ b/src/morePromotions.py @@ -41,8 +41,8 @@ def completeMorePromotions(self): else: # Default to completing search self.activities.completeSearch() - except Exception: # pylint: disable=broad-except - logging.exception(Exception) + except Exception as e: # pylint: disable=broad-except + logging.warning(e) # Reset tabs in case of an exception self.browser.utils.resetTabs() logging.info("[MORE PROMOS] Completed More Promotions successfully !") diff --git a/src/punchCards.py b/src/punchCards.py index eb63ac8b..75a051d8 100644 --- a/src/punchCards.py +++ b/src/punchCards.py @@ -71,8 +71,8 @@ def completePunchCards(self): punchCard["parentPromotion"]["attributes"]["destination"], punchCard["childPromotions"], ) - except Exception: # pylint: disable=broad-except - logging.exception(Exception) + except Exception as e: # pylint: disable=broad-except + logging.warning(e) self.browser.utils.resetTabs() logging.info("[PUNCH CARDS] Completed the Punch Cards successfully !") time.sleep(random.randint(100, 700) / 100) From 94c845d5fcde1fcc55b7491d0b8be7b2d8ea0a70 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Thu, 13 Jun 2024 20:34:21 -0400 Subject: [PATCH 17/74] Fix exception logging --- src/searches.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/searches.py b/src/searches.py index 2cd7606b..ec597ad6 100644 --- a/src/searches.py +++ b/src/searches.py @@ -73,8 +73,8 @@ def getRelatedTerms(self, word: str) -> list[str]: headers={"User-agent": self.browser.userAgent}, ) return r.json()[1] - except Exception: # pylint: disable=broad-except - logging.warn(Exception) + except Exception as e: # pylint: disable=broad-except + logging.warning(e) return [word] def bingSearches(self, numberOfSearches: int, pointsCounter: int = 0): From b9f7460329e0df7a5917722c6620b10edae37128 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Thu, 13 Jun 2024 20:36:56 -0400 Subject: [PATCH 18/74] Reflect defaults in config.yaml --- config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config.yaml b/config.yaml index 88a273ba..4f998776 100644 --- a/config.yaml +++ b/config.yaml @@ -1,9 +1,9 @@ # config.yaml apprise: - summary: on_error + summary: always urls: - 'discord://WebhookID/WebhookToken' # Replace with your actual Apprise service URLs attempts: base_delay_in_seconds: 60 max: 6 - strategy: exponential + strategy: constant From e600cf5dc68d242c62b524b3c1f7a3d705dac847 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Fri, 14 Jun 2024 17:30:20 -0400 Subject: [PATCH 19/74] Include traceback in logs --- src/dailySet.py | 8 ++++---- src/login.py | 6 +++--- src/morePromotions.py | 4 ++-- src/punchCards.py | 4 ++-- src/searches.py | 4 ++-- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/dailySet.py b/src/dailySet.py index f0dda44b..ea6e90fd 100644 --- a/src/dailySet.py +++ b/src/dailySet.py @@ -80,12 +80,12 @@ def completeDailySet(self): try: # Try completing ABC activity self.activities.completeABC() - except Exception as e: # pylint: disable=broad-except - logging.warning(e) + except Exception: # pylint: disable=broad-except + logging.warning("", exc_info=True) # Default to completing quiz self.activities.completeQuiz() - except Exception as e: # pylint: disable=broad-except - logging.warning(e) + except Exception: # pylint: disable=broad-except + logging.warning("", exc_info=True) # Reset tabs in case of an exception self.browser.utils.resetTabs() logging.info("[DAILY SET] Completed the Daily Set successfully !") diff --git a/src/login.py b/src/login.py index 40995b1f..ef13e757 100644 --- a/src/login.py +++ b/src/login.py @@ -28,12 +28,12 @@ def login(self) -> int: alreadyLoggedIn = True break except Exception as e: # pylint: disable=broad-except - logging.warning(e) + logging.warning("", exc_info=True) try: self.utils.waitUntilVisible(By.ID, "i0116", 10) break - except Exception as e: # pylint: disable=broad-except - logging.warning(e) + except Exception: # pylint: disable=broad-except + logging.warning("", exc_info=True) if self.utils.tryDismissAllMessages(): continue diff --git a/src/morePromotions.py b/src/morePromotions.py index f73366ae..2005f54e 100644 --- a/src/morePromotions.py +++ b/src/morePromotions.py @@ -41,8 +41,8 @@ def completeMorePromotions(self): else: # Default to completing search self.activities.completeSearch() - except Exception as e: # pylint: disable=broad-except - logging.warning(e) + except Exception: # pylint: disable=broad-except + logging.warning("", exc_info=True) # Reset tabs in case of an exception self.browser.utils.resetTabs() logging.info("[MORE PROMOS] Completed More Promotions successfully !") diff --git a/src/punchCards.py b/src/punchCards.py index 75a051d8..938c7d2f 100644 --- a/src/punchCards.py +++ b/src/punchCards.py @@ -71,8 +71,8 @@ def completePunchCards(self): punchCard["parentPromotion"]["attributes"]["destination"], punchCard["childPromotions"], ) - except Exception as e: # pylint: disable=broad-except - logging.warning(e) + except Exception: # pylint: disable=broad-except + logging.warning("", exc_info=True) self.browser.utils.resetTabs() logging.info("[PUNCH CARDS] Completed the Punch Cards successfully !") time.sleep(random.randint(100, 700) / 100) diff --git a/src/searches.py b/src/searches.py index ec597ad6..645620c9 100644 --- a/src/searches.py +++ b/src/searches.py @@ -73,8 +73,8 @@ def getRelatedTerms(self, word: str) -> list[str]: headers={"User-agent": self.browser.userAgent}, ) return r.json()[1] - except Exception as e: # pylint: disable=broad-except - logging.warning(e) + except Exception: # pylint: disable=broad-except + logging.warning("", exc_info=True) return [word] def bingSearches(self, numberOfSearches: int, pointsCounter: int = 0): From ff447a052dd38cea3b85b875029f7c7c4ab6c3f2 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Fri, 14 Jun 2024 17:30:54 -0400 Subject: [PATCH 20/74] Add more and better type hints --- src/browser.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/browser.py b/src/browser.py index 102304ab..ae93d212 100644 --- a/src/browser.py +++ b/src/browser.py @@ -1,3 +1,4 @@ +import argparse import logging import random from pathlib import Path @@ -8,6 +9,8 @@ import seleniumwire.undetected_chromedriver as webdriver from selenium.webdriver import ChromeOptions from selenium.webdriver.chrome.webdriver import WebDriver +from seleniumwire import undetected_chromedriver +from seleniumwire.undetected_chromedriver import webdriver from src import Account from src.userAgentGenerator import GenerateUserAgent @@ -17,7 +20,7 @@ class Browser: """WebDriver wrapper class.""" - def __init__(self, mobile: bool, account: Account, args: Any) -> None: + def __init__(self, mobile: bool, account: Account, args: argparse.Namespace) -> None: # Initialize browser instance logging.debug("in __init__") self.mobile = mobile @@ -53,15 +56,15 @@ def __exit__(self, exc_type: Type[BaseException] | None, exc_value: BaseExceptio traceback: TracebackType | None) -> None: # Cleanup actions when exiting the browser context logging.debug(f"in __exit__ exc_type={exc_type} exc_value={exc_value} traceback={traceback}") - # self.webdriver.close() # just closes window, doesn't lose driver, see https://stackoverflow.com/a/32447644/4164390 + self.webdriver.close() # just closes window, doesn't lose driver, see https://stackoverflow.com/a/32447644/4164390 # self.webdriver.__exit__(None, None, None) # doesn't seem to work # doesn't work self.webdriver.quit() def browserSetup( self, - ) -> WebDriver: + ) -> undetected_chromedriver.webdriver.Chrome: # Configure and setup the Chrome browser - options = webdriver.ChromeOptions() + options = undetected_chromedriver.ChromeOptions() options.headless = self.headless options.add_argument(f"--lang={self.localeLang}") options.add_argument("--log-level=3") From c788c4eb22d2ac7e468397ae31b09f7a47d29ffe Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Fri, 14 Jun 2024 17:33:26 -0400 Subject: [PATCH 21/74] Wrap main in try-catch --- main.py | 41 ++++++++++++++++++++--------------------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/main.py b/main.py index 4eb71da8..f674109f 100644 --- a/main.py +++ b/main.py @@ -33,30 +33,23 @@ def main(): previous_points_data = load_previous_points_data() for currentAccount in loadedAccounts: - try: - earned_points = executeBot(currentAccount, args) - previous_points = previous_points_data.get(currentAccount.username, 0) + earned_points = executeBot(currentAccount, args) + previous_points = previous_points_data.get(currentAccount.username, 0) - # Calculate the difference in points from the prior day - points_difference = earned_points - previous_points + # Calculate the difference in points from the prior day + points_difference = earned_points - previous_points - # Append the daily points and points difference to CSV and Excel - log_daily_points_to_csv( - earned_points, points_difference - ) + # Append the daily points and points difference to CSV and Excel + log_daily_points_to_csv( + earned_points, points_difference + ) - # Update the previous day's points data - previous_points_data[currentAccount.username] = earned_points + # Update the previous day's points data + previous_points_data[currentAccount.username] = earned_points - logging.info( - f"[POINTS] Data for '{currentAccount.username}' appended to the file." - ) - except Exception as e: - Utils.sendNotification( - "⚠️ Error occurred, please check the log", f"{e}\n{e.__traceback__}" - ) - logging.exception(f"{e.__class__.__name__}: {e}") - exit(1) + logging.info( + f"[POINTS] Data for '{currentAccount.username}' appended to the file." + ) # Save the current day's points data for the next day in the "logs" folder save_previous_points_data(previous_points_data) @@ -319,4 +312,10 @@ def save_previous_points_data(data): if __name__ == "__main__": - main() + try: + main() + except Exception as e: + logging.exception("") + Utils.sendNotification( + "⚠️ Error occurred, please check the log", f"{e}\n{e.__traceback__}" + ) From 262d60231251a0a2e49c147301d777bbf1a9e009 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Fri, 14 Jun 2024 17:40:43 -0400 Subject: [PATCH 22/74] Reformat and fix exception logging --- main.py | 10 +++++----- src/browser.py | 30 ++++++++++++++++++++---------- 2 files changed, 25 insertions(+), 15 deletions(-) diff --git a/main.py b/main.py index f674109f..dbd400c7 100644 --- a/main.py +++ b/main.py @@ -40,9 +40,7 @@ def main(): points_difference = earned_points - previous_points # Append the daily points and points difference to CSV and Excel - log_daily_points_to_csv( - earned_points, points_difference - ) + log_daily_points_to_csv(earned_points, points_difference) # Update the previous day's points data previous_points_data[currentAccount.username] = earned_points @@ -255,8 +253,10 @@ def executeBot(currentAccount: Account, args: argparse.Namespace): f"[POINTS] You are now at {(utils.formatNumber((accountPointsCounter / goalPoints) * 100))}% of your " f"goal ({goalTitle}) !" ) - goalNotifier = (f"🎯 Goal reached: {(utils.formatNumber((accountPointsCounter / goalPoints) * 100))}%" - f" ({goalTitle})") + goalNotifier = ( + f"🎯 Goal reached: {(utils.formatNumber((accountPointsCounter / goalPoints) * 100))}%" + f" ({goalTitle})" + ) Utils.sendNotification( "Daily Points Update", diff --git a/src/browser.py b/src/browser.py index ae93d212..85855dc1 100644 --- a/src/browser.py +++ b/src/browser.py @@ -10,7 +10,7 @@ from selenium.webdriver import ChromeOptions from selenium.webdriver.chrome.webdriver import WebDriver from seleniumwire import undetected_chromedriver -from seleniumwire.undetected_chromedriver import webdriver +from seleniumwire.undetected_chromedriver import webdriver, Chrome from src import Account from src.userAgentGenerator import GenerateUserAgent @@ -20,7 +20,11 @@ class Browser: """WebDriver wrapper class.""" - def __init__(self, mobile: bool, account: Account, args: argparse.Namespace) -> None: + webdriver: Chrome + + def __init__( + self, mobile: bool, account: Account, args: argparse.Namespace + ) -> None: # Initialize browser instance logging.debug("in __init__") self.mobile = mobile @@ -52,12 +56,18 @@ def __enter__(self) -> "Browser": logging.debug("in __enter__") return self - def __exit__(self, exc_type: Type[BaseException] | None, exc_value: BaseException | None, - traceback: TracebackType | None) -> None: + def __exit__( + self, + exc_type: Type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> None: # Cleanup actions when exiting the browser context - logging.debug(f"in __exit__ exc_type={exc_type} exc_value={exc_value} traceback={traceback}") - self.webdriver.close() # just closes window, doesn't lose driver, see https://stackoverflow.com/a/32447644/4164390 - # self.webdriver.__exit__(None, None, None) # doesn't seem to work # doesn't work + logging.debug( + f"in __exit__ exc_type={exc_type} exc_value={exc_value} traceback={traceback}" + ) + # turns out close is needed for undetected_chromedriver + self.webdriver.close() self.webdriver.quit() def browserSetup( @@ -78,7 +88,7 @@ def browserSetup( options.add_argument("--disable-gpu") options.add_argument("--disable-default-apps") options.add_argument("--disable-features=Translate") - options.add_argument('--disable-features=PrivacySandboxSettings4') + options.add_argument("--disable-features=PrivacySandboxSettings4") seleniumwireOptions: dict[str, Any] = {"verify_ssl": False} @@ -198,8 +208,8 @@ def getCCodeLang(lang: str, geo: str) -> tuple: if geo is None: geo = nfo["country"] except Exception: # pylint: disable=broad-except - logging.debug(Exception) - return "en", "US" + logging.warning("", exc_info=True) + return "en", "US" return lang, geo def getChromeVersion(self) -> str: From be434f5cc95ef40a28d4582b35f410f06f563ce4 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Sat, 15 Jun 2024 13:45:33 -0400 Subject: [PATCH 23/74] Put sys back --- src/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/utils.py b/src/utils.py index 6e1a66fe..060b729d 100644 --- a/src/utils.py +++ b/src/utils.py @@ -1,6 +1,7 @@ import contextlib import json import locale as pylocale +import sys import time import urllib.parse from pathlib import Path From ca97462b7b583b6518a5b12b114fffab2698288b Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Sun, 16 Jun 2024 18:32:24 -0400 Subject: [PATCH 24/74] Reformat main --- main.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/main.py b/main.py index dbd400c7..f7b1a160 100644 --- a/main.py +++ b/main.py @@ -245,7 +245,9 @@ def executeBot(currentAccount: Account, args: argparse.Namespace): logging.info( f"[POINTS] You are now at {utils.formatNumber(accountPointsCounter)} points !" ) - appriseSummary = AppriseSummary[utils.config.get("apprise", {}).get("summary", AppriseSummary.always.name)] + appriseSummary = AppriseSummary[ + utils.config.get("apprise", {}).get("summary", AppriseSummary.always.name) + ] if appriseSummary == AppriseSummary.always: goalNotifier = "" if goalPoints > 0: @@ -296,9 +298,10 @@ def export_points_to_csv(points_data): # Define a function to load the previous day's points data from a file in the "logs" folder def load_previous_points_data(): - logs_directory = Utils.getProjectRoot() / "logs" try: - with open(logs_directory / "previous_points_data.json", "r") as file: + with open( + Utils.getProjectRoot() / "logs" / "previous_points_data.json", "r" + ) as file: return json.load(file) except FileNotFoundError: return {} From f13a682815aa331615def43b0608d74e3fe566bf Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Sun, 16 Jun 2024 18:54:20 -0400 Subject: [PATCH 25/74] Configure apprise summary via command line arg --- main.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/main.py b/main.py index f7b1a160..b7849bab 100644 --- a/main.py +++ b/main.py @@ -9,7 +9,7 @@ import sys import time from datetime import datetime -from enum import Enum, auto +from enum import Enum from src import ( Browser, @@ -141,6 +141,14 @@ def argumentParser() -> argparse.Namespace: default=None, help="Optional: Set fixed Chrome version (ex. 118)", ) + parser.add_argument( + "-ap", + "--apprise-summary", + type=AppriseSummary, + choices=list(AppriseSummary), + default=None, + help="Optional: Configure Apprise summary type, overrides config.yaml", + ) return parser.parse_args() @@ -180,8 +188,12 @@ def validEmail(email: str) -> bool: class AppriseSummary(Enum): - always = auto() - on_error = auto() + always = "always" + on_error = "on_error" + never = "never" + + def __str__(self): + return self.value def executeBot(currentAccount: Account, args: argparse.Namespace): @@ -277,6 +289,8 @@ def executeBot(currentAccount: Account, args: argparse.Namespace): "Error: remaining searches", f"account username: {currentAccount.username}, {remainingSearches}", ) + elif appriseSummary == AppriseSummary.never: + pass return accountPointsCounter From a286c0fab816ed617ab4362213ecc0c2e565acbf Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Sun, 16 Jun 2024 18:56:47 -0400 Subject: [PATCH 26/74] Share JetBrains run config --- .idea/runConfigurations/main.xml | 24 +++++++++++++++++++++++ .idea/runConfigurations/main_headless.xml | 24 +++++++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 .idea/runConfigurations/main.xml create mode 100644 .idea/runConfigurations/main_headless.xml diff --git a/.idea/runConfigurations/main.xml b/.idea/runConfigurations/main.xml new file mode 100644 index 00000000..042a23a4 --- /dev/null +++ b/.idea/runConfigurations/main.xml @@ -0,0 +1,24 @@ + + + + + \ No newline at end of file diff --git a/.idea/runConfigurations/main_headless.xml b/.idea/runConfigurations/main_headless.xml new file mode 100644 index 00000000..92b73deb --- /dev/null +++ b/.idea/runConfigurations/main_headless.xml @@ -0,0 +1,24 @@ + + + + + \ No newline at end of file From 08b551db10e3beeb9afc4c71b9827065e1dbf812 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Sun, 16 Jun 2024 18:57:43 -0400 Subject: [PATCH 27/74] Configure apprise summary via command line arg --- main.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/main.py b/main.py index b7849bab..1cd133e7 100644 --- a/main.py +++ b/main.py @@ -257,9 +257,13 @@ def executeBot(currentAccount: Account, args: argparse.Namespace): logging.info( f"[POINTS] You are now at {utils.formatNumber(accountPointsCounter)} points !" ) - appriseSummary = AppriseSummary[ - utils.config.get("apprise", {}).get("summary", AppriseSummary.always.name) - ] + appriseSummary: AppriseSummary + if args.apprise_summary is not None: + appriseSummary = args.apprise_summary + else: + appriseSummary = AppriseSummary[ + utils.config.get("apprise", {}).get("summary", AppriseSummary.always.name) + ] if appriseSummary == AppriseSummary.always: goalNotifier = "" if goalPoints > 0: From b2c44800b3c42f601265bbc247cb24fe42ed99bf Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Mon, 17 Jun 2024 13:32:01 -0400 Subject: [PATCH 28/74] Persist Google trends to disk and read/write --- src/searches.py | 30 ++++++++++++++++++++---------- src/utils.py | 4 +++- 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/src/searches.py b/src/searches.py index 645620c9..ff05b783 100644 --- a/src/searches.py +++ b/src/searches.py @@ -1,6 +1,7 @@ import json import logging import random +import shelve import time from datetime import date, timedelta from enum import Enum, auto @@ -14,6 +15,8 @@ from src.browser import Browser from src.utils import Utils, RemainingSearches +LOAD_DATE = "loadDate" + class AttemptsStrategy(Enum): exponential = auto() @@ -32,14 +35,19 @@ class Searches: def __init__(self, browser: Browser, searches: RemainingSearches): self.browser = browser self.webdriver = browser.webdriver - # Share search terms across instances to get rid of duplicates - if Searches.searchTerms is None: - Searches.searchTerms = self.getGoogleTrends( - searches.desktop + searches.mobile - ) - # Shuffle in case not only run of the day - random.shuffle(Searches.searchTerms) - # todo write shuffled searchTerms to disk to better emulate actual searches + + self.googleTrendsShelf: shelve.Shelf = shelve.open("google_trends") + loadDate: date | None = None + if LOAD_DATE in self.googleTrendsShelf: + loadDate = self.googleTrendsShelf[LOAD_DATE] + + if loadDate is None or loadDate != date.today(): + self.googleTrendsShelf.clear() + self.googleTrendsShelf[LOAD_DATE] = date.today() + trends = self.getGoogleTrends(searches.getTotal()) + random.shuffle(trends) + for trend in trends: + self.googleTrendsShelf[trend] = None def getGoogleTrends(self, wordsCount: int) -> list[str]: # Function to retrieve Google Trends search terms @@ -87,15 +95,15 @@ def bingSearches(self, numberOfSearches: int, pointsCounter: int = 0): for searchCount in range(1, numberOfSearches + 1): logging.info(f"[BING] {searchCount}/{numberOfSearches}") - searchTerm = Searches.searchTerms[0] + searchTerm = list(self.googleTrendsShelf.keys())[0] pointsCounter = self.bingSearch(searchTerm) - Searches.searchTerms.remove(searchTerm) if not Utils.isDebuggerAttached(): time.sleep(random.randint(10, 15)) logging.info( f"[BING] Finished {self.browser.browserType.capitalize()} Edge Bing searches !" ) + self.googleTrendsShelf.close() return pointsCounter def bingSearch(self, word: str) -> int: @@ -104,6 +112,7 @@ def bingSearch(self, word: str) -> int: wordsCycle: cycle[str] = cycle(self.getRelatedTerms(word)) baseDelay = Searches.baseDelay + originalWord = word for i in range(self.maxAttempts): try: @@ -128,6 +137,7 @@ def bingSearch(self, word: str) -> int: bingAccountPointsNow: int = self.browser.utils.getBingAccountPoints() if bingAccountPointsNow > bingAccountPointsBefore: + del self.googleTrendsShelf[originalWord] return bingAccountPointsNow raise TimeoutException diff --git a/src/utils.py b/src/utils.py index 060b729d..3fee14f4 100644 --- a/src/utils.py +++ b/src/utils.py @@ -1,7 +1,6 @@ import contextlib import json import locale as pylocale -import sys import time import urllib.parse from pathlib import Path @@ -22,6 +21,9 @@ class RemainingSearches(NamedTuple): desktop: int mobile: int + def getTotal(self) -> int: + return self.desktop + self.mobile + class Utils: def __init__(self, webdriver: WebDriver): From 0f4bc507a4cdb54048735eb2a2da154868223e58 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Tue, 18 Jun 2024 00:03:02 -0400 Subject: [PATCH 29/74] Remove isDebuggerAttached --- src/searches.py | 6 ++---- src/utils.py | 4 ---- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/src/searches.py b/src/searches.py index ff05b783..c168c74e 100644 --- a/src/searches.py +++ b/src/searches.py @@ -97,8 +97,7 @@ def bingSearches(self, numberOfSearches: int, pointsCounter: int = 0): logging.info(f"[BING] {searchCount}/{numberOfSearches}") searchTerm = list(self.googleTrendsShelf.keys())[0] pointsCounter = self.bingSearch(searchTerm) - if not Utils.isDebuggerAttached(): - time.sleep(random.randint(10, 15)) + time.sleep(random.randint(10, 15)) logging.info( f"[BING] Finished {self.browser.browserType.capitalize()} Edge Bing searches !" @@ -154,8 +153,7 @@ def bingSearch(self, word: str) -> int: f"[BING] Search attempt failed {i + 1}/{Searches.maxAttempts}, retrying after sleeping {baseDelay}" f" seconds..." ) - if not Utils.isDebuggerAttached(): - time.sleep(baseDelay) + time.sleep(baseDelay) if Searches.attemptsStrategy == AttemptsStrategy.exponential: baseDelay *= 2 diff --git a/src/utils.py b/src/utils.py index 3fee14f4..3e9dfbc6 100644 --- a/src/utils.py +++ b/src/utils.py @@ -302,7 +302,3 @@ def saveBrowserConfig(sessionPath: Path, config: dict): configFile = sessionPath.joinpath("config.json") with open(configFile, "w") as f: json.dump(config, f) - - @staticmethod - def isDebuggerAttached() -> bool: - return sys.gettrace() is not None \ No newline at end of file From 5e9172df7e279d51acec842b54c5a65fe50afafa Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Tue, 18 Jun 2024 00:16:18 -0400 Subject: [PATCH 30/74] Be more explicit --- src/searches.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/searches.py b/src/searches.py index c168c74e..5f6e9e7f 100644 --- a/src/searches.py +++ b/src/searches.py @@ -41,7 +41,7 @@ def __init__(self, browser: Browser, searches: RemainingSearches): if LOAD_DATE in self.googleTrendsShelf: loadDate = self.googleTrendsShelf[LOAD_DATE] - if loadDate is None or loadDate != date.today(): + if loadDate is None or loadDate < date.today(): self.googleTrendsShelf.clear() self.googleTrendsShelf[LOAD_DATE] = date.today() trends = self.getGoogleTrends(searches.getTotal()) From 0135714a3c8376472f5123544fef457b63aa727f Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Tue, 18 Jun 2024 00:16:33 -0400 Subject: [PATCH 31/74] Alter imports --- src/browser.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/browser.py b/src/browser.py index 85855dc1..eb7bcb03 100644 --- a/src/browser.py +++ b/src/browser.py @@ -7,10 +7,9 @@ import ipapi import seleniumwire.undetected_chromedriver as webdriver +import undetected_chromedriver from selenium.webdriver import ChromeOptions from selenium.webdriver.chrome.webdriver import WebDriver -from seleniumwire import undetected_chromedriver -from seleniumwire.undetected_chromedriver import webdriver, Chrome from src import Account from src.userAgentGenerator import GenerateUserAgent @@ -20,7 +19,7 @@ class Browser: """WebDriver wrapper class.""" - webdriver: Chrome + webdriver: undetected_chromedriver.Chrome def __init__( self, mobile: bool, account: Account, args: argparse.Namespace From 9972b0ebc593a63e957dcd768b07e935359e9af5 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Tue, 18 Jun 2024 00:21:17 -0400 Subject: [PATCH 32/74] Add warning logging and remove unused variable --- src/login.py | 2 +- src/utils.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/login.py b/src/login.py index ef13e757..cb1fcb3f 100644 --- a/src/login.py +++ b/src/login.py @@ -27,7 +27,7 @@ def login(self) -> int: ) alreadyLoggedIn = True break - except Exception as e: # pylint: disable=broad-except + except Exception: # pylint: disable=broad-except logging.warning("", exc_info=True) try: self.utils.waitUntilVisible(By.ID, "i0116", 10) diff --git a/src/utils.py b/src/utils.py index 3e9dfbc6..6c808b41 100644 --- a/src/utils.py +++ b/src/utils.py @@ -1,6 +1,7 @@ import contextlib import json import locale as pylocale +import logging import time import urllib.parse from pathlib import Path @@ -75,6 +76,7 @@ def waitForMSRewardElement(self, by: str, selector: str): self.webdriver.find_element(by, selector) return True except Exception: + logging.warning("", exc_info=True) if tries < checks: tries += 1 time.sleep(checkingInterval) @@ -132,6 +134,7 @@ def resetTabs(self): time.sleep(0.5) self.goHome() except Exception: + logging.warning("", exc_info=True) self.goHome() def goHome(self): @@ -226,9 +229,11 @@ def tryDismissAllMessages(self): for element in elements: element.click() except Exception: + logging.warning("", exc_info=True) continue result = True except Exception: + logging.warning("", exc_info=True) continue return result From 4b930c9859460c0c6993246d3f46ce0322f0688f Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Tue, 18 Jun 2024 09:16:16 -0400 Subject: [PATCH 33/74] Fix import error and return --- src/browser.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/browser.py b/src/browser.py index eb7bcb03..7b6afc07 100644 --- a/src/browser.py +++ b/src/browser.py @@ -71,7 +71,7 @@ def __exit__( def browserSetup( self, - ) -> undetected_chromedriver.webdriver.Chrome: + ) -> undetected_chromedriver.Chrome: # Configure and setup the Chrome browser options = undetected_chromedriver.ChromeOptions() options.headless = self.headless @@ -208,10 +208,11 @@ def getCCodeLang(lang: str, geo: str) -> tuple: geo = nfo["country"] except Exception: # pylint: disable=broad-except logging.warning("", exc_info=True) - return "en", "US" + return "en", "US" return lang, geo - def getChromeVersion(self) -> str: + @staticmethod + def getChromeVersion() -> str: chrome_options = ChromeOptions() chrome_options.add_argument("--headless=new") From 059ecd9b7fb88a27be21fe9b4e0f458c42c39fc2 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Tue, 18 Jun 2024 09:31:49 -0400 Subject: [PATCH 34/74] Add google_trends to .gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index be3cbb2b..3a797e99 100644 --- a/.gitignore +++ b/.gitignore @@ -183,3 +183,6 @@ sessions logs runbot.bat .DS_Store +/google_trends.dat +/google_trends.dir +/google_trends.bak From b737c884bb422eefc0e1cb168aaebedfd3b5e6b7 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Tue, 18 Jun 2024 16:57:05 -0400 Subject: [PATCH 35/74] Use new method --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index 1cd133e7..037fd7e7 100644 --- a/main.py +++ b/main.py @@ -288,7 +288,7 @@ def executeBot(currentAccount: Account, args: argparse.Namespace): ), ) elif appriseSummary == AppriseSummary.on_error: - if remainingSearches.desktop > 0 or remainingSearches.mobile > 0: + if remainingSearches.getTotal() > 0: Utils.sendNotification( "Error: remaining searches", f"account username: {currentAccount.username}, {remainingSearches}", From e862a0a5709abf058d7412348d552b3879ce381c Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Tue, 18 Jun 2024 20:34:20 -0400 Subject: [PATCH 36/74] Don't remove load date from google_trends; add logging --- src/searches.py | 328 ++++++++++++++++++++++++------------------------ 1 file changed, 166 insertions(+), 162 deletions(-) diff --git a/src/searches.py b/src/searches.py index 5f6e9e7f..934829cc 100644 --- a/src/searches.py +++ b/src/searches.py @@ -1,162 +1,166 @@ -import json -import logging -import random -import shelve -import time -from datetime import date, timedelta -from enum import Enum, auto -from itertools import cycle - -import requests -from selenium.common.exceptions import TimeoutException -from selenium.webdriver.common.by import By -from selenium.webdriver.remote.webelement import WebElement - -from src.browser import Browser -from src.utils import Utils, RemainingSearches - -LOAD_DATE = "loadDate" - - -class AttemptsStrategy(Enum): - exponential = auto() - constant = auto() - - -class Searches: - config = Utils.loadConfig() - maxAttempts: int = config.get("attempts", {}).get("max", 6) - baseDelay: int = config.get("attempts", {}).get("base_delay_in_seconds", 60) - attemptsStrategy = AttemptsStrategy[ - config.get("attempts", {}).get("strategy", AttemptsStrategy.constant.name) - ] - searchTerms: list[str] | None = None - - def __init__(self, browser: Browser, searches: RemainingSearches): - self.browser = browser - self.webdriver = browser.webdriver - - self.googleTrendsShelf: shelve.Shelf = shelve.open("google_trends") - loadDate: date | None = None - if LOAD_DATE in self.googleTrendsShelf: - loadDate = self.googleTrendsShelf[LOAD_DATE] - - if loadDate is None or loadDate < date.today(): - self.googleTrendsShelf.clear() - self.googleTrendsShelf[LOAD_DATE] = date.today() - trends = self.getGoogleTrends(searches.getTotal()) - random.shuffle(trends) - for trend in trends: - self.googleTrendsShelf[trend] = None - - def getGoogleTrends(self, wordsCount: int) -> list[str]: - # Function to retrieve Google Trends search terms - searchTerms: list[str] = [] - i = 0 - while len(searchTerms) < wordsCount: - i += 1 - # Fetching daily trends from Google Trends API - r = requests.get( - f"https://trends.google.com/trends/api/dailytrends?hl={self.browser.localeLang}" - f'&ed={(date.today() - timedelta(days=i)).strftime("%Y%m%d")}&geo={self.browser.localeGeo}&ns=15' - ) - trends = json.loads(r.text[6:]) - for topic in trends["default"]["trendingSearchesDays"][0][ - "trendingSearches" - ]: - searchTerms.append(topic["title"]["query"].lower()) - searchTerms.extend( - relatedTopic["query"].lower() - for relatedTopic in topic["relatedQueries"] - ) - searchTerms = list(set(searchTerms)) - del searchTerms[wordsCount : (len(searchTerms) + 1)] - return searchTerms - - def getRelatedTerms(self, word: str) -> list[str]: - # Function to retrieve related terms from Bing API - try: - r = requests.get( - f"https://api.bing.com/osjson.aspx?query={word}", - headers={"User-agent": self.browser.userAgent}, - ) - return r.json()[1] - except Exception: # pylint: disable=broad-except - logging.warning("", exc_info=True) - return [word] - - def bingSearches(self, numberOfSearches: int, pointsCounter: int = 0): - # Function to perform Bing searches - logging.info( - f"[BING] Starting {self.browser.browserType.capitalize()} Edge Bing searches..." - ) - - self.webdriver.get("https://bing.com") - - for searchCount in range(1, numberOfSearches + 1): - logging.info(f"[BING] {searchCount}/{numberOfSearches}") - searchTerm = list(self.googleTrendsShelf.keys())[0] - pointsCounter = self.bingSearch(searchTerm) - time.sleep(random.randint(10, 15)) - - logging.info( - f"[BING] Finished {self.browser.browserType.capitalize()} Edge Bing searches !" - ) - self.googleTrendsShelf.close() - return pointsCounter - - def bingSearch(self, word: str) -> int: - # Function to perform a single Bing search - bingAccountPointsBefore: int = self.browser.utils.getBingAccountPoints() - - wordsCycle: cycle[str] = cycle(self.getRelatedTerms(word)) - baseDelay = Searches.baseDelay - originalWord = word - - for i in range(self.maxAttempts): - try: - searchbar: WebElement - for _ in range(100): # todo make configurable - self.browser.utils.waitUntilClickable(By.ID, "sb_form_q") - searchbar = self.webdriver.find_element(By.ID, "sb_form_q") - searchbar.clear() - word = next(wordsCycle) - logging.debug(f"word={word}") - searchbar.send_keys(word) - typed_word = searchbar.get_attribute("value") - if typed_word == word: - break - logging.debug(f"typed_word != word, {typed_word} != {word}") - self.browser.webdriver.refresh() - else: - raise Exception("Problem sending words to searchbar") - - searchbar.submit() - time.sleep(2) # wait a bit for search to complete - - bingAccountPointsNow: int = self.browser.utils.getBingAccountPoints() - if bingAccountPointsNow > bingAccountPointsBefore: - del self.googleTrendsShelf[originalWord] - return bingAccountPointsNow - - raise TimeoutException - - except TimeoutException: - # todo - # if i == (maxAttempts / 2): - # logging.info("[BING] " + "TIMED OUT GETTING NEW PROXY") - # self.webdriver.proxy = self.browser.giveMeProxy() - self.browser.utils.tryDismissAllMessages() - - baseDelay += random.randint(1, 10) # add some jitter - logging.debug( - f"[BING] Search attempt failed {i + 1}/{Searches.maxAttempts}, retrying after sleeping {baseDelay}" - f" seconds..." - ) - time.sleep(baseDelay) - - if Searches.attemptsStrategy == AttemptsStrategy.exponential: - baseDelay *= 2 - # todo debug why we get to this point occasionally even though searches complete - logging.error("[BING] Reached max search attempt retries") - return bingAccountPointsBefore +import json +import logging +import random +import shelve +import time +from datetime import date, timedelta +from enum import Enum, auto +from itertools import cycle + +import requests +from selenium.common.exceptions import TimeoutException +from selenium.webdriver.common.by import By +from selenium.webdriver.remote.webelement import WebElement + +from src.browser import Browser +from src.utils import Utils, RemainingSearches + +LOAD_DATE_KEY = "loadDate" + + +class AttemptsStrategy(Enum): + exponential = auto() + constant = auto() + + +class Searches: + config = Utils.loadConfig() + maxAttempts: int = config.get("attempts", {}).get("max", 6) + baseDelay: int = config.get("attempts", {}).get("base_delay_in_seconds", 60) + attemptsStrategy = AttemptsStrategy[ + config.get("attempts", {}).get("strategy", AttemptsStrategy.constant.name) + ] + searchTerms: list[str] | None = None + + def __init__(self, browser: Browser, searches: RemainingSearches): + self.browser = browser + self.webdriver = browser.webdriver + + self.googleTrendsShelf: shelve.Shelf = shelve.open("google_trends") + logging.debug(f"Before load = {list(self.googleTrendsShelf.items())}") + loadDate: date | None = None + if LOAD_DATE_KEY in self.googleTrendsShelf: + loadDate = self.googleTrendsShelf[LOAD_DATE_KEY] + + if loadDate is None or loadDate < date.today(): + self.googleTrendsShelf.clear() + self.googleTrendsShelf[LOAD_DATE_KEY] = date.today() + trends = self.getGoogleTrends(searches.getTotal()) + random.shuffle(trends) + for trend in trends: + self.googleTrendsShelf[trend] = None + logging.debug(f"After load = {list(self.googleTrendsShelf.items())}") + + def getGoogleTrends(self, wordsCount: int) -> list[str]: + # Function to retrieve Google Trends search terms + searchTerms: list[str] = [] + i = 0 + while len(searchTerms) < wordsCount: + i += 1 + # Fetching daily trends from Google Trends API + r = requests.get( + f"https://trends.google.com/trends/api/dailytrends?hl={self.browser.localeLang}" + f'&ed={(date.today() - timedelta(days=i)).strftime("%Y%m%d")}&geo={self.browser.localeGeo}&ns=15' + ) + trends = json.loads(r.text[6:]) + for topic in trends["default"]["trendingSearchesDays"][0][ + "trendingSearches" + ]: + searchTerms.append(topic["title"]["query"].lower()) + searchTerms.extend( + relatedTopic["query"].lower() + for relatedTopic in topic["relatedQueries"] + ) + searchTerms = list(set(searchTerms)) + del searchTerms[wordsCount : (len(searchTerms) + 1)] + return searchTerms + + def getRelatedTerms(self, word: str) -> list[str]: + # Function to retrieve related terms from Bing API + try: + r = requests.get( + f"https://api.bing.com/osjson.aspx?query={word}", + headers={"User-agent": self.browser.userAgent}, + ) + return r.json()[1] + except Exception: # pylint: disable=broad-except + logging.warning("", exc_info=True) + return [word] + + def bingSearches(self, numberOfSearches: int, pointsCounter: int = 0): + # Function to perform Bing searches + logging.info( + f"[BING] Starting {self.browser.browserType.capitalize()} Edge Bing searches..." + ) + + self.webdriver.get("https://bing.com") + + for searchCount in range(1, numberOfSearches + 1): + logging.info(f"[BING] {searchCount}/{numberOfSearches}") + googleTrends: list[str] = list(self.googleTrendsShelf.keys()) + logging.debug(f"self.googleTrendsShelf.keys() = {googleTrends}") + searchTerm = list(self.googleTrendsShelf.keys())[1] + pointsCounter = self.bingSearch(searchTerm) + time.sleep(random.randint(10, 15)) + + logging.info( + f"[BING] Finished {self.browser.browserType.capitalize()} Edge Bing searches !" + ) + self.googleTrendsShelf.close() + return pointsCounter + + def bingSearch(self, word: str) -> int: + # Function to perform a single Bing search + bingAccountPointsBefore: int = self.browser.utils.getBingAccountPoints() + + wordsCycle: cycle[str] = cycle(self.getRelatedTerms(word)) + baseDelay = Searches.baseDelay + originalWord = word + + for i in range(self.maxAttempts): + try: + searchbar: WebElement + for _ in range(100): # todo make configurable + self.browser.utils.waitUntilClickable(By.ID, "sb_form_q") + searchbar = self.webdriver.find_element(By.ID, "sb_form_q") + searchbar.clear() + word = next(wordsCycle) + logging.debug(f"word={word}") + searchbar.send_keys(word) + typed_word = searchbar.get_attribute("value") + if typed_word == word: + break + logging.debug(f"typed_word != word, {typed_word} != {word}") + self.browser.webdriver.refresh() + else: + raise Exception("Problem sending words to searchbar") + + searchbar.submit() + time.sleep(2) # wait a bit for search to complete + + bingAccountPointsNow: int = self.browser.utils.getBingAccountPoints() + if bingAccountPointsNow > bingAccountPointsBefore: + del self.googleTrendsShelf[originalWord] + return bingAccountPointsNow + + raise TimeoutException + + except TimeoutException: + # todo + # if i == (maxAttempts / 2): + # logging.info("[BING] " + "TIMED OUT GETTING NEW PROXY") + # self.webdriver.proxy = self.browser.giveMeProxy() + self.browser.utils.tryDismissAllMessages() + + baseDelay += random.randint(1, 10) # add some jitter + logging.debug( + f"[BING] Search attempt failed {i + 1}/{Searches.maxAttempts}, retrying after sleeping {baseDelay}" + f" seconds..." + ) + time.sleep(baseDelay) + + if Searches.attemptsStrategy == AttemptsStrategy.exponential: + baseDelay *= 2 + # todo debug why we get to this point occasionally even though searches complete + logging.error("[BING] Reached max search attempt retries") + return bingAccountPointsBefore From 4051e258203243beb86b80530b0e6af9307e5dac Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Tue, 18 Jun 2024 20:35:07 -0400 Subject: [PATCH 37/74] Attempt fix where actual points aren't correct --- src/searches.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/searches.py b/src/searches.py index 934829cc..dc14bd0a 100644 --- a/src/searches.py +++ b/src/searches.py @@ -138,6 +138,7 @@ def bingSearch(self, word: str) -> int: searchbar.submit() time.sleep(2) # wait a bit for search to complete + self.browser.webdriver.refresh() # or scroll so points update? bingAccountPointsNow: int = self.browser.utils.getBingAccountPoints() if bingAccountPointsNow > bingAccountPointsBefore: del self.googleTrendsShelf[originalWord] @@ -162,5 +163,6 @@ def bingSearch(self, word: str) -> int: if Searches.attemptsStrategy == AttemptsStrategy.exponential: baseDelay *= 2 # todo debug why we get to this point occasionally even though searches complete + # update - Seems like account points aren't refreshing correctly see logging.error("[BING] Reached max search attempt retries") return bingAccountPointsBefore From e42d9c4dd64545ea36b2ab973d55414f4f8da7e8 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Tue, 18 Jun 2024 22:27:48 -0400 Subject: [PATCH 38/74] Update logging --- src/searches.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/searches.py b/src/searches.py index dc14bd0a..92b516cb 100644 --- a/src/searches.py +++ b/src/searches.py @@ -30,14 +30,13 @@ class Searches: attemptsStrategy = AttemptsStrategy[ config.get("attempts", {}).get("strategy", AttemptsStrategy.constant.name) ] - searchTerms: list[str] | None = None def __init__(self, browser: Browser, searches: RemainingSearches): self.browser = browser self.webdriver = browser.webdriver self.googleTrendsShelf: shelve.Shelf = shelve.open("google_trends") - logging.debug(f"Before load = {list(self.googleTrendsShelf.items())}") + logging.debug(f"google_trends = {list(self.googleTrendsShelf.items())}") loadDate: date | None = None if LOAD_DATE_KEY in self.googleTrendsShelf: loadDate = self.googleTrendsShelf[LOAD_DATE_KEY] @@ -49,7 +48,9 @@ def __init__(self, browser: Browser, searches: RemainingSearches): random.shuffle(trends) for trend in trends: self.googleTrendsShelf[trend] = None - logging.debug(f"After load = {list(self.googleTrendsShelf.items())}") + logging.debug( + f"google_trends after load = {list(self.googleTrendsShelf.items())}" + ) def getGoogleTrends(self, wordsCount: int) -> list[str]: # Function to retrieve Google Trends search terms From 5d27c436aed4173a05adbf60bb346a483e57d462 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Tue, 18 Jun 2024 22:58:21 -0400 Subject: [PATCH 39/74] Print 2FA to console regardless of log level --- src/login.py | 264 +++++++++++++++++++++++++-------------------------- 1 file changed, 132 insertions(+), 132 deletions(-) diff --git a/src/login.py b/src/login.py index cb1fcb3f..8a60cf7e 100644 --- a/src/login.py +++ b/src/login.py @@ -1,132 +1,132 @@ -import contextlib -import logging -import time -import urllib.parse - -from selenium.webdriver.common.by import By - -from src.browser import Browser - - -class Login: - def __init__(self, browser: Browser): - self.browser = browser - self.webdriver = browser.webdriver - self.utils = browser.utils - - def login(self) -> int: - logging.info("[LOGIN] " + "Logging-in...") - self.webdriver.get( - "https://rewards.bing.com/Signin/" - ) # changed site to allow bypassing when M$ blocks access to login.live.com randomly - alreadyLoggedIn = False - while True: - try: - self.utils.waitUntilVisible( - By.CSS_SELECTOR, 'html[data-role-name="RewardsPortal"]', 0.1 - ) - alreadyLoggedIn = True - break - except Exception: # pylint: disable=broad-except - logging.warning("", exc_info=True) - try: - self.utils.waitUntilVisible(By.ID, "i0116", 10) - break - except Exception: # pylint: disable=broad-except - logging.warning("", exc_info=True) - if self.utils.tryDismissAllMessages(): - continue - - if not alreadyLoggedIn: - if isLocked := self.executeLogin(): - return "Locked" - self.utils.tryDismissCookieBanner() - - logging.info("[LOGIN] " + "Logged-in !") - - self.utils.goHome() - points = self.utils.getAccountPoints() - - logging.info("[LOGIN] " + "Ensuring you are logged into Bing...") - self.checkBingLogin() - logging.info("[LOGIN] Logged-in successfully !") - return points - - def executeLogin(self): - self.utils.waitUntilVisible(By.ID, "i0116", 10) - logging.info("[LOGIN] " + "Entering email...") - self.utils.waitUntilClickable(By.NAME, "loginfmt", 10) - email_field = self.webdriver.find_element(By.NAME, "loginfmt") - - while True: - email_field.send_keys(self.browser.username) - time.sleep(3) - if email_field.get_attribute("value") == self.browser.username: - self.webdriver.find_element(By.ID, "idSIButton9").click() - break - - email_field.clear() - time.sleep(3) - - try: - self.enterPassword(self.browser.password) - except Exception: # pylint: disable=broad-except - logging.info("[LOGIN] " + "2FA Code required !") - with contextlib.suppress(Exception): - code = self.webdriver.find_element( - By.ID, "idRemoteNGC_DisplaySign" - ).get_attribute("innerHTML") - logging.info(f"[LOGIN] 2FA code: {code}") - logging.info("[LOGIN] Press enter when confirmed on your device...") - input() - - while not ( - urllib.parse.urlparse(self.webdriver.current_url).path == "/" - and urllib.parse.urlparse(self.webdriver.current_url).hostname - == "account.microsoft.com" - ): - if urllib.parse.urlparse(self.webdriver.current_url).hostname == "rewards.bing.com": - self.webdriver.get("https://account.microsoft.com") - - if "Abuse" in str(self.webdriver.current_url): - logging.error(f"[LOGIN] {self.browser.username} is locked") - return True - self.utils.tryDismissAllMessages() - time.sleep(1) - - self.utils.waitUntilVisible( - By.CSS_SELECTOR, 'html[data-role-name="MeePortal"]', 10 - ) - - def enterPassword(self, password): - self.utils.waitUntilClickable(By.NAME, "passwd", 10) - self.utils.waitUntilClickable(By.ID, "idSIButton9", 10) - - logging.info("[LOGIN] " + "Writing password...") - - password_field = self.webdriver.find_element(By.NAME, "passwd") - - while True: - password_field.send_keys(password) - time.sleep(3) - if password_field.get_attribute("value") == password: - self.webdriver.find_element(By.ID, "idSIButton9").click() - break - - password_field.clear() - time.sleep(3) - time.sleep(3) - - def checkBingLogin(self): - self.webdriver.get( - "https://www.bing.com/fd/auth/signin?action=interactive&provider=windows_live_id&return_url=https%3A%2F%2Fwww.bing.com%2F" - ) - while True: - currentUrl = urllib.parse.urlparse(self.webdriver.current_url) - if currentUrl.hostname == "www.bing.com" and currentUrl.path == "/": - time.sleep(3) - self.utils.tryDismissBingCookieBanner() - with contextlib.suppress(Exception): - if self.utils.checkBingLogin(): - return - time.sleep(1) +import contextlib +import logging +import time +import urllib.parse + +from selenium.webdriver.common.by import By + +from src.browser import Browser + + +class Login: + def __init__(self, browser: Browser): + self.browser = browser + self.webdriver = browser.webdriver + self.utils = browser.utils + + def login(self) -> int: + logging.info("[LOGIN] " + "Logging-in...") + self.webdriver.get( + "https://rewards.bing.com/Signin/" + ) # changed site to allow bypassing when M$ blocks access to login.live.com randomly + alreadyLoggedIn = False + while True: + try: + self.utils.waitUntilVisible( + By.CSS_SELECTOR, 'html[data-role-name="RewardsPortal"]', 0.1 + ) + alreadyLoggedIn = True + break + except Exception: # pylint: disable=broad-except + logging.warning("", exc_info=True) + try: + self.utils.waitUntilVisible(By.ID, "i0116", 10) + break + except Exception: # pylint: disable=broad-except + logging.warning("", exc_info=True) + if self.utils.tryDismissAllMessages(): + continue + + if not alreadyLoggedIn: + if isLocked := self.executeLogin(): + return "Locked" + self.utils.tryDismissCookieBanner() + + logging.info("[LOGIN] " + "Logged-in !") + + self.utils.goHome() + points = self.utils.getAccountPoints() + + logging.info("[LOGIN] " + "Ensuring you are logged into Bing...") + self.checkBingLogin() + logging.info("[LOGIN] Logged-in successfully !") + return points + + def executeLogin(self): + self.utils.waitUntilVisible(By.ID, "i0116", 10) + logging.info("[LOGIN] " + "Entering email...") + self.utils.waitUntilClickable(By.NAME, "loginfmt", 10) + email_field = self.webdriver.find_element(By.NAME, "loginfmt") + + while True: + email_field.send_keys(self.browser.username) + time.sleep(3) + if email_field.get_attribute("value") == self.browser.username: + self.webdriver.find_element(By.ID, "idSIButton9").click() + break + + email_field.clear() + time.sleep(3) + + try: + self.enterPassword(self.browser.password) + except Exception: # pylint: disable=broad-except + print("[LOGIN] 2FA Code required !") + with contextlib.suppress(Exception): + code = self.webdriver.find_element( + By.ID, "idRemoteNGC_DisplaySign" + ).get_attribute("innerHTML") + logging.info(f"[LOGIN] 2FA code: {code}") + print("[LOGIN] Press enter when confirmed on your device...") + input() + + while not ( + urllib.parse.urlparse(self.webdriver.current_url).path == "/" + and urllib.parse.urlparse(self.webdriver.current_url).hostname + == "account.microsoft.com" + ): + if urllib.parse.urlparse(self.webdriver.current_url).hostname == "rewards.bing.com": + self.webdriver.get("https://account.microsoft.com") + + if "Abuse" in str(self.webdriver.current_url): + logging.error(f"[LOGIN] {self.browser.username} is locked") + return True + self.utils.tryDismissAllMessages() + time.sleep(1) + + self.utils.waitUntilVisible( + By.CSS_SELECTOR, 'html[data-role-name="MeePortal"]', 10 + ) + + def enterPassword(self, password): + self.utils.waitUntilClickable(By.NAME, "passwd", 10) + self.utils.waitUntilClickable(By.ID, "idSIButton9", 10) + + logging.info("[LOGIN] " + "Writing password...") + + password_field = self.webdriver.find_element(By.NAME, "passwd") + + while True: + password_field.send_keys(password) + time.sleep(3) + if password_field.get_attribute("value") == password: + self.webdriver.find_element(By.ID, "idSIButton9").click() + break + + password_field.clear() + time.sleep(3) + time.sleep(3) + + def checkBingLogin(self): + self.webdriver.get( + "https://www.bing.com/fd/auth/signin?action=interactive&provider=windows_live_id&return_url=https%3A%2F%2Fwww.bing.com%2F" + ) + while True: + currentUrl = urllib.parse.urlparse(self.webdriver.current_url) + if currentUrl.hostname == "www.bing.com" and currentUrl.path == "/": + time.sleep(3) + self.utils.tryDismissBingCookieBanner() + with contextlib.suppress(Exception): + if self.utils.checkBingLogin(): + return + time.sleep(1) From f3f70fe3bfb2bc2c666cecf3c1cdbc7f08440a5f Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Tue, 18 Jun 2024 22:59:57 -0400 Subject: [PATCH 40/74] Raise exceptions versus return error codes; add more type hints; try-except all accounts again --- main.py | 21 +++++++++------------ src/login.py | 26 ++++++++++++++++---------- src/utils.py | 8 ++++++-- 3 files changed, 31 insertions(+), 24 deletions(-) diff --git a/main.py b/main.py index 037fd7e7..9cd58923 100644 --- a/main.py +++ b/main.py @@ -33,7 +33,13 @@ def main(): previous_points_data = load_previous_points_data() for currentAccount in loadedAccounts: - earned_points = executeBot(currentAccount, args) + try: + earned_points = executeBot(currentAccount, args) + except Exception as e: + logging.error("", exc_info=True) + Utils.sendNotification(f"⚠️ Error executing {currentAccount.username}, please check the log", + f"{e}\n{e.__traceback__}") + continue previous_points = previous_points_data.get(currentAccount.username, 0) # Calculate the difference in points from the prior day @@ -205,18 +211,9 @@ def executeBot(currentAccount: Account, args: argparse.Namespace): with Browser(mobile=False, account=currentAccount, args=args) as desktopBrowser: utils = desktopBrowser.utils - accountPointsCounter = Login(desktopBrowser).login() - startingPoints = accountPointsCounter - if startingPoints == "Locked": - Utils.sendNotification("🚫 Account is Locked", currentAccount.username) - return 0 - if startingPoints == "Verify": - Utils.sendNotification( - "❗️ Account needs to be verified", currentAccount.username - ) - return 0 + startingPoints = Login(desktopBrowser).login() logging.info( - f"[POINTS] You have {utils.formatNumber(accountPointsCounter)} points on your account" + f"[POINTS] You have {utils.formatNumber(startingPoints)} points on your account" ) # todo - make quicker if done DailySet(desktopBrowser).completeDailySet() diff --git a/src/login.py b/src/login.py index 8a60cf7e..e8993631 100644 --- a/src/login.py +++ b/src/login.py @@ -8,6 +8,10 @@ from src.browser import Browser +class AccountLockedException(Exception): + pass + + class Login: def __init__(self, browser: Browser): self.browser = browser @@ -38,8 +42,7 @@ def login(self) -> int: continue if not alreadyLoggedIn: - if isLocked := self.executeLogin(): - return "Locked" + self.executeLogin() self.utils.tryDismissCookieBanner() logging.info("[LOGIN] " + "Logged-in !") @@ -52,7 +55,7 @@ def login(self) -> int: logging.info("[LOGIN] Logged-in successfully !") return points - def executeLogin(self): + def executeLogin(self) -> None: self.utils.waitUntilVisible(By.ID, "i0116", 10) logging.info("[LOGIN] " + "Entering email...") self.utils.waitUntilClickable(By.NAME, "loginfmt", 10) @@ -85,12 +88,14 @@ def executeLogin(self): and urllib.parse.urlparse(self.webdriver.current_url).hostname == "account.microsoft.com" ): - if urllib.parse.urlparse(self.webdriver.current_url).hostname == "rewards.bing.com": + if ( + urllib.parse.urlparse(self.webdriver.current_url).hostname + == "rewards.bing.com" + ): self.webdriver.get("https://account.microsoft.com") - + if "Abuse" in str(self.webdriver.current_url): - logging.error(f"[LOGIN] {self.browser.username} is locked") - return True + raise AccountLockedException self.utils.tryDismissAllMessages() time.sleep(1) @@ -98,7 +103,7 @@ def executeLogin(self): By.CSS_SELECTOR, 'html[data-role-name="MeePortal"]', 10 ) - def enterPassword(self, password): + def enterPassword(self, password) -> None: self.utils.waitUntilClickable(By.NAME, "passwd", 10) self.utils.waitUntilClickable(By.ID, "idSIButton9", 10) @@ -117,9 +122,10 @@ def enterPassword(self, password): time.sleep(3) time.sleep(3) - def checkBingLogin(self): + def checkBingLogin(self) -> None: self.webdriver.get( - "https://www.bing.com/fd/auth/signin?action=interactive&provider=windows_live_id&return_url=https%3A%2F%2Fwww.bing.com%2F" + "https://www.bing.com/fd/auth/signin?action=interactive&provider=windows_live_id&return_url=https%3A%2F" + "%2Fwww.bing.com%2F" ) while True: currentUrl = urllib.parse.urlparse(self.webdriver.current_url) diff --git a/src/utils.py b/src/utils.py index 6c808b41..8a16b942 100644 --- a/src/utils.py +++ b/src/utils.py @@ -18,6 +18,10 @@ from .constants import BASE_URL +class VerifyAccountException(Exception): + pass + + class RemainingSearches(NamedTuple): desktop: int mobile: int @@ -137,7 +141,7 @@ def resetTabs(self): logging.warning("", exc_info=True) self.goHome() - def goHome(self): + def goHome(self) -> None: reloadThreshold = 5 reloadInterval = 10 targetUrl = urllib.parse.urlparse(BASE_URL) @@ -158,7 +162,7 @@ def goHome(self): self.webdriver.get(BASE_URL) time.sleep(interval) if "proofs" in str(self.webdriver.current_url): - return "Verify" + raise VerifyAccountException intervalCount += 1 if intervalCount >= reloadInterval: intervalCount = 0 From b32231757d2ee49296174dcba912232ad73fa647 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Tue, 18 Jun 2024 23:09:26 -0400 Subject: [PATCH 41/74] Raise exceptions in exceptional conditions; add more typing --- src/utils.py | 93 ++++++++++++++++++---------------------------------- 1 file changed, 32 insertions(+), 61 deletions(-) diff --git a/src/utils.py b/src/utils.py index 8a16b942..44271818 100644 --- a/src/utils.py +++ b/src/utils.py @@ -5,7 +5,7 @@ import time import urllib.parse from pathlib import Path -from typing import NamedTuple +from typing import NamedTuple, Any import requests import yaml @@ -49,24 +49,26 @@ def loadConfig(config_file=getProjectRoot() / "config.yaml") -> dict: return yaml.safe_load(file) @staticmethod - def sendNotification(title, body): + def sendNotification(title, body) -> None: apprise = Apprise() urls: list[str] = Utils.loadConfig().get("apprise", {}).get("urls", []) for url in urls: apprise.add(url) apprise.notify(body=body, title=title) - def waitUntilVisible(self, by: str, selector: str, timeToWait: float = 10): + def waitUntilVisible(self, by: str, selector: str, timeToWait: float = 10) -> None: WebDriverWait(self.webdriver, timeToWait).until( ec.visibility_of_element_located((by, selector)) ) - def waitUntilClickable(self, by: str, selector: str, timeToWait: float = 10): + def waitUntilClickable( + self, by: str, selector: str, timeToWait: float = 10 + ) -> None: WebDriverWait(self.webdriver, timeToWait).until( ec.element_to_be_clickable((by, selector)) ) - def waitForMSRewardElement(self, by: str, selector: str): + def waitForMSRewardElement(self, by: str, selector: str) -> None: loadingTimeAllowed = 5 refreshesAllowed = 5 @@ -78,7 +80,7 @@ def waitForMSRewardElement(self, by: str, selector: str): while True: try: self.webdriver.find_element(by, selector) - return True + return except Exception: logging.warning("", exc_info=True) if tries < checks: @@ -90,56 +92,27 @@ def waitForMSRewardElement(self, by: str, selector: str): tries = 0 time.sleep(5) else: - return False + raise Exception - def waitUntilQuestionRefresh(self): + def waitUntilQuestionRefresh(self) -> None: return self.waitForMSRewardElement(By.CLASS_NAME, "rqECredits") - def waitUntilQuizLoads(self): + def waitUntilQuizLoads(self) -> None: return self.waitForMSRewardElement(By.XPATH, '//*[@id="rqStartQuiz"]') - def waitUntilJS(self, jsSrc: str): - loadingTimeAllowed = 5 - refreshesAllowed = 5 + def resetTabs(self) -> None: + curr = self.webdriver.current_window_handle - checkingInterval = 0.5 - checks = loadingTimeAllowed / checkingInterval + for handle in self.webdriver.window_handles: + if handle != curr: + self.webdriver.switch_to.window(handle) + time.sleep(0.5) + self.webdriver.close() + time.sleep(0.5) - tries = 0 - refreshCount = 0 - while True: - elem = self.webdriver.execute_script(jsSrc) - if elem: - return elem - - if tries < checks: - tries += 1 - time.sleep(checkingInterval) - elif refreshCount < refreshesAllowed: - self.webdriver.refresh() - refreshCount += 1 - tries = 0 - time.sleep(5) - else: - return elem - - def resetTabs(self): - try: - curr = self.webdriver.current_window_handle - - for handle in self.webdriver.window_handles: - if handle != curr: - self.webdriver.switch_to.window(handle) - time.sleep(0.5) - self.webdriver.close() - time.sleep(0.5) - - self.webdriver.switch_to.window(curr) - time.sleep(0.5) - self.goHome() - except Exception: - logging.warning("", exc_info=True) - self.goHome() + self.webdriver.switch_to.window(curr) + time.sleep(0.5) + self.goHome() def goHome(self) -> None: reloadThreshold = 5 @@ -151,9 +124,7 @@ def goHome(self) -> None: intervalCount = 0 while True: self.tryDismissCookieBanner() - with contextlib.suppress(Exception): - self.webdriver.find_element(By.ID, "more-activities") - break + self.webdriver.find_element(By.ID, "more-activities") currentUrl = urllib.parse.urlparse(self.webdriver.current_url) if ( currentUrl.hostname != targetUrl.hostname @@ -181,25 +152,25 @@ def getDashboardData(self) -> dict: self.goHome() return self.webdriver.execute_script("return dashboard") - def getBingInfo(self): + def getBingInfo(self) -> Any: cookieJar = self.webdriver.get_cookies() cookies = {cookie["name"]: cookie["value"] for cookie in cookieJar} maxTries = 5 for _ in range(maxTries): - with contextlib.suppress(Exception): - response = requests.get( - "https://www.bing.com/rewards/panelflyout/getuserinfo", - cookies=cookies, - ) - if response.status_code == requests.codes.ok: - return response.json() + response = requests.get( + "https://www.bing.com/rewards/panelflyout/getuserinfo", + cookies=cookies, + ) + if response.status_code == requests.codes.ok: + return response.json() time.sleep(1) - return None + raise Exception def checkBingLogin(self): if data := self.getBingInfo(): return data["userInfo"]["isRewardsUser"] else: + # todo - throw exception? return False def getAccountPoints(self) -> int: From a205f7dbf2edc1c293f56e4c1cd5a832835a957b Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Tue, 18 Jun 2024 23:20:12 -0400 Subject: [PATCH 42/74] Raise exceptions if error, don't handle since not recoverable --- src/utils.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/utils.py b/src/utils.py index 44271818..a34675f9 100644 --- a/src/utils.py +++ b/src/utils.py @@ -166,18 +166,14 @@ def getBingInfo(self) -> Any: time.sleep(1) raise Exception - def checkBingLogin(self): - if data := self.getBingInfo(): - return data["userInfo"]["isRewardsUser"] - else: - # todo - throw exception? - return False + def checkBingLogin(self) -> bool: + return self.getBingInfo()["userInfo"]["isRewardsUser"] def getAccountPoints(self) -> int: return self.getDashboardData()["userStatus"]["availablePoints"] def getBingAccountPoints(self) -> int: - return data["userInfo"]["balance"] if (data := self.getBingInfo()) else 0 + return self.getBingInfo()["userInfo"]["balance"] def getGoalPoints(self) -> int: return self.getDashboardData()["userStatus"]["redeemGoal"]["price"] From dcf21a87a5fbb354c0a8dcf1d8d68352a9281f80 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Wed, 19 Jun 2024 00:03:33 -0400 Subject: [PATCH 43/74] Fix error when counting points when no remaining searches --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index 9cd58923..425fcfd7 100644 --- a/main.py +++ b/main.py @@ -211,7 +211,7 @@ def executeBot(currentAccount: Account, args: argparse.Namespace): with Browser(mobile=False, account=currentAccount, args=args) as desktopBrowser: utils = desktopBrowser.utils - startingPoints = Login(desktopBrowser).login() + startingPoints = accountPointsCounter = Login(desktopBrowser).login() logging.info( f"[POINTS] You have {utils.formatNumber(startingPoints)} points on your account" ) From 354cecc4c4ae37477d331a3e6b7de4a46c8f953b Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Wed, 19 Jun 2024 00:04:44 -0400 Subject: [PATCH 44/74] Better error-handling --- src/activities.py | 20 +++++++++++--- src/dailySet.py | 3 ++- src/morePromotions.py | 3 ++- src/punchCards.py | 3 ++- src/userAgentGenerator.py | 7 +++-- src/utils.py | 56 +++++++++++++++++---------------------- 6 files changed, 49 insertions(+), 43 deletions(-) diff --git a/src/activities.py b/src/activities.py index bb1343c8..57a65703 100644 --- a/src/activities.py +++ b/src/activities.py @@ -1,6 +1,8 @@ +import logging import random import time +from selenium.common import NoSuchElementException from selenium.webdriver.common.by import By from src.browser import Browser @@ -41,7 +43,10 @@ def completeSurvey(self): def completeQuiz(self): # Simulate completing a quiz activity - if not self.browser.utils.waitUntilQuizLoads(): + try: + self.browser.utils.waitUntilQuizLoads() + except NoSuchElementException: + logging.warning("", exc_info=True) self.browser.utils.resetTabs() return self.webdriver.find_element(By.XPATH, '//*[@id="rqStartQuiz"]').click() @@ -67,7 +72,10 @@ def completeQuiz(self): for answer in answers: self.webdriver.find_element(By.ID, answer).click() time.sleep(random.randint(10, 15)) - if not self.browser.utils.waitUntilQuestionRefresh(): + try: + self.browser.utils.waitUntilQuestionRefresh() + except NoSuchElementException: + logging.warning("", exc_info=True) self.browser.utils.resetTabs() return elif numberOfOptions in [2, 3, 4]: @@ -110,7 +118,10 @@ def completeABC(self): def completeThisOrThat(self): # Simulate completing a This or That activity - if not self.browser.utils.waitUntilQuizLoads(): + try: + self.browser.utils.waitUntilQuizLoads() + except NoSuchElementException: + logging.warning("", exc_info=True) self.browser.utils.resetTabs() return self.webdriver.find_element(By.XPATH, '//*[@id="rqStartQuiz"]').click() @@ -145,4 +156,5 @@ def getAnswerAndCode(self, answerId: str) -> tuple: self.browser.utils.getAnswerCode(answerEncodeKey, answerTitle), ) else: - return (answer, None) + # todo - throw exception? + return answer, None diff --git a/src/dailySet.py b/src/dailySet.py index ea6e90fd..9aaced2b 100644 --- a/src/dailySet.py +++ b/src/dailySet.py @@ -85,7 +85,8 @@ def completeDailySet(self): # Default to completing quiz self.activities.completeQuiz() except Exception: # pylint: disable=broad-except - logging.warning("", exc_info=True) + logging.error("[DAILY SET] Error Daily Set", exc_info=True) # Reset tabs in case of an exception self.browser.utils.resetTabs() + return logging.info("[DAILY SET] Completed the Daily Set successfully !") diff --git a/src/morePromotions.py b/src/morePromotions.py index 2005f54e..ad172e22 100644 --- a/src/morePromotions.py +++ b/src/morePromotions.py @@ -42,7 +42,8 @@ def completeMorePromotions(self): # Default to completing search self.activities.completeSearch() except Exception: # pylint: disable=broad-except - logging.warning("", exc_info=True) + logging.error("[MORE PROMOS] Error More Promotions", exc_info=True) # Reset tabs in case of an exception self.browser.utils.resetTabs() + return logging.info("[MORE PROMOS] Completed More Promotions successfully !") diff --git a/src/punchCards.py b/src/punchCards.py index 938c7d2f..e665f59d 100644 --- a/src/punchCards.py +++ b/src/punchCards.py @@ -72,8 +72,9 @@ def completePunchCards(self): punchCard["childPromotions"], ) except Exception: # pylint: disable=broad-except - logging.warning("", exc_info=True) + logging.error("[PUNCH CARDS] Error Punch Cards", exc_info=True) self.browser.utils.resetTabs() + return logging.info("[PUNCH CARDS] Completed the Punch Cards successfully !") time.sleep(random.randint(100, 700) / 100) self.webdriver.get(BASE_URL) diff --git a/src/userAgentGenerator.py b/src/userAgentGenerator.py index 022d0fc7..dbd48f94 100644 --- a/src/userAgentGenerator.py +++ b/src/userAgentGenerator.py @@ -31,7 +31,7 @@ class GenerateUserAgent: def userAgent( self, - browserConfig: dict[str, Any], + browserConfig: dict[str, Any] | None, mobile: bool = False, ) -> tuple[str, dict[str, Any], Any]: """ @@ -53,9 +53,8 @@ def userAgent( ) newBrowserConfig = None - if userAgentMetadata := browserConfig.get("userAgentMetadata"): - platformVersion = userAgentMetadata["platformVersion"] - + if browserConfig is not None: + platformVersion = browserConfig.get("userAgentMetadata")["platformVersion"] else: # ref : https://textslashplain.com/2021/09/21/determining-os-platform-version/ platformVersion = ( diff --git a/src/utils.py b/src/utils.py index a34675f9..b9758041 100644 --- a/src/utils.py +++ b/src/utils.py @@ -10,6 +10,7 @@ import requests import yaml from apprise import Apprise +from selenium.common import NoSuchElementException from selenium.webdriver.chrome.webdriver import WebDriver from selenium.webdriver.common.by import By from selenium.webdriver.support import expected_conditions as ec @@ -81,7 +82,7 @@ def waitForMSRewardElement(self, by: str, selector: str) -> None: try: self.webdriver.find_element(by, selector) return - except Exception: + except NoSuchElementException: logging.warning("", exc_info=True) if tries < checks: tries += 1 @@ -92,7 +93,7 @@ def waitForMSRewardElement(self, by: str, selector: str) -> None: tries = 0 time.sleep(5) else: - raise Exception + raise NoSuchElementException def waitUntilQuestionRefresh(self) -> None: return self.waitForMSRewardElement(By.CLASS_NAME, "rqECredits") @@ -181,7 +182,7 @@ def getGoalPoints(self) -> int: def getGoalTitle(self) -> str: return self.getDashboardData()["userStatus"]["redeemGoal"]["title"] - def tryDismissAllMessages(self): + def tryDismissAllMessages(self) -> None: buttons = [ (By.ID, "iLandingViewAction"), (By.ID, "iShowSkip"), @@ -192,47 +193,39 @@ def tryDismissAllMessages(self): (By.ID, "bnp_btn_accept"), (By.ID, "acceptButton"), ] - result = False for button in buttons: try: - elements = self.webdriver.find_elements(button[0], button[1]) - try: - for element in elements: - element.click() - except Exception: - logging.warning("", exc_info=True) - continue - result = True - except Exception: - logging.warning("", exc_info=True) + elements = self.webdriver.find_elements(by=button[0], value=button[1]) + except NoSuchElementException: # Expected? continue - return result + for element in elements: + element.click() - def tryDismissCookieBanner(self): - with contextlib.suppress(Exception): + def tryDismissCookieBanner(self) -> None: + with contextlib.suppress(NoSuchElementException): # Expected self.webdriver.find_element(By.ID, "cookie-banner").find_element( By.TAG_NAME, "button" ).click() time.sleep(2) - def tryDismissBingCookieBanner(self): - with contextlib.suppress(Exception): + def tryDismissBingCookieBanner(self) -> None: + with contextlib.suppress(NoSuchElementException): # Expected self.webdriver.find_element(By.ID, "bnp_btn_accept").click() time.sleep(2) - def switchToNewTab(self, timeToWait: int = 0): + def switchToNewTab(self, timeToWait: int = 0) -> None: time.sleep(0.5) self.webdriver.switch_to.window(window_name=self.webdriver.window_handles[1]) if timeToWait > 0: time.sleep(timeToWait) - def closeCurrentTab(self): + def closeCurrentTab(self) -> None: self.webdriver.close() time.sleep(0.5) self.webdriver.switch_to.window(window_name=self.webdriver.window_handles[0]) time.sleep(0.5) - def visitNewTab(self, timeToWait: int = 0): + def visitNewTab(self, timeToWait: int = 0) -> None: self.switchToNewTab(timeToWait) self.closeCurrentTab() @@ -259,22 +252,21 @@ def getRemainingSearches(self) -> RemainingSearches: return RemainingSearches(desktop=remainingDesktop, mobile=remainingMobile) @staticmethod - def formatNumber(number, num_decimals=2): + def formatNumber(number, num_decimals=2) -> str: return pylocale.format_string( f"%10.{num_decimals}f", number, grouping=True ).strip() @staticmethod - def getBrowserConfig(sessionPath: Path) -> dict: - configFile = sessionPath.joinpath("config.json") - if configFile.exists(): - with open(configFile, "r") as f: - return json.load(f) - else: - return {} + def getBrowserConfig(sessionPath: Path) -> dict | None: + configFile = sessionPath / "config.json" + if not configFile.exists(): + return + with open(configFile, "r") as f: + return json.load(f) @staticmethod - def saveBrowserConfig(sessionPath: Path, config: dict): - configFile = sessionPath.joinpath("config.json") + def saveBrowserConfig(sessionPath: Path, config: dict) -> None: + configFile = sessionPath / "config.json" with open(configFile, "w") as f: json.dump(config, f) From a6da35fb917fceb142f308170da236b675893443 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Wed, 19 Jun 2024 00:20:33 -0400 Subject: [PATCH 45/74] Add type hint --- src/searches.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/searches.py b/src/searches.py index 92b516cb..46e0fc6e 100644 --- a/src/searches.py +++ b/src/searches.py @@ -88,7 +88,7 @@ def getRelatedTerms(self, word: str) -> list[str]: logging.warning("", exc_info=True) return [word] - def bingSearches(self, numberOfSearches: int, pointsCounter: int = 0): + def bingSearches(self, numberOfSearches: int, pointsCounter: int = 0) -> int: # Function to perform Bing searches logging.info( f"[BING] Starting {self.browser.browserType.capitalize()} Edge Bing searches..." From f1e0bd69e8216895ba6da5fbf21bfd6716468d0f Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Wed, 19 Jun 2024 12:43:31 -0400 Subject: [PATCH 46/74] Remove redundant method and just throw an exception instead of infinite loop --- src/login.py | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/src/login.py b/src/login.py index e8993631..c3084f68 100644 --- a/src/login.py +++ b/src/login.py @@ -51,7 +51,8 @@ def login(self) -> int: points = self.utils.getAccountPoints() logging.info("[LOGIN] " + "Ensuring you are logged into Bing...") - self.checkBingLogin() + if not self.utils.checkBingLogin(): + raise Exception logging.info("[LOGIN] Logged-in successfully !") return points @@ -121,18 +122,3 @@ def enterPassword(self, password) -> None: password_field.clear() time.sleep(3) time.sleep(3) - - def checkBingLogin(self) -> None: - self.webdriver.get( - "https://www.bing.com/fd/auth/signin?action=interactive&provider=windows_live_id&return_url=https%3A%2F" - "%2Fwww.bing.com%2F" - ) - while True: - currentUrl = urllib.parse.urlparse(self.webdriver.current_url) - if currentUrl.hostname == "www.bing.com" and currentUrl.path == "/": - time.sleep(3) - self.utils.tryDismissBingCookieBanner() - with contextlib.suppress(Exception): - if self.utils.checkBingLogin(): - return - time.sleep(1) From 87ff9dd79f63ba96ff1fd947d0f533a48774dd21 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Wed, 19 Jun 2024 12:44:16 -0400 Subject: [PATCH 47/74] Just try once and if not successful raise exception --- src/utils.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/src/utils.py b/src/utils.py index b9758041..ccbcc22f 100644 --- a/src/utils.py +++ b/src/utils.py @@ -156,16 +156,13 @@ def getDashboardData(self) -> dict: def getBingInfo(self) -> Any: cookieJar = self.webdriver.get_cookies() cookies = {cookie["name"]: cookie["value"] for cookie in cookieJar} - maxTries = 5 - for _ in range(maxTries): - response = requests.get( - "https://www.bing.com/rewards/panelflyout/getuserinfo", - cookies=cookies, - ) - if response.status_code == requests.codes.ok: - return response.json() - time.sleep(1) - raise Exception + response = requests.get( + "https://www.bing.com/rewards/panelflyout/getuserinfo", + cookies=cookies, + ) + if response.status_code != requests.codes.ok: + raise Exception + return response.json() def checkBingLogin(self) -> bool: return self.getBingInfo()["userInfo"]["isRewardsUser"] From 41a665684d8f7efc5ddeb96322f1190c9cd021b9 Mon Sep 17 00:00:00 2001 From: Cal Williams <9409256+cal4@users.noreply.github.com> Date: Wed, 19 Jun 2024 12:45:24 -0400 Subject: [PATCH 48/74] Replace Apprise summary parameter with ability to disable Apprise via a parameter --- .idea/runConfigurations/main.xml | 2 +- .idea/runConfigurations/main_headless.xml | 2 +- main.py | 21 ++++++++------------- src/utils.py | 5 +++++ 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.idea/runConfigurations/main.xml b/.idea/runConfigurations/main.xml index 042a23a4..179919aa 100644 --- a/.idea/runConfigurations/main.xml +++ b/.idea/runConfigurations/main.xml @@ -13,7 +13,7 @@