From e338ee27daf71f5584488b8f80dba4d3c3bd9d8a Mon Sep 17 00:00:00 2001 From: alya Date: Tue, 19 Mar 2024 18:58:02 +0200 Subject: [PATCH] add missing abstracts/ dir --- abstracts/comparison_methods.py | 41 ++++++++ abstracts/dbs.py | 170 ++++++++++++++++++++++++++++++++ abstracts/observer.py | 23 +++++ abstracts/parsers.py | 48 +++++++++ 4 files changed, 282 insertions(+) create mode 100644 abstracts/comparison_methods.py create mode 100644 abstracts/dbs.py create mode 100644 abstracts/observer.py create mode 100644 abstracts/parsers.py diff --git a/abstracts/comparison_methods.py b/abstracts/comparison_methods.py new file mode 100644 index 0000000..5153aad --- /dev/null +++ b/abstracts/comparison_methods.py @@ -0,0 +1,41 @@ +from abc import ABC, abstractmethod +from multiprocessing import Process +from database.sqlite_db import SQLiteDB +from abstracts.observer import IObservable +from logger.logger import Logger + + +class ComparisonMethod(IObservable, ABC): + name = '' + def __init__(self, + output_dir, + **kwargs): + self.output_dir = output_dir + Process.__init__(self) + IObservable.__init__(self) + self.logger = Logger(self.name, self.output_dir) + # add the logger as an observer so each msg printed to the cli will be sent to it too + self.add_observer(self.logger) + + self.db = SQLiteDB(self.output_dir) + self.init(**kwargs) + + @abstractmethod + def init(self, **kwargs): + """ + the goal of this is to have one common __init__() + for all modules, which is the one in this file + this init will have access to all keyword args passes + when initializing the module + """ + + + def log(self, green_txt, normal_txt, log_to_results_file=True, end="\n"): + """ + gives the txt to the logger to log it to stdout and results.txt + """ + self.notify_observers((normal_txt, green_txt, log_to_results_file, end)) + + + def __del__(self): + self.db.close() diff --git a/abstracts/dbs.py b/abstracts/dbs.py new file mode 100644 index 0000000..e8b4c90 --- /dev/null +++ b/abstracts/dbs.py @@ -0,0 +1,170 @@ +from abc import abstractmethod, ABC +import sqlite3 +from os import path +from time import sleep +from threading import Lock +from termcolor import colored +from abstracts.observer import IObservable +from logger.logger import Logger + +class IDB(IObservable, ABC ): + """ + Interface for every sqlite db + """ + name = '' + # used to lock each call to commit() + cursor_lock = Lock() + db_newly_created = False + + def __init__( + self, + output_dir=None, + db_full_path=None, + ): + """ + :param output_dir: output dir. when this param is given a new db is created in this output dir + :param db_path: full path to the db to connect to + """ + self.output_dir = output_dir + self.path: str = db_full_path + + IObservable.__init__(self) + self.logger = Logger(self.name, self.output_dir) + # add the logger as an observer so each msg printed to the cli will be sent to it too + self.add_observer(self.logger) + + self.connect() + self.init() + + def log(self, green_txt, normal_txt, log_to_results_file=True, end="\n"): + """ + gives the txt to the logger to log it to stdout and results.txt + """ + self.notify_observers((normal_txt, green_txt, log_to_results_file, end)) + + + def connect(self): + """ + Creates the db if it doesn't exist and connects to it + sets the db_newly_created to True if the db didn't already exist + """ + if self.path: + if not path.exists(self.path): + raise(f"Invalid databse path: {self.path}") + elif self.output_dir: + self.path = path.join(self.output_dir, 'db.sqlite') + if not path.exists(self.path): + # db not created, mark it as first time accessing it + # so we can init tables once we connect + self.db_newly_created = True + open(self.path,'w').close() + + self.conn = sqlite3.connect( + self.path, + check_same_thread=False) + + self.cursor = self.conn.cursor() + + @abstractmethod + def init(self): + """""" + + + def create_table(self, table_name, schema): + query = f"CREATE TABLE IF NOT EXISTS {table_name} ({schema})" + self.cursor.execute(query) + self.conn.commit() + + + def get_count(self, table, condition=None): + """ + returns the number of matching rows in the given table based on a specific contioins + """ + res = self.select(table, 'COUNT(*)', condition=condition, fetch='one') + return res[0] + + + def close(self): + self.cursor.close() + self.conn.close() + + def fetchall(self): + """ + wrapper for sqlite fetchall to be able to use a lock + """ + self.cursor_lock.acquire(True) + res = self.cursor.fetchall() + self.cursor_lock.release() + return res + + + def fetchone(self): + """ + wrapper for sqlite fetchone to be able to use a lock + """ + self.cursor_lock.acquire(True) + res = self.cursor.fetchone() + self.cursor_lock.release() + return res + + def execute(self, query, params=None): + """ + wrapper for sqlite execute() To avoid 'Recursive use of cursors not allowed' error + and to be able to use a Lock() + since sqlite is terrible with multi-process applications + this should be used instead of all calls to commit() and execute() + """ + + try: + self.cursor_lock.acquire(True) + #start a transaction + self.cursor.execute('BEGIN') + + if not params: + self.cursor.execute(query) + else: + self.cursor.execute(query, params) + + self.conn.commit() + + self.cursor_lock.release() + except sqlite3.Error as e: + # An error occurred during execution + self.conn.rollback() + + if "database is locked" in str(e): + self.cursor_lock.release() + # Retry after a short delay + sleep(0.1) + self.execute(query, params=params) + else: + # An error occurred during execution + print(f"Error executing query ({query}): {e} - params: {params}") + + + def select(self, table_name, columns="*", condition=None, fetch='all'): + query = f"SELECT {columns} FROM {table_name}" + if condition: + query += f" WHERE {condition}" + self.execute(query) + if fetch == 'all': + result = self.fetchall() + else: + result = self.fetchone() + return result + + + def insert(self, table_name, values): + query = f"INSERT INTO {table_name} VALUES ({values})" + self.execute(query) + + + def update(self, table_name, set_clause, condition): + query = f"UPDATE {table_name} SET {set_clause} WHERE {condition}" + self.execute(query) + + + def delete(self, table_name, condition): + query = f"DELETE FROM {table_name} WHERE {condition}" + self.execute(query) + diff --git a/abstracts/observer.py b/abstracts/observer.py new file mode 100644 index 0000000..471f8df --- /dev/null +++ b/abstracts/observer.py @@ -0,0 +1,23 @@ +from abc import ABC, abstractmethod +class IObserver(ABC): + """ + gets notified whenever an observable has a new msg for it + """ + @abstractmethod + def update(self, msg): + """gets called whenever there's a new msg""" + pass + +class IObservable(ABC): + def __init__(self): + self.observers = [] + + def add_observer(self, observer): + self.observers.append(observer) + + def remove_observer(self, observer): + self.observers.remove(observer) + + def notify_observers(self, msg): + for observer in self.observers: + observer.update(msg) \ No newline at end of file diff --git a/abstracts/parsers.py b/abstracts/parsers.py new file mode 100644 index 0000000..20a62b4 --- /dev/null +++ b/abstracts/parsers.py @@ -0,0 +1,48 @@ +from abc import ABC, abstractmethod +from multiprocessing import Process +from termcolor import colored +from database.sqlite_db import SQLiteDB +from abstracts.observer import IObservable +from logger.logger import Logger +from typing import Optional + + +class Parser(IObservable): + name = '' + def __init__(self, + output_dir: str, + results_path: Optional[str]=None, + **kwargs): + super(Parser, self).__init__() + # init the logger + self.results_path = results_path + self.logger = Logger(self.name, output_dir) + # add the logger as an observer so each msg printed to the cli will be sent to it too + self.add_observer(self.logger) + + Process.__init__(self) + self.db = SQLiteDB(output_dir) + + self.init(**kwargs) + + @abstractmethod + def init(self, **kwargs): + """ + the goal of this is to have one common __init__() + for all modules, which is the one in this file + this init will have access to all keyword args passes + when initializing the module + """ + + def log(self, green_txt, normal_txt, log_to_results_file=True, end="\n"): + """ + gives the txt to the logger to log it to stdout and results.txt + """ + self.notify_observers((normal_txt, green_txt, log_to_results_file, end)) + + @abstractmethod + def parse(self): + """main method of each parser""" + + def __del__(self): + self.db.close()