From b7d4557f8a50b1975223d1abe5015ffb0b20d8cd Mon Sep 17 00:00:00 2001 From: Mike Montano Date: Thu, 16 Nov 2023 11:31:28 -0600 Subject: [PATCH] Upgrade-db now does not add a run-id col to un-gathered files --- .github/workflows/ci.yaml | 50 +++++++++++++++++++-------------------- logpyle/upgrade_db.py | 46 ++++++++++++++++++++++++++--------- 2 files changed, 60 insertions(+), 36 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 81f25df..693d271 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -261,46 +261,46 @@ jobs: # test ungathered_v3 default name upgrade-db .github/workflows/log_ungathered_v3.sqlite - runalyzer .github/workflows/log_ungathered_v3_upgrade.sqlite -c 'db.print_cursor(db.q("select * from warnings"))' - runalyzer .github/workflows/log_ungathered_v3_upgrade.sqlite -c 'print([l[0] for l in q("select * from warnings").description])' - runalyzer .github/workflows/log_ungathered_v3_upgrade.sqlite -c 'print([l[0] for l in q("select * from logging").description])' - runalyzer .github/workflows/log_ungathered_v3_upgrade.sqlite -c 'assert "unixtime" in [l[0] for l in q("select * from warnings").description]' - runalyzer .github/workflows/log_ungathered_v3_upgrade.sqlite -c 'assert "rank" in [l[0] for l in q("select * from warnings").description]' + runalyzer .github/workflows/log_ungathered_v3_upgrade.sqlite -c 'db.print_cursor(db.q("select * from warnings"))' --nogather + runalyzer .github/workflows/log_ungathered_v3_upgrade.sqlite -c 'print([l[0] for l in q("select * from warnings").description])' --nogather + runalyzer .github/workflows/log_ungathered_v3_upgrade.sqlite -c 'print([l[0] for l in q("select * from logging").description])' --nogather + runalyzer .github/workflows/log_ungathered_v3_upgrade.sqlite -c 'assert "unixtime" in [l[0] for l in q("select * from warnings").description]' --nogather + runalyzer .github/workflows/log_ungathered_v3_upgrade.sqlite -c 'assert "rank" in [l[0] for l in q("select * from warnings").description]' --nogather # test gathered_v3 default name upgrade-db .github/workflows/log_gathered_v3.sqlite - runalyzer .github/workflows/log_gathered_v3_upgrade.sqlite -c 'db.print_cursor(db.q("select * from warnings"))' - runalyzer .github/workflows/log_gathered_v3_upgrade.sqlite -c 'print([l[0] for l in q("select * from warnings").description])' - runalyzer .github/workflows/log_gathered_v3_upgrade.sqlite -c 'print([l[0] for l in q("select * from logging").description])' - runalyzer .github/workflows/log_gathered_v3_upgrade.sqlite -c 'assert "unixtime" in [l[0] for l in q("select * from warnings").description]' - runalyzer .github/workflows/log_gathered_v3_upgrade.sqlite -c 'assert "rank" in [l[0] for l in q("select * from warnings").description]' + runalyzer .github/workflows/log_gathered_v3_upgrade.sqlite -c 'db.print_cursor(db.q("select * from warnings"))' --nogather + runalyzer .github/workflows/log_gathered_v3_upgrade.sqlite -c 'print([l[0] for l in q("select * from warnings").description])' --nogather + runalyzer .github/workflows/log_gathered_v3_upgrade.sqlite -c 'print([l[0] for l in q("select * from logging").description])' --nogather + runalyzer .github/workflows/log_gathered_v3_upgrade.sqlite -c 'assert "unixtime" in [l[0] for l in q("select * from warnings").description]' --nogather + runalyzer .github/workflows/log_gathered_v3_upgrade.sqlite -c 'assert "rank" in [l[0] for l in q("select * from warnings").description]' --nogather # test ungathered_v2 default name upgrade-db .github/workflows/log_ungathered_v2.sqlite - runalyzer .github/workflows/log_ungathered_v2_upgrade.sqlite -c 'db.print_cursor(db.q("select * from warnings"))' - runalyzer .github/workflows/log_ungathered_v2_upgrade.sqlite -c 'print([l[0] for l in q("select * from warnings").description])' - runalyzer .github/workflows/log_ungathered_v2_upgrade.sqlite -c 'print([l[0] for l in q("select * from logging").description])' - runalyzer .github/workflows/log_ungathered_v2_upgrade.sqlite -c 'assert "unixtime" in [l[0] for l in q("select * from warnings").description]' - runalyzer .github/workflows/log_ungathered_v2_upgrade.sqlite -c 'assert "rank" in [l[0] for l in q("select * from warnings").description]' + runalyzer .github/workflows/log_ungathered_v2_upgrade.sqlite -c 'db.print_cursor(db.q("select * from warnings"))' --nogather + runalyzer .github/workflows/log_ungathered_v2_upgrade.sqlite -c 'print([l[0] for l in q("select * from warnings").description])' --nogather + runalyzer .github/workflows/log_ungathered_v2_upgrade.sqlite -c 'print([l[0] for l in q("select * from logging").description])' --nogather + runalyzer .github/workflows/log_ungathered_v2_upgrade.sqlite -c 'assert "unixtime" in [l[0] for l in q("select * from warnings").description]' --nogather + runalyzer .github/workflows/log_ungathered_v2_upgrade.sqlite -c 'assert "rank" in [l[0] for l in q("select * from warnings").description]' --nogather # test gathered_v2 default name upgrade-db .github/workflows/log_gathered_v2.sqlite - runalyzer .github/workflows/log_gathered_v2_upgrade.sqlite -c 'db.print_cursor(db.q("select * from warnings"))' - runalyzer .github/workflows/log_gathered_v2_upgrade.sqlite -c 'print([l[0] for l in q("select * from warnings").description])' - runalyzer .github/workflows/log_gathered_v2_upgrade.sqlite -c 'print([l[0] for l in q("select * from logging").description])' - runalyzer .github/workflows/log_gathered_v2_upgrade.sqlite -c 'assert "unixtime" in [l[0] for l in q("select * from warnings").description]' - runalyzer .github/workflows/log_gathered_v2_upgrade.sqlite -c 'assert "rank" in [l[0] for l in q("select * from warnings").description]' + runalyzer .github/workflows/log_gathered_v2_upgrade.sqlite -c 'db.print_cursor(db.q("select * from warnings"))' --nogather + runalyzer .github/workflows/log_gathered_v2_upgrade.sqlite -c 'print([l[0] for l in q("select * from warnings").description])' --nogather + runalyzer .github/workflows/log_gathered_v2_upgrade.sqlite -c 'print([l[0] for l in q("select * from logging").description])' --nogather + runalyzer .github/workflows/log_gathered_v2_upgrade.sqlite -c 'assert "unixtime" in [l[0] for l in q("select * from warnings").description]' --nogather + runalyzer .github/workflows/log_gathered_v2_upgrade.sqlite -c 'assert "rank" in [l[0] for l in q("select * from warnings").description]' --nogather # test gathered custom name upgrade-db .github/workflows/log_gathered_v2.sqlite --suffix '_new' - runalyzer .github/workflows/log_gathered_v2_new.sqlite -c 'db.print_cursor(db.q("select * from warnings"))' - runalyzer .github/workflows/log_gathered_v2_new.sqlite -c 'print([l[0] for l in q("select * from warnings").description])' - runalyzer .github/workflows/log_gathered_v2_new.sqlite -c 'print([l[0] for l in q("select * from logging").description])' - runalyzer .github/workflows/log_gathered_v2_new.sqlite -c 'assert "unixtime" in [l[0] for l in q("select * from warnings").description]' - runalyzer .github/workflows/log_gathered_v2_new.sqlite -c 'assert "rank" in [l[0] for l in q("select * from warnings").description]' + runalyzer .github/workflows/log_gathered_v2_new.sqlite -c 'db.print_cursor(db.q("select * from warnings"))' --nogather + runalyzer .github/workflows/log_gathered_v2_new.sqlite -c 'print([l[0] for l in q("select * from warnings").description])' --nogather + runalyzer .github/workflows/log_gathered_v2_new.sqlite -c 'print([l[0] for l in q("select * from logging").description])' --nogather + runalyzer .github/workflows/log_gathered_v2_new.sqlite -c 'assert "unixtime" in [l[0] for l in q("select * from warnings").description]' --nogather + runalyzer .github/workflows/log_gathered_v2_new.sqlite -c 'assert "rank" in [l[0] for l in q("select * from warnings").description]' --nogather diff --git a/logpyle/upgrade_db.py b/logpyle/upgrade_db.py index 701cb54..04396ef 100644 --- a/logpyle/upgrade_db.py +++ b/logpyle/upgrade_db.py @@ -6,6 +6,17 @@ def upgrade_conn(conn: sqlite3.Connection) -> sqlite3.Connection: tmp = conn.execute("select * from warnings").description warning_columns = [col[0] for col in tmp] + # check if any of the provided files have been gathered + gathered = False + # get a list of tables with the name of 'runs' + res = list(conn.execute(""" + SELECT name + FROM sqlite_master + WHERE type='table' AND name='runs' + """)) + if len(res) == 1: + gathered = True + # ensure that warnings table has unixtime column if ("unixtime" not in warning_columns): print("Adding a unixtime column in the warnings table") @@ -25,17 +36,30 @@ def upgrade_conn(conn: sqlite3.Connection) -> sqlite3.Connection: """) print("Ensuring a logging table exists") - conn.execute(""" - CREATE TABLE IF NOT EXISTS logging ( - run_id integer, - rank integer, - step integer, - unixtime integer, - level text, - message text, - filename text, - lineno integer - )""") + if gathered: + conn.execute(""" + CREATE TABLE IF NOT EXISTS logging ( + run_id integer, + rank integer, + step integer, + unixtime integer, + level text, + message text, + filename text, + lineno integer + )""") + else: + conn.execute(""" + CREATE TABLE IF NOT EXISTS logging ( + rank integer, + step integer, + unixtime integer, + level text, + message text, + filename text, + lineno integer + )""") + return conn