From c2890b7bb679e3da075f330e8bf750a61551564d Mon Sep 17 00:00:00 2001 From: Daniel Thom Date: Fri, 10 Jan 2025 19:15:49 -0700 Subject: [PATCH 1/4] SQL transactions for time series and supplemental attributes --- src/PowerSystems.jl | 2 ++ src/base.jl | 57 ++++++++++++++++++++++++++++++++------------- test/test_system.jl | 33 ++++++++++++++++++++++++++ 3 files changed, 76 insertions(+), 16 deletions(-) diff --git a/src/PowerSystems.jl b/src/PowerSystems.jl index ef031063b1..46df265d1b 100644 --- a/src/PowerSystems.jl +++ b/src/PowerSystems.jl @@ -313,6 +313,7 @@ export parse_file export open_time_series_store! export add_time_series! export bulk_add_time_series! +export begin_time_series_update export remove_time_series! export check_time_series_consistency export clear_time_series! @@ -357,6 +358,7 @@ export get_supplemental_attribute export get_supplemental_attributes export has_supplemental_attributes export iterate_supplemental_attributes +export begin_supplemental_attributes_update export get_time_series export get_time_series_type export get_time_series_array diff --git a/src/base.jl b/src/base.jl index c2862783d8..5d518bd460 100644 --- a/src/base.jl +++ b/src/base.jl @@ -852,10 +852,10 @@ open_time_series_store!(sys, "r+") do end end ``` -You can also use this function to make reads faster. Change the mode from `"r+"` to `"r"` to open -the file read-only. +You can also use this function to make reads faster. +Change the mode from `"r+"` to `"r"` to open the file read-only. -See also: [`bulk_add_time_series!`](@ref) +See also: [`begin_time_series_update!`](@ref) """ function open_time_series_store!( func::Function, @@ -867,6 +867,25 @@ function open_time_series_store!( IS.open_time_series_store!(func, sys.data, mode, args...; kwargs...) end +""" +Begin an update of time series. Use this function when adding many time series arrays +in order to improve performance. + +If an error occurs during the update, changes will be reverted. + +Using this function to remove time series is currently not supported. + +# Examples +```julia +begin_time_series_update(sys) do + add_time_series!(sys, component1, time_series1) + add_time_series!(sys, component2, time_series2) +end +``` +""" +begin_time_series_update(func::Function, sys::System) = + begin_time_series_update(func, sys.data.time_series_manager) + """ Add time series data from a metadata file or metadata descriptors. @@ -1383,17 +1402,9 @@ function add_time_series!( end """ -Add many time series in bulk +Add time series in bulk. -This method is advantageous when adding thousands of time -series arrays because of the overhead in writing the time series to the underlying storage. - -# Arguments -- `sys::System`: system -- `associations`: Iterable of [`TimeSeriesAssociation`](@ref) instances. Using a Vector is not - recommended. Pass a Generator or Iterator to avoid loading all time series data into - system memory at once. -- `batch_size::Int`: (Default = 100) Number of time series to add per batch. +Prefer use of [`begin_time_series_update!`](@ref). # Examples ```julia @@ -1412,9 +1423,6 @@ associations = ( ) bulk_add_time_series!(sys, associations) ``` - -See also: [`open_time_series_store!`](@ref) to minimize HDF5 file handle overhead if you -must add time series arrays one at a time """ function bulk_add_time_series!( sys::System, @@ -1630,6 +1638,23 @@ function add_supplemental_attribute!( return IS.add_supplemental_attribute!(sys.data, component, attribute) end +""" +Begin an update of supplemental attributes. Use this function when adding +or removing many supplemental attributes in order to improve performance. + +If an error occurs during the update, changes will be reverted. + +# Examples +```julia +begin_supplemental_attributes_update(sys) do + add_supplemental_attribute!(sys, component1, attribute1) + add_supplemental_attribute!(sys, component2, attribute2) +end +``` +""" +begin_supplemental_attributes_update(func::Function, sys::System) = + IS.begin_supplemental_attributes_update(func, sys.data) + """ Remove the supplemental attribute from the component. The attribute will be removed from the system if it is not attached to any other component. diff --git a/test/test_system.jl b/test/test_system.jl index ac7a2df55b..61560ba7f1 100644 --- a/test/test_system.jl +++ b/test/test_system.jl @@ -289,6 +289,39 @@ end end end +@testset "Test begin_time_series_update" begin + sys = System(100.0) + bus = ACBus(nothing) + bus.bustype = ACBusTypes.REF + add_component!(sys, bus) + components = [] + len = 2 + component = ThermalStandard(nothing) + component.name = "gen" + component.bus = bus + add_component!(sys, component) + initial_time = Dates.DateTime("2020-09-01") + resolution = Dates.Hour(1) + len = 24 + timestamps = range(initial_time; length = len, step = resolution) + arrays = [TimeSeries.TimeArray(timestamps, rand(len)) for _ in 1:5] + ts_name = "test" + + begin_time_series_update(sys) do + for (i, ta) in enumerate(arrays) + ts = SingleTimeSeries(; data = ta, name = "$(ts_name)_$(i)") + add_time_series!(sys, component, ts) + end + end + + open_time_series_store!(sys, "r") do + for (i, expected_array) in enumerate(arrays) + ts = IS.get_time_series(IS.SingleTimeSeries, component, "$(ts_name)_$(i)") + @test ts.data == expected_array + end + end +end + @testset "Test set_name! of system component" begin sys = System(100.0) bus = ACBus(nothing) From ccc32a86cb8abd987fa931e63cfdb72d857fa750 Mon Sep 17 00:00:00 2001 From: Daniel Thom Date: Sat, 11 Jan 2025 10:58:21 -0700 Subject: [PATCH 2/4] Fix bugs and add test --- src/base.jl | 4 ++-- test/common.jl | 58 +++++++++++++++++++++++++++----------------------- 2 files changed, 33 insertions(+), 29 deletions(-) diff --git a/src/base.jl b/src/base.jl index 5d518bd460..069435d778 100644 --- a/src/base.jl +++ b/src/base.jl @@ -884,7 +884,7 @@ end ``` """ begin_time_series_update(func::Function, sys::System) = - begin_time_series_update(func, sys.data.time_series_manager) + IS.begin_time_series_update(func, sys.data.time_series_manager) """ Add time series data from a metadata file or metadata descriptors. @@ -1653,7 +1653,7 @@ end ``` """ begin_supplemental_attributes_update(func::Function, sys::System) = - IS.begin_supplemental_attributes_update(func, sys.data) + IS.begin_supplemental_attributes_update(func, sys.data.supplemental_attribute_manager) """ Remove the supplemental attribute from the component. The attribute will be removed from the diff --git a/test/common.jl b/test/common.jl index 008b336db8..22d6e32656 100644 --- a/test/common.jl +++ b/test/common.jl @@ -175,33 +175,37 @@ function create_system_with_outages() gen2 = gens[2] geo1 = GeographicInfo(; geo_json = Dict("x" => 1.0, "y" => 2.0)) geo2 = GeographicInfo(; geo_json = Dict("x" => 3.0, "y" => 4.0)) - add_supplemental_attribute!(sys, gen1, geo1) - add_supplemental_attribute!(sys, gen1.bus, geo1) - add_supplemental_attribute!(sys, gen2, geo2) - add_supplemental_attribute!(sys, gen2.bus, geo2) - initial_time = Dates.DateTime("2020-01-01T00:00:00") - end_time = Dates.DateTime("2020-01-01T23:00:00") - dates = collect(initial_time:Dates.Hour(1):end_time) - fo1 = GeometricDistributionForcedOutage(; - mean_time_to_recovery = 1.0, - outage_transition_probability = 0.5, - ) - fo2 = GeometricDistributionForcedOutage(; - mean_time_to_recovery = 2.0, - outage_transition_probability = 0.5, - ) - po1 = PlannedOutage(; outage_schedule = "1") - po2 = PlannedOutage(; outage_schedule = "2") - add_supplemental_attribute!(sys, gen1, fo1) - add_supplemental_attribute!(sys, gen1, po1) - add_supplemental_attribute!(sys, gen2, fo2) - add_supplemental_attribute!(sys, gen2, po2) - for (i, outage) in enumerate((fo1, fo2, po1, po2)) - data = collect(i:(i + 23)) - ta = TimeSeries.TimeArray(dates, data, ["1"]) - name = "ts_$(i)" - ts = SingleTimeSeries(; name = name, data = ta) - add_time_series!(sys, outage, ts) + begin_time_series_update(sys) do + begin_supplemental_attributes_update(sys) do + add_supplemental_attribute!(sys, gen1, geo1) + add_supplemental_attribute!(sys, gen1.bus, geo1) + add_supplemental_attribute!(sys, gen2, geo2) + add_supplemental_attribute!(sys, gen2.bus, geo2) + initial_time = Dates.DateTime("2020-01-01T00:00:00") + end_time = Dates.DateTime("2020-01-01T23:00:00") + dates = collect(initial_time:Dates.Hour(1):end_time) + fo1 = GeometricDistributionForcedOutage(; + mean_time_to_recovery = 1.0, + outage_transition_probability = 0.5, + ) + fo2 = GeometricDistributionForcedOutage(; + mean_time_to_recovery = 2.0, + outage_transition_probability = 0.5, + ) + po1 = PlannedOutage(; outage_schedule = "1") + po2 = PlannedOutage(; outage_schedule = "2") + add_supplemental_attribute!(sys, gen1, fo1) + add_supplemental_attribute!(sys, gen1, po1) + add_supplemental_attribute!(sys, gen2, fo2) + add_supplemental_attribute!(sys, gen2, po2) + for (i, outage) in enumerate((fo1, fo2, po1, po2)) + data = collect(i:(i + 23)) + ta = TimeSeries.TimeArray(dates, data, ["1"]) + name = "ts_$(i)" + ts = SingleTimeSeries(; name = name, data = ta) + add_time_series!(sys, outage, ts) + end + end end return sys From 2ad5d8bb6c5ecb9593a80a93f57c91dd4813ef1f Mon Sep 17 00:00:00 2001 From: Daniel Thom Date: Sat, 11 Jan 2025 10:58:50 -0700 Subject: [PATCH 3/4] Update docs --- docs/src/how_to/improve_ts_performance.md | 26 +++++++---------------- 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/docs/src/how_to/improve_ts_performance.md b/docs/src/how_to/improve_ts_performance.md index 4e6686d5c9..40058cb9e0 100644 --- a/docs/src/how_to/improve_ts_performance.md +++ b/docs/src/how_to/improve_ts_performance.md @@ -83,26 +83,16 @@ add_time_series!(sys, generator, forecast_max_reactive_power) By default, the call to [`add_time_series!`](@ref) will open the HDF5 file, write the data to the file, and close the file. It will also add a row to an SQLite database. These operations have overhead. -If you will add thousands of time series arrays, consider using [`bulk_add_time_series!`](@ref). -All arrays will be written with one file handle. The -bulk SQLite operations are much more efficient. As a fallback option, use -[`open_time_series_store!`](@ref) if timeseries must be added one at a time. +If you will add thousands of time series arrays, consider using [`begin_time_series_update`](@ref). +All arrays will be written with one file handle. The bulk SQLite operations are much more +efficient. ```julia -# Assumes `read_time_series` will return data appropriate for Deterministic forecasts -# based on the generator name and the filenames match the component and time series names. -resolution = Dates.Hour(1) -associations = ( - IS.TimeSeriesAssociation( - gen, - Deterministic(; - data = read_time_series(get_name(gen) * ".csv"), - name = "get_max_active_power", - resolution = resolution), - ) - for gen in get_components(ThermalStandard, sys) -) -bulk_add_time_series!(sys, associations) +begin_time_series_update(sys) do + add_time_series!(sys, component1, time_series1) + add_time_series!(sys, component2, time_series2) + add_time_series!(sys, component3, time_series3) +end ``` ## Using Forecast Caches for Simulations From b9a19e2417afe5dcfcdf7f9fc9c93b94b6e5d9a9 Mon Sep 17 00:00:00 2001 From: Daniel Thom Date: Mon, 13 Jan 2025 17:45:46 -0700 Subject: [PATCH 4/4] Fix spelling of function --- src/base.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/base.jl b/src/base.jl index 069435d778..11e246bdec 100644 --- a/src/base.jl +++ b/src/base.jl @@ -855,7 +855,7 @@ end You can also use this function to make reads faster. Change the mode from `"r+"` to `"r"` to open the file read-only. -See also: [`begin_time_series_update!`](@ref) +See also: [`begin_time_series_update`](@ref) """ function open_time_series_store!( func::Function, @@ -1404,7 +1404,7 @@ end """ Add time series in bulk. -Prefer use of [`begin_time_series_update!`](@ref). +Prefer use of [`begin_time_series_update`](@ref). # Examples ```julia