From 42f1beba394df84f88f1b84a5c85e0c9bc60ddb1 Mon Sep 17 00:00:00 2001 From: adrienmellot Date: Wed, 10 Apr 2024 18:10:50 +0200 Subject: [PATCH 01/24] Process data for custom constraints --- Snakefile | 4 + config/default.yaml | 6 +- config/schema.yaml | 18 +++- rules/transport.smk | 22 +++- .../road_transport_controlled_constraints.py | 101 ++++++++++++++++++ .../techs/demand/electrified-transport.yaml | 13 +++ 6 files changed, 159 insertions(+), 5 deletions(-) create mode 100644 scripts/transport/road_transport_controlled_constraints.py diff --git a/Snakefile b/Snakefile index dd68bb52..fe866f43 100644 --- a/Snakefile +++ b/Snakefile @@ -180,6 +180,10 @@ rule model_template: "build/models/{resolution}/timeseries/demand/uncontrolled-road-transport-historic-electrification.csv", "build/models/{resolution}/timeseries/demand/electrified-heat-demand.csv", "build/models/{resolution}/timeseries/demand/heat-demand-historic-electrification.csv", + "build/models/{resolution}/timeseries/demand/demand-shape-min-ev.csv", + "build/models/{resolution}/timeseries/demand/demand-shape-max-ev.csv", + "build/models/{resolution}/timeseries/demand/demand-shape-equals-ev.csv", + "build/models/{resolution}/timeseries/demand/plugin-profiles-ev.csv", ), optional_input_files = lambda wildcards: expand( f"build/models/{wildcards.resolution}/{{input_file}}", diff --git a/config/default.yaml b/config/default.yaml index 1dc50ca3..9e9fad01 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -19,7 +19,7 @@ data-sources: swiss-end-use: https://www.bfe.admin.ch/bfe/en/home/versorgung/statistik-und-geodaten/energiestatistiken/energieverbrauch-nach-verwendungszweck.exturl.html/aHR0cHM6Ly9wdWJkYi5iZmUuYWRtaW4uY2gvZGUvcHVibGljYX/Rpb24vZG93bmxvYWQvOTg1NA==.html swiss-energy-balance: https://www.bfe.admin.ch/bfe/en/home/versorgung/statistik-und-geodaten/energiestatistiken/gesamtenergiestatistik.exturl.html/aHR0cHM6Ly9wdWJkYi5iZmUuYWRtaW4uY2gvZGUvcHVibGljYX/Rpb24vZG93bmxvYWQvNzUxOQ==.html swiss-industry-energy-balance: https://www.bfe.admin.ch/bfe/en/home/versorgung/statistik-und-geodaten/energiestatistiken/teilstatistiken.exturl.html/aHR0cHM6Ly9wdWJkYi5iZmUuYWRtaW4uY2gvZGUvcHVibGljYX/Rpb24vZG93bmxvYWQvODc4OA==.html - ev-data: https://zenodo.org/record/6579421/files/ramp-ev-consumption-profiles.csv.gz?download=1 + controlled-ev-data: https://zenodo.org/record/6579421/files/ramp-ev-consumption-profiles.csv.gz?download=1 data-pre-processing: fill-missing-values: jrc-idees: @@ -163,6 +163,10 @@ parameters: passenger-cars: Passenger cars motorcycles: Powered 2-wheelers uncontrolled-ev-charging-share: 0.5 + monthly_demand_ranges: + min: 0.9 + max: 1.1 + equals: 1 entsoe-tyndp: scenario: National Trends grid: Reference diff --git a/config/schema.yaml b/config/schema.yaml index d8e7c175..adc7c135 100644 --- a/config/schema.yaml +++ b/config/schema.yaml @@ -103,10 +103,10 @@ properties: type: string pattern: ^(https?|http?):\/\/.+ description: Web address of Swiss industry energy balance data. - ev-data: + controlled-ev-data: type: string pattern: ^(https?|http?):\/\/.+ - description: Web address of electric vehicle data. + description: Web address of electric vehicle data for controlled charging. data-pre-processing: type: object description: Parameters for the pre-processing of raw data. @@ -351,6 +351,20 @@ properties: uncontrolled-ev-charging-share: type: number description: Share of uncontrolled charging. + monthly_demand_ranges: + type: object + description: Charging flexibility parameters. + additionalProperties: false + properties: + min: + type: number + description: Minimum monthly demand range. + max: + type: number + description: Maximum monthly demand range. + equals: + type: number + description: Equal monthly demand range. entsoe-tyndp: type: object description: Parameters to define scenario choice for data accessed from the ENTSO-E ten-year network development plan 2020. For more information, see https://2020.entsos-tyndp-scenarios.eu/ diff --git a/rules/transport.smk b/rules/transport.smk index 118f375b..c8be6530 100644 --- a/rules/transport.smk +++ b/rules/transport.smk @@ -5,9 +5,12 @@ rule download_transport_timeseries: # TODO have correct timeseries data once RAMP has generated the new charging profile and it's been put on Zenodo message: "Get EV data from RAMP" params: - url = config["data-sources"]["ev-data"] + url = config["data-sources"]["controlled-ev-data"] conda: "../envs/shell.yaml" - output: protected("data/automatic/ramp-ev-consumption-profiles.csv.gz") + output: + protected("data/automatic/ramp-ev-{dataset}.csv.gz") + wildcard_constraints: + dataset = "consumption-profiles|plugin-profiles" localrule: True shell: "curl -sLo {output} {params.url}" @@ -63,6 +66,21 @@ rule create_controlled_road_transport_annual_demand: main = "build/data/{resolution}/demand/electrified-transport.csv", script: "../scripts/transport/road_transport_controlled_charging.py" +rule create_controlled_ev_charging_parameters: + message: "Create timeseries parameters for controlled EV charging" + input: + locations = "build/data/regional/units.csv", + ev_profiles = lambda wildcards: "data/automatic/ramp-ev-consumption-profiles.csv.gz" if "demand" in wildcards.dataset_name else f"data/automatic/ramp-ev-{wildcards.dataset_name}.csv.gz", + populations = "build/data/regional/population.csv", + params: + demand_range = config["parameters"]["transport"]["monthly_demand_ranges"], + first_year = config["scope"]["temporal"]["first-year"], + final_year = config["scope"]["temporal"]["final-year"], + country_neighbour_dict = config["data-pre-processing"]["fill-missing-values"]["ramp"], + countries = config["scope"]["spatial"]["countries"], + conda: "../envs/default.yaml" + output: "build/models/{resolution}/timeseries/demand/{dataset_name}-ev.csv" + script: "../scripts/transport/road_transport_controlled_constraints.py" rule create_uncontrolled_road_transport_timeseries: message: "Create timeseries for road transport demand (uncontrolled charging)" diff --git a/scripts/transport/road_transport_controlled_constraints.py b/scripts/transport/road_transport_controlled_constraints.py new file mode 100644 index 00000000..208d635f --- /dev/null +++ b/scripts/transport/road_transport_controlled_constraints.py @@ -0,0 +1,101 @@ +import pandas as pd +import pycountry + + +def scale_to_resolution_and_create_file( + df, region_country_mapping, populations, resolution, output_path +): + if resolution == "national": + df = df + elif resolution == "continental": + df = df.sum(axis=1).to_frame("EUR") + elif resolution == "regional": + df = scale_national_to_regional(df, region_country_mapping, populations) + else: + raise ValueError(f"Resolution {resolution} is not supported") + df.to_csv(output_path) + + +def scale_national_to_regional(df, region_country_mapping, populations): + df_population_share = ( + populations.loc[:, "population_sum"] + .reindex(region_country_mapping.keys()) + .groupby(by=region_country_mapping) + .transform(lambda df: df / df.sum()) + ) + + regional_df = ( + pd.DataFrame( + index=df.index, + data={ + id: df[country_code] + for id, country_code in region_country_mapping.items() + }, + ) + .mul(df_population_share) + .rename(columns=lambda col_name: col_name.replace(".", "-")) + ) + pd.testing.assert_series_equal(regional_df.sum(axis=1), df.sum(axis=1)) + return regional_df + + +def get_national_ev_profiles( + ev_profiles_path: str, + dataset_name: str, + demand_range: dict[str, int], + first_year: int, + final_year: int, + country_neighbour_dict: dict[str, list[str]], + country_codes: list[str], +): + df_timeseries = ( + pd.read_csv(ev_profiles_path, index_col=[0, 1, 2], parse_dates=[0]) + .xs(slice(first_year, final_year), level="year") + .unstack("country_code") + .droplevel(level=0, axis="columns") + .groupby(by=lambda idx: idx.year) + .transform(lambda x: x / x.sum()) + .pipe(fill_empty_country, country_neighbour_dict) + .loc[:, country_codes] + ) + if "demand" in dataset_name: + df = df_timeseries.mul(demand_range[dataset_name.split("-")[-1]]) + elif "plugin" in dataset_name: + df = df_timeseries + return df + + +def fill_empty_country(df, country_neighbour_dict): + for country, neighbours in country_neighbour_dict.items(): + assert country not in df.columns + df[country] = df[neighbours].mean(axis=1) + return df + + +if __name__ == "__main__": + region_country_mapping = ( + pd.read_csv(snakemake.input.locations, index_col=0) + .loc[:, "country_code"] + .to_dict() + ) + populations = pd.read_csv(snakemake.input.populations, index_col=0) + + df = get_national_ev_profiles( + ev_profiles_path=snakemake.input.ev_profiles, + dataset_name=snakemake.wildcards.dataset_name, + demand_range=snakemake.params.demand_range, + first_year=snakemake.params.first_year, + final_year=snakemake.params.final_year, + country_neighbour_dict=snakemake.params.country_neighbour_dict, + country_codes=([ + pycountry.countries.lookup(c).alpha_3 for c in snakemake.params.countries + ]), + ) + + scale_to_resolution_and_create_file( + df=df, + region_country_mapping=region_country_mapping, + populations=populations, + resolution=snakemake.wildcards.resolution, + output_path=snakemake.output[0], + ) diff --git a/templates/models/techs/demand/electrified-transport.yaml b/templates/models/techs/demand/electrified-transport.yaml index 78da9565..24213e4c 100644 --- a/templates/models/techs/demand/electrified-transport.yaml +++ b/templates/models/techs/demand/electrified-transport.yaml @@ -43,6 +43,19 @@ overrides: {% endfor %} {% endfor %} + monthly_transport_demand_range: + techs: + demand_road_transport_electrified_controlled: + constraints: + demand_shape_per_month_min: file=demand/demand-shape-min-ev.csv + demand_shape_per_month_max: file=demand/demand-shape-max-ev.csv + + monthly_transport_demand_equality: + techs: + demand_road_transport_electrified_controlled: + constraints: + demand_shape_per_month_equals: file=demand/demand-shape-equals-ev.csv + locations: {% for id, location in locations.iterrows() %} {{ id }}.techs.demand_road_transport_electrified_uncontrolled: From 9623afe86812140849c832a527606b614335c7b6 Mon Sep 17 00:00:00 2001 From: adrienmellot Date: Wed, 10 Apr 2024 22:31:29 +0200 Subject: [PATCH 02/24] edit csv file format --- scripts/transport/road_transport_controlled_constraints.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/transport/road_transport_controlled_constraints.py b/scripts/transport/road_transport_controlled_constraints.py index 208d635f..af7a8f2b 100644 --- a/scripts/transport/road_transport_controlled_constraints.py +++ b/scripts/transport/road_transport_controlled_constraints.py @@ -13,7 +13,7 @@ def scale_to_resolution_and_create_file( df = scale_national_to_regional(df, region_country_mapping, populations) else: raise ValueError(f"Resolution {resolution} is not supported") - df.to_csv(output_path) + df.tz_localize(None).rename_axis("utc-timestamp").to_csv(output_path) def scale_national_to_regional(df, region_country_mapping, populations): From e451c4a3bb183d4b3a5fd0b42bd54f49ea844715 Mon Sep 17 00:00:00 2001 From: adrienmellot Date: Wed, 10 Apr 2024 18:10:50 +0200 Subject: [PATCH 03/24] Process data for custom constraints --- Snakefile | 4 + config/default.yaml | 6 +- config/schema.yaml | 18 +++- rules/transport.smk | 22 +++- .../road_transport_controlled_constraints.py | 101 ++++++++++++++++++ .../techs/demand/electrified-transport.yaml | 13 +++ 6 files changed, 159 insertions(+), 5 deletions(-) create mode 100644 scripts/transport/road_transport_controlled_constraints.py diff --git a/Snakefile b/Snakefile index ec6306e5..2d54552f 100644 --- a/Snakefile +++ b/Snakefile @@ -185,6 +185,10 @@ rule model_template: "build/models/{resolution}/timeseries/demand/uncontrolled-road-transport-historic-electrification.csv", "build/models/{resolution}/timeseries/demand/electrified-heat-demand.csv", "build/models/{resolution}/timeseries/demand/heat-demand-historic-electrification.csv", + "build/models/{resolution}/timeseries/demand/demand-shape-min-ev.csv", + "build/models/{resolution}/timeseries/demand/demand-shape-max-ev.csv", + "build/models/{resolution}/timeseries/demand/demand-shape-equals-ev.csv", + "build/models/{resolution}/timeseries/demand/plugin-profiles-ev.csv", ), optional_input_files = lambda wildcards: expand( f"build/models/{wildcards.resolution}/{{input_file}}", diff --git a/config/default.yaml b/config/default.yaml index 1dc50ca3..9e9fad01 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -19,7 +19,7 @@ data-sources: swiss-end-use: https://www.bfe.admin.ch/bfe/en/home/versorgung/statistik-und-geodaten/energiestatistiken/energieverbrauch-nach-verwendungszweck.exturl.html/aHR0cHM6Ly9wdWJkYi5iZmUuYWRtaW4uY2gvZGUvcHVibGljYX/Rpb24vZG93bmxvYWQvOTg1NA==.html swiss-energy-balance: https://www.bfe.admin.ch/bfe/en/home/versorgung/statistik-und-geodaten/energiestatistiken/gesamtenergiestatistik.exturl.html/aHR0cHM6Ly9wdWJkYi5iZmUuYWRtaW4uY2gvZGUvcHVibGljYX/Rpb24vZG93bmxvYWQvNzUxOQ==.html swiss-industry-energy-balance: https://www.bfe.admin.ch/bfe/en/home/versorgung/statistik-und-geodaten/energiestatistiken/teilstatistiken.exturl.html/aHR0cHM6Ly9wdWJkYi5iZmUuYWRtaW4uY2gvZGUvcHVibGljYX/Rpb24vZG93bmxvYWQvODc4OA==.html - ev-data: https://zenodo.org/record/6579421/files/ramp-ev-consumption-profiles.csv.gz?download=1 + controlled-ev-data: https://zenodo.org/record/6579421/files/ramp-ev-consumption-profiles.csv.gz?download=1 data-pre-processing: fill-missing-values: jrc-idees: @@ -163,6 +163,10 @@ parameters: passenger-cars: Passenger cars motorcycles: Powered 2-wheelers uncontrolled-ev-charging-share: 0.5 + monthly_demand_ranges: + min: 0.9 + max: 1.1 + equals: 1 entsoe-tyndp: scenario: National Trends grid: Reference diff --git a/config/schema.yaml b/config/schema.yaml index d8e7c175..adc7c135 100644 --- a/config/schema.yaml +++ b/config/schema.yaml @@ -103,10 +103,10 @@ properties: type: string pattern: ^(https?|http?):\/\/.+ description: Web address of Swiss industry energy balance data. - ev-data: + controlled-ev-data: type: string pattern: ^(https?|http?):\/\/.+ - description: Web address of electric vehicle data. + description: Web address of electric vehicle data for controlled charging. data-pre-processing: type: object description: Parameters for the pre-processing of raw data. @@ -351,6 +351,20 @@ properties: uncontrolled-ev-charging-share: type: number description: Share of uncontrolled charging. + monthly_demand_ranges: + type: object + description: Charging flexibility parameters. + additionalProperties: false + properties: + min: + type: number + description: Minimum monthly demand range. + max: + type: number + description: Maximum monthly demand range. + equals: + type: number + description: Equal monthly demand range. entsoe-tyndp: type: object description: Parameters to define scenario choice for data accessed from the ENTSO-E ten-year network development plan 2020. For more information, see https://2020.entsos-tyndp-scenarios.eu/ diff --git a/rules/transport.smk b/rules/transport.smk index 4f247bf8..48988da6 100644 --- a/rules/transport.smk +++ b/rules/transport.smk @@ -5,9 +5,12 @@ rule download_transport_timeseries: # TODO have correct timeseries data once RAMP has generated the new charging profile and it's been put on Zenodo message: "Get EV data from RAMP" params: - url = config["data-sources"]["ev-data"] + url = config["data-sources"]["controlled-ev-data"] conda: "../envs/shell.yaml" - output: protected("data/automatic/ramp-ev-consumption-profiles.csv.gz") + output: + protected("data/automatic/ramp-ev-{dataset}.csv.gz") + wildcard_constraints: + dataset = "consumption-profiles|plugin-profiles" localrule: True shell: "curl -sSLo {output} {params.url}" @@ -63,6 +66,21 @@ rule create_controlled_road_transport_annual_demand: main = "build/data/{resolution}/demand/electrified-transport.csv", script: "../scripts/transport/road_transport_controlled_charging.py" +rule create_controlled_ev_charging_parameters: + message: "Create timeseries parameters for controlled EV charging" + input: + locations = "build/data/regional/units.csv", + ev_profiles = lambda wildcards: "data/automatic/ramp-ev-consumption-profiles.csv.gz" if "demand" in wildcards.dataset_name else f"data/automatic/ramp-ev-{wildcards.dataset_name}.csv.gz", + populations = "build/data/regional/population.csv", + params: + demand_range = config["parameters"]["transport"]["monthly_demand_ranges"], + first_year = config["scope"]["temporal"]["first-year"], + final_year = config["scope"]["temporal"]["final-year"], + country_neighbour_dict = config["data-pre-processing"]["fill-missing-values"]["ramp"], + countries = config["scope"]["spatial"]["countries"], + conda: "../envs/default.yaml" + output: "build/models/{resolution}/timeseries/demand/{dataset_name}-ev.csv" + script: "../scripts/transport/road_transport_controlled_constraints.py" rule create_uncontrolled_road_transport_timeseries: message: "Create timeseries for road transport demand (uncontrolled charging)" diff --git a/scripts/transport/road_transport_controlled_constraints.py b/scripts/transport/road_transport_controlled_constraints.py new file mode 100644 index 00000000..208d635f --- /dev/null +++ b/scripts/transport/road_transport_controlled_constraints.py @@ -0,0 +1,101 @@ +import pandas as pd +import pycountry + + +def scale_to_resolution_and_create_file( + df, region_country_mapping, populations, resolution, output_path +): + if resolution == "national": + df = df + elif resolution == "continental": + df = df.sum(axis=1).to_frame("EUR") + elif resolution == "regional": + df = scale_national_to_regional(df, region_country_mapping, populations) + else: + raise ValueError(f"Resolution {resolution} is not supported") + df.to_csv(output_path) + + +def scale_national_to_regional(df, region_country_mapping, populations): + df_population_share = ( + populations.loc[:, "population_sum"] + .reindex(region_country_mapping.keys()) + .groupby(by=region_country_mapping) + .transform(lambda df: df / df.sum()) + ) + + regional_df = ( + pd.DataFrame( + index=df.index, + data={ + id: df[country_code] + for id, country_code in region_country_mapping.items() + }, + ) + .mul(df_population_share) + .rename(columns=lambda col_name: col_name.replace(".", "-")) + ) + pd.testing.assert_series_equal(regional_df.sum(axis=1), df.sum(axis=1)) + return regional_df + + +def get_national_ev_profiles( + ev_profiles_path: str, + dataset_name: str, + demand_range: dict[str, int], + first_year: int, + final_year: int, + country_neighbour_dict: dict[str, list[str]], + country_codes: list[str], +): + df_timeseries = ( + pd.read_csv(ev_profiles_path, index_col=[0, 1, 2], parse_dates=[0]) + .xs(slice(first_year, final_year), level="year") + .unstack("country_code") + .droplevel(level=0, axis="columns") + .groupby(by=lambda idx: idx.year) + .transform(lambda x: x / x.sum()) + .pipe(fill_empty_country, country_neighbour_dict) + .loc[:, country_codes] + ) + if "demand" in dataset_name: + df = df_timeseries.mul(demand_range[dataset_name.split("-")[-1]]) + elif "plugin" in dataset_name: + df = df_timeseries + return df + + +def fill_empty_country(df, country_neighbour_dict): + for country, neighbours in country_neighbour_dict.items(): + assert country not in df.columns + df[country] = df[neighbours].mean(axis=1) + return df + + +if __name__ == "__main__": + region_country_mapping = ( + pd.read_csv(snakemake.input.locations, index_col=0) + .loc[:, "country_code"] + .to_dict() + ) + populations = pd.read_csv(snakemake.input.populations, index_col=0) + + df = get_national_ev_profiles( + ev_profiles_path=snakemake.input.ev_profiles, + dataset_name=snakemake.wildcards.dataset_name, + demand_range=snakemake.params.demand_range, + first_year=snakemake.params.first_year, + final_year=snakemake.params.final_year, + country_neighbour_dict=snakemake.params.country_neighbour_dict, + country_codes=([ + pycountry.countries.lookup(c).alpha_3 for c in snakemake.params.countries + ]), + ) + + scale_to_resolution_and_create_file( + df=df, + region_country_mapping=region_country_mapping, + populations=populations, + resolution=snakemake.wildcards.resolution, + output_path=snakemake.output[0], + ) diff --git a/templates/models/techs/demand/electrified-transport.yaml b/templates/models/techs/demand/electrified-transport.yaml index bfba831b..4c8e8003 100644 --- a/templates/models/techs/demand/electrified-transport.yaml +++ b/templates/models/techs/demand/electrified-transport.yaml @@ -44,6 +44,19 @@ overrides: {% endfor %} {% endfor %} + monthly_transport_demand_range: + techs: + demand_road_transport_electrified_controlled: + constraints: + demand_shape_per_month_min: file=demand/demand-shape-min-ev.csv + demand_shape_per_month_max: file=demand/demand-shape-max-ev.csv + + monthly_transport_demand_equality: + techs: + demand_road_transport_electrified_controlled: + constraints: + demand_shape_per_month_equals: file=demand/demand-shape-equals-ev.csv + locations: {% for id, location in locations.iterrows() %} {{ id }}.techs: From ba65333f85bf69f5f45db9d307cb83808e7aa459 Mon Sep 17 00:00:00 2001 From: adrienmellot Date: Wed, 10 Apr 2024 22:31:29 +0200 Subject: [PATCH 04/24] edit csv file format --- scripts/transport/road_transport_controlled_constraints.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/transport/road_transport_controlled_constraints.py b/scripts/transport/road_transport_controlled_constraints.py index 208d635f..af7a8f2b 100644 --- a/scripts/transport/road_transport_controlled_constraints.py +++ b/scripts/transport/road_transport_controlled_constraints.py @@ -13,7 +13,7 @@ def scale_to_resolution_and_create_file( df = scale_national_to_regional(df, region_country_mapping, populations) else: raise ValueError(f"Resolution {resolution} is not supported") - df.to_csv(output_path) + df.tz_localize(None).rename_axis("utc-timestamp").to_csv(output_path) def scale_national_to_regional(df, region_country_mapping, populations): From 6cb137c88ab0253b1a0b8e223052691fa4fec78e Mon Sep 17 00:00:00 2001 From: adrienmellot Date: Thu, 11 Apr 2024 16:20:42 +0200 Subject: [PATCH 05/24] Extract EV chargeable capacities Co-Authored-By: Francesco Sanvito <68587472+FraSanvit@users.noreply.github.com> --- Snakefile | 2 +- config/default.yaml | 7 + config/schema.yaml | 23 +++ rules/transport.smk | 21 +++ .../extract_ev_installed_capacities.py | 137 ++++++++++++++++++ 5 files changed, 189 insertions(+), 1 deletion(-) create mode 100644 scripts/transport/extract_ev_installed_capacities.py diff --git a/Snakefile b/Snakefile index 2d54552f..704e2221 100644 --- a/Snakefile +++ b/Snakefile @@ -32,7 +32,7 @@ wildcard_constraints: ruleorder: area_to_capacity_limits > hydro_capacities > biofuels > nuclear_regional_capacity > dummy_tech_locations_template ruleorder: bio_techs_and_locations_template > techs_and_locations_template -ruleorder: create_controlled_road_transport_annual_demand > dummy_tech_locations_template +ruleorder: create_controlled_road_transport_annual_demand > extract_ev_installed_capacities_for_controlled_charging > dummy_tech_locations_template ALL_CF_TECHNOLOGIES = [ "wind-onshore", "wind-offshore", "open-field-pv", diff --git a/config/default.yaml b/config/default.yaml index 9e9fad01..0e8007c7 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -50,6 +50,7 @@ scaling-factors: # values are tuned for models with a few hours resolution and o power: 0.00001 # from MW(h) to 100 GW(h) area: 0.0001 # from km2 to 10,000 km2 monetary: 0.000000001 # from EUR to 1 billion EUR + transport: 0.01 # from Mio km to 100 Mio km capacity-factors: min: 0.001 max: 10 # 0.001 -> 10 leads to a numerical range of 1e5 (hourly resolution) @@ -162,6 +163,12 @@ parameters: coaches-and-buses: Motor coaches, buses and trolley buses passenger-cars: Passenger cars motorcycles: Powered 2-wheelers + ev-battery-sizes: # MWh + heavy-duty-vehicles: 0.2 # average from [EUCAR_2019] + light-duty-vehicles: 0.1 # own assumption based on passenger cars from [EUCAR_2019] + motorcycles: 0.01 # own assumption + coaches-and-buses: 0.2 # own assumption based on HDVs from [EUCAR_2019] + passenger-cars: 0.08 # average from [EUCAR_2019] uncontrolled-ev-charging-share: 0.5 monthly_demand_ranges: min: 0.9 diff --git a/config/schema.yaml b/config/schema.yaml index adc7c135..463ff18d 100644 --- a/config/schema.yaml +++ b/config/schema.yaml @@ -178,6 +178,9 @@ properties: monetary: type: number description: "Applied to all quantities of monetary cost (base: EUR)." + transport: + type: number + description: "Applied to all quantities of transport (base: Mio km)." parameters: type: object description: Parameter values of the models. @@ -348,6 +351,26 @@ properties: motorcycles: type: string description: JRC-IDEES name of motorcycles. + ev-battery-sizes: + type: object + description: EV battery size per vehicle type. + additionalProperties: false + properties: + light-duty-vehicles: + type: number + description: Light duty vehicles / trucks. + heavy-duty-vehicles: + type: number + description: Heavy duty vehicles / trucks. + coaches-and-buses: + type: number + description: Coaches and buses. + passenger-cars: + type: number + description: Passenger cars. + motorcycles: + type: number + description: Motorcylces. uncontrolled-ev-charging-share: type: number description: Share of uncontrolled charging. diff --git a/rules/transport.smk b/rules/transport.smk index 48988da6..d774299e 100644 --- a/rules/transport.smk +++ b/rules/transport.smk @@ -66,6 +66,27 @@ rule create_controlled_road_transport_annual_demand: main = "build/data/{resolution}/demand/electrified-transport.csv", script: "../scripts/transport/road_transport_controlled_charging.py" +rule extract_ev_installed_capacities_for_controlled_charging: + message: "Extract electrified road transport installed capacities for controlled charging at {wildcards.resolution} resolution" + input: + ev_vehicle_number = "build/data/jrc-idees/transport/processed-road-vehicles.csv", + jrc_road_distance = "build/data/jrc-idees/transport/processed-road-distance.csv", + locations = "build/data/regional/units.csv", + populations = "build/data/regional/population.csv", + params: + first_year = config["scope"]["temporal"]["first-year"], + final_year = config["scope"]["temporal"]["final-year"], + transport_scaling_factor = config["scaling-factors"]["transport"], + conversion_factors = config["parameters"]["transport"]["road-transport-conversion-factors"], + battery_sizes = config["parameters"]["transport"]["ev-battery-sizes"], + countries = config["scope"]["spatial"]["countries"], + country_neighbour_dict = config["data-pre-processing"]["fill-missing-values"]["ramp"], + fill_missing_values_jrc = config["data-pre-processing"]["fill-missing-values"]["jrc-idees"], + conda: "../envs/default.yaml" + output: + main = "build/data/{resolution}/transport/ev-installed-capacities.csv", + script: "../scripts/transport/extract_ev_installed_capacities.py" + rule create_controlled_ev_charging_parameters: message: "Create timeseries parameters for controlled EV charging" input: diff --git a/scripts/transport/extract_ev_installed_capacities.py b/scripts/transport/extract_ev_installed_capacities.py new file mode 100644 index 00000000..3f7f67cc --- /dev/null +++ b/scripts/transport/extract_ev_installed_capacities.py @@ -0,0 +1,137 @@ +import pandas as pd +import pycountry + + +def scale_to_regional_resolution(df, region_country_mapping, populations): + """ + Create regional electricity demand for controlled charging. + ASSUME all road transport is subnationally distributed in proportion to population. + """ + df_population_share = ( + populations.loc[:, "population_sum"] + .reindex(region_country_mapping.keys()) + .groupby(by=region_country_mapping) + .transform(lambda df: df / df.sum()) + ) + regional_df = ( + pd.DataFrame( + index=df.index, + data={ + id: df[country_code] + for id, country_code in region_country_mapping.items() + }, + ) + .mul(df_population_share) + .rename(columns=lambda col_name: col_name.replace(".", "-")) + ) + pd.testing.assert_series_equal(regional_df.sum(axis=1), df.sum(axis=1)) + return regional_df + + +def scale_to_national_resolution(df): + df.columns.name = None + return df + + +def scale_to_continental_resolution(df): + return df.sum(axis=1).to_frame("EUR") + + +def extract_national_ev_numbers( + path_to_ev_numbers: str, + transport_scaling_factor: float, + first_year: int, + final_year: int, + conversion_factors: dict[str, float], + battery_sizes: dict[str, float], + country_codes: list[str], +): + # Extract number of EVs per vehicle type + df_ev_numbers = ( + pd.read_csv(path_to_ev_numbers, index_col=[0, 1, 2, 3, 4]) + .squeeze() + .droplevel(["vehicle_subtype", "section"]) + ) + assert df_ev_numbers.isnull().values.any() == False + # Compute max. distance travelled per full battery for one EV [in Mio km / vehicle] + battery_size = pd.DataFrame.from_dict( + { + vehicle: battery_sizes[vehicle] / conversion_factors[vehicle] + for vehicle in battery_sizes + }, + orient="index", + columns=["value"], + ) + + # Compute available chargeable distance per vehicle type [in transport scaling unit km] + df_ev_chargeable_distance = ( + df_ev_numbers.align(battery_size, level="vehicle_type")[1] + .squeeze() + .mul(df_ev_numbers) + .groupby(level=["country_code", "year"]) + .sum() + .loc[country_codes] + .unstack("year") + .mul(transport_scaling_factor) + ) + df_ev_chargeable_distance = df_ev_chargeable_distance.assign(**{ + str(year): df_ev_chargeable_distance[2015] for year in range(2016, 2019) + }) + + df_ev_chargeable_distance.columns = df_ev_chargeable_distance.columns.astype(int) + df_ev_chargeable_distance.index.name = None + + return df_ev_chargeable_distance[range(first_year, final_year + 1)].T + + +def compute_weighted_share_per_vehicle_type(path_to_vehicle_type_distance): + return ( + pd.read_csv(path_to_vehicle_type_distance, index_col=[0, 1, 2, 3, 4]) + .droplevel(["vehicle_subtype", "section"]) + .groupby(level=["country_code", "year"]) + .transform(lambda x: (x / x.sum())) + ) + + +if __name__ == "__main__": + resolution = snakemake.wildcards.resolution + + path_to_ev_numbers = snakemake.input.ev_vehicle_number + transport_scaling_factor = snakemake.params.transport_scaling_factor + first_year = snakemake.params.first_year + final_year = snakemake.params.final_year + conversion_factors = snakemake.params.conversion_factors + battery_sizes = snakemake.params.battery_sizes + path_to_output = snakemake.output[0] + country_codes = [ + pycountry.countries.lookup(c).alpha_3 for c in snakemake.params.countries + ] + region_country_mapping = ( + pd.read_csv(snakemake.input.locations, index_col=0) + .loc[:, "country_code"] + .to_dict() + ) + populations = pd.read_csv(snakemake.input.populations, index_col=0) + + df = extract_national_ev_numbers( + path_to_ev_numbers, + transport_scaling_factor, + first_year, + final_year, + conversion_factors, + battery_sizes, + country_codes, + ) + breakpoint() + if resolution == "continental": + df = scale_to_continental_resolution(df) + elif resolution == "national": + df = scale_to_national_resolution(df) + elif resolution == "regional": + df = scale_to_regional_resolution( + df, region_country_mapping=region_country_mapping, populations=populations + ) + else: + raise ValueError("Input resolution is not recognised") + breakpoint() + df.T.to_csv(path_to_output, index_label=["id"]) From 2cd9b9f5f25bade089be0e5a3e603673b1798042 Mon Sep 17 00:00:00 2001 From: adrienmellot Date: Thu, 11 Apr 2024 17:32:51 +0200 Subject: [PATCH 06/24] Have max charging potential exported to yaml Co-Authored-By: Francesco Sanvito <68587472+FraSanvit@users.noreply.github.com> --- Snakefile | 2 +- rules/transport.smk | 29 +---- .../road_transport_controlled_charging.py | 104 ++++++++++++++++-- .../techs/demand/electrified-transport.yaml | 29 ++++- 4 files changed, 126 insertions(+), 38 deletions(-) diff --git a/Snakefile b/Snakefile index 704e2221..6c621d01 100644 --- a/Snakefile +++ b/Snakefile @@ -32,7 +32,7 @@ wildcard_constraints: ruleorder: area_to_capacity_limits > hydro_capacities > biofuels > nuclear_regional_capacity > dummy_tech_locations_template ruleorder: bio_techs_and_locations_template > techs_and_locations_template -ruleorder: create_controlled_road_transport_annual_demand > extract_ev_installed_capacities_for_controlled_charging > dummy_tech_locations_template +ruleorder: create_controlled_road_transport_annual_demand_and_installed_capacities > dummy_tech_locations_template ALL_CF_TECHNOLOGIES = [ "wind-onshore", "wind-offshore", "open-field-pv", diff --git a/rules/transport.smk b/rules/transport.smk index d774299e..15f3b5b2 100644 --- a/rules/transport.smk +++ b/rules/transport.smk @@ -48,27 +48,10 @@ rule annual_transport_demand: road_distance_historically_electrified = "build/data/transport/annual-road-transport-distance-demand-historic-electrification.csv", script: "../scripts/transport/annual_transport_demand.py" -rule create_controlled_road_transport_annual_demand: - message: "Create annual demand for controlled charging at {wildcards.resolution} resolution" +rule create_controlled_road_transport_annual_demand_and_installed_capacities: + message: "Create annual demand for controlled charging and corresponding charging potentials at {wildcards.resolution} resolution" input: annual_controlled_demand = "build/data/transport/annual-road-transport-distance-demand-controlled.csv", - locations = "build/data/regional/units.csv", - populations = "build/data/regional/population.csv", - params: - first_year = config["scope"]["temporal"]["first-year"], - final_year = config["scope"]["temporal"]["final-year"], - power_scaling_factor = config["scaling-factors"]["power"], - conversion_factors = config["parameters"]["transport"]["road-transport-conversion-factors"], - countries = config["scope"]["spatial"]["countries"], - country_neighbour_dict = config["data-pre-processing"]["fill-missing-values"]["ramp"], - conda: "../envs/default.yaml" - output: - main = "build/data/{resolution}/demand/electrified-transport.csv", - script: "../scripts/transport/road_transport_controlled_charging.py" - -rule extract_ev_installed_capacities_for_controlled_charging: - message: "Extract electrified road transport installed capacities for controlled charging at {wildcards.resolution} resolution" - input: ev_vehicle_number = "build/data/jrc-idees/transport/processed-road-vehicles.csv", jrc_road_distance = "build/data/jrc-idees/transport/processed-road-distance.csv", locations = "build/data/regional/units.csv", @@ -76,16 +59,16 @@ rule extract_ev_installed_capacities_for_controlled_charging: params: first_year = config["scope"]["temporal"]["first-year"], final_year = config["scope"]["temporal"]["final-year"], + power_scaling_factor = config["scaling-factors"]["power"], transport_scaling_factor = config["scaling-factors"]["transport"], - conversion_factors = config["parameters"]["transport"]["road-transport-conversion-factors"], battery_sizes = config["parameters"]["transport"]["ev-battery-sizes"], + conversion_factors = config["parameters"]["transport"]["road-transport-conversion-factors"], countries = config["scope"]["spatial"]["countries"], country_neighbour_dict = config["data-pre-processing"]["fill-missing-values"]["ramp"], - fill_missing_values_jrc = config["data-pre-processing"]["fill-missing-values"]["jrc-idees"], conda: "../envs/default.yaml" output: - main = "build/data/{resolution}/transport/ev-installed-capacities.csv", - script: "../scripts/transport/extract_ev_installed_capacities.py" + main = "build/data/{resolution}/demand/electrified-transport.csv", + script: "../scripts/transport/road_transport_controlled_charging.py" rule create_controlled_ev_charging_parameters: message: "Create timeseries parameters for controlled EV charging" diff --git a/scripts/transport/road_transport_controlled_charging.py b/scripts/transport/road_transport_controlled_charging.py index ccb9fe91..b95c018e 100644 --- a/scripts/transport/road_transport_controlled_charging.py +++ b/scripts/transport/road_transport_controlled_charging.py @@ -65,6 +65,57 @@ def convert_annual_distance_to_electricity_demand( return -df_energy_demand +def extract_national_ev_charging_potentials( + path_to_ev_numbers: str, + transport_scaling_factor: float, + first_year: int, + final_year: int, + conversion_factors: dict[str, float], + battery_sizes: dict[str, float], + country_codes: list[str], +): + # Extract number of EVs per vehicle type + df_ev_numbers = ( + pd.read_csv(path_to_ev_numbers, index_col=[0, 1, 2, 3, 4]) + .squeeze() + .droplevel(["vehicle_subtype", "section"]) + ) + assert df_ev_numbers.isnull().values.any() == False + # Compute max. distance travelled per full battery for one EV [in Mio km / vehicle] + battery_size = pd.DataFrame.from_dict( + { + vehicle: battery_sizes[vehicle] / conversion_factors[vehicle] + for vehicle in battery_sizes + }, + orient="index", + columns=["value"], + ) + + # Compute available chargeable distance per vehicle type [in transport scaling unit km] + df_ev_chargeable_distance = ( + df_ev_numbers.align(battery_size, level="vehicle_type")[1] + .squeeze() + .mul(df_ev_numbers) + .groupby(level=["country_code", "year"]) + .sum() + .loc[country_codes] + .unstack("year") + .mul(transport_scaling_factor) + ) + df_ev_chargeable_distance = df_ev_chargeable_distance.assign(**{ + str(year): df_ev_chargeable_distance[2015] for year in range(2016, 2019) + }) + + df_ev_chargeable_distance.columns = df_ev_chargeable_distance.columns.astype(int) + # df_ev_chargeable_distance.index.name = None + + return df_ev_chargeable_distance[range(first_year, final_year + 1)].T + + +def reshape_and_add_suffix(df, suffix): + return df.T.add_suffix(suffix) + + if __name__ == "__main__": resolution = snakemake.wildcards.resolution @@ -83,8 +134,12 @@ def convert_annual_distance_to_electricity_demand( .to_dict() ) populations = pd.read_csv(snakemake.input.populations, index_col=0) + battery_sizes = snakemake.params.battery_sizes + transport_scaling_factor = snakemake.params.transport_scaling_factor + path_to_ev_numbers = snakemake.input.ev_vehicle_number - df = convert_annual_distance_to_electricity_demand( + # Convert annual distance driven demand to electricity demand for controlled charging + df_demand = convert_annual_distance_to_electricity_demand( path_to_controlled_annual_demand, power_scaling_factor, first_year, @@ -93,14 +148,39 @@ def convert_annual_distance_to_electricity_demand( country_codes, ) - if resolution == "continental": - df = scale_to_continental_resolution(df) - elif resolution == "national": - df = scale_to_national_resolution(df) - elif resolution == "regional": - df = scale_to_regional_resolution( - df, region_country_mapping=region_country_mapping, populations=populations - ) - else: - raise ValueError("Input resolution is not recognised") - df.T.to_csv(path_to_output, index_label=["id"]) + # Extract national EV charging potentials + df_charging_potentials = extract_national_ev_charging_potentials( + path_to_ev_numbers, + transport_scaling_factor, + first_year, + final_year, + conversion_factors, + battery_sizes, + country_codes, + ) + + # Add prefix for yaml template + parameters_evs = { + "_demand": df_demand, + "_charging": df_charging_potentials, + } + + # Rescale to desired resolution and add suffix + dfs = [] + for key, df in parameters_evs.items(): + if resolution == "continental": + df = scale_to_continental_resolution(df) + elif resolution == "national": + df = scale_to_national_resolution(df) + elif resolution == "regional": + df = scale_to_regional_resolution( + df, + region_country_mapping=region_country_mapping, + populations=populations, + ) + else: + raise ValueError("Input resolution is not recognised") + dfs.append(reshape_and_add_suffix(df, key)) + + # Export to csv + pd.concat(dfs, axis=1).to_csv(path_to_output, index_label=["id"]) diff --git a/templates/models/techs/demand/electrified-transport.yaml b/templates/models/techs/demand/electrified-transport.yaml index 4c8e8003..35b334d3 100644 --- a/templates/models/techs/demand/electrified-transport.yaml +++ b/templates/models/techs/demand/electrified-transport.yaml @@ -25,6 +25,15 @@ techs: force_resource: false resource: -.inf + road_transport_controlled_dummy: + essentials: + name: 'Dummy tech for controlled road transport demand -- required for max potential charging' + parent: conversion + carrier_in: electricity + carrier_out: electricity + constraints: + energy_eff: 1 + overrides: keep-historic-electricity-demand-from-road-transport: # TODO: possibly remove this override as there may be no use-cases for it. @@ -33,15 +42,17 @@ overrides: {% endfor %} {% for year in locations.columns %} - {{ year }}_transport_controlled_electrified_demand: + {% if "demand" in year %} + {{ year }}_transport_controlled_electrified: group_constraints: {% for location in locations.index %} - {{ location }}_annual_controlled_electricity_demand: + {{ location }}_annual_controlled_electricity: locs: [{{ location }}] techs: [demand_road_transport_electrified_controlled] carrier_con_equals: electricity: {{ locations.loc[location, year] }} # {{ (1 / scaling_factors.power) | unit("MWh") }} {% endfor %} + {% endif %} {% endfor %} monthly_transport_demand_range: @@ -56,6 +67,20 @@ overrides: demand_road_transport_electrified_controlled: constraints: demand_shape_per_month_equals: file=demand/demand-shape-equals-ev.csv + + {% for year in locations.columns %} + {% if "charging" in year %} + {{ year }}_max_ev_potential: + locations: + {% for location in locations.index %} + {{ location }}: + techs: + road_transport_controlled_dummy: + constraints: + energy_cap_max: {{ locations.loc[location, year] }} # {{ (1 / scaling_factors.transport) | unit("Mio km") }} + {% endfor %} + {% endif %} + {% endfor %} locations: {% for id, location in locations.iterrows() %} From 4c4621f3e6c65076f2571fbdb4652adf0fc85fa4 Mon Sep 17 00:00:00 2001 From: adrienmellot Date: Thu, 11 Apr 2024 17:53:29 +0200 Subject: [PATCH 07/24] minor yaml update Co-Authored-By: Francesco Sanvito <68587472+FraSanvit@users.noreply.github.com> --- Snakefile | 16 ++++++++-------- .../techs/demand/electrified-transport.yaml | 3 +++ 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/Snakefile b/Snakefile index 6c621d01..dac207fe 100644 --- a/Snakefile +++ b/Snakefile @@ -69,20 +69,20 @@ onerror: rule all: message: "Generate euro-calliope pre-built models and run tests." input: - "build/logs/continental/test.success", - "build/logs/national/test.success", + #"build/logs/continental/test.success", + #"build/logs/national/test.success", "build/models/continental/example-model.yaml", "build/models/national/example-model.yaml", "build/models/regional/example-model.yaml", "build/models/continental/build-metadata.yaml", "build/models/national/build-metadata.yaml", "build/models/regional/build-metadata.yaml", - "build/models/regional/summary-of-potentials.nc", - "build/models/regional/summary-of-potentials.csv", - "build/models/national/summary-of-potentials.nc", - "build/models/national/summary-of-potentials.csv", - "build/models/continental/summary-of-potentials.nc", - "build/models/continental/summary-of-potentials.csv" + # "build/models/regional/summary-of-potentials.nc", + # "build/models/regional/summary-of-potentials.csv", + # "build/models/national/summary-of-potentials.nc", + # "build/models/national/summary-of-potentials.csv", + # "build/models/continental/summary-of-potentials.nc", + # "build/models/continental/summary-of-potentials.csv" rule all_tests: diff --git a/templates/models/techs/demand/electrified-transport.yaml b/templates/models/techs/demand/electrified-transport.yaml index 35b334d3..f5bb32d1 100644 --- a/templates/models/techs/demand/electrified-transport.yaml +++ b/templates/models/techs/demand/electrified-transport.yaml @@ -26,6 +26,7 @@ techs: resource: -.inf road_transport_controlled_dummy: + exists: false essentials: name: 'Dummy tech for controlled road transport demand -- required for max potential charging' parent: conversion @@ -71,6 +72,8 @@ overrides: {% for year in locations.columns %} {% if "charging" in year %} {{ year }}_max_ev_potential: + techs: + road_transport_controlled_dummy.exists: true locations: {% for location in locations.index %} {{ location }}: From 0e1cebc43bc8095199f6a7097528e2b9ac1597a4 Mon Sep 17 00:00:00 2001 From: adrienmellot Date: Fri, 12 Apr 2024 10:56:02 +0200 Subject: [PATCH 08/24] Fix ramp data download Co-Authored-By: Francesco Sanvito <68587472+FraSanvit@users.noreply.github.com> --- config/default.yaml | 2 +- .../road_transport_controlled_constraints.py | 14 ++++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/config/default.yaml b/config/default.yaml index 0e8007c7..6a6aacb9 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -19,7 +19,7 @@ data-sources: swiss-end-use: https://www.bfe.admin.ch/bfe/en/home/versorgung/statistik-und-geodaten/energiestatistiken/energieverbrauch-nach-verwendungszweck.exturl.html/aHR0cHM6Ly9wdWJkYi5iZmUuYWRtaW4uY2gvZGUvcHVibGljYX/Rpb24vZG93bmxvYWQvOTg1NA==.html swiss-energy-balance: https://www.bfe.admin.ch/bfe/en/home/versorgung/statistik-und-geodaten/energiestatistiken/gesamtenergiestatistik.exturl.html/aHR0cHM6Ly9wdWJkYi5iZmUuYWRtaW4uY2gvZGUvcHVibGljYX/Rpb24vZG93bmxvYWQvNzUxOQ==.html swiss-industry-energy-balance: https://www.bfe.admin.ch/bfe/en/home/versorgung/statistik-und-geodaten/energiestatistiken/teilstatistiken.exturl.html/aHR0cHM6Ly9wdWJkYi5iZmUuYWRtaW4uY2gvZGUvcHVibGljYX/Rpb24vZG93bmxvYWQvODc4OA==.html - controlled-ev-data: https://zenodo.org/record/6579421/files/ramp-ev-consumption-profiles.csv.gz?download=1 + controlled-ev-data: https://zenodo.org/record/6579421/files/ramp-ev-{dataset}.csv.gz?download=1 data-pre-processing: fill-missing-values: jrc-idees: diff --git a/scripts/transport/road_transport_controlled_constraints.py b/scripts/transport/road_transport_controlled_constraints.py index af7a8f2b..b4c4b15d 100644 --- a/scripts/transport/road_transport_controlled_constraints.py +++ b/scripts/transport/road_transport_controlled_constraints.py @@ -53,16 +53,18 @@ def get_national_ev_profiles( .xs(slice(first_year, final_year), level="year") .unstack("country_code") .droplevel(level=0, axis="columns") - .groupby(by=lambda idx: idx.year) - .transform(lambda x: x / x.sum()) - .pipe(fill_empty_country, country_neighbour_dict) - .loc[:, country_codes] ) if "demand" in dataset_name: - df = df_timeseries.mul(demand_range[dataset_name.split("-")[-1]]) + # Normalise demand and create min-max-equals timeseries + df = ( + df_timeseries.groupby(by=lambda idx: idx.year) + .transform(lambda x: x / x.sum()) + .mul(demand_range[dataset_name.split("-")[-1]]) + ) elif "plugin" in dataset_name: + # plugin-profiles are already normalised df = df_timeseries - return df + return df.pipe(fill_empty_country, country_neighbour_dict).loc[:, country_codes] def fill_empty_country(df, country_neighbour_dict): From ecd75fe9049c5cef0326f98cf0a03f41c02a03c4 Mon Sep 17 00:00:00 2001 From: adrienmellot Date: Fri, 12 Apr 2024 11:28:25 +0200 Subject: [PATCH 09/24] Update custom constaint from file by adding _time_varying Co-Authored-By: Francesco Sanvito <68587472+FraSanvit@users.noreply.github.com> --- Snakefile | 16 ++++++++-------- .../transport/extract_ev_installed_capacities.py | 2 -- .../techs/demand/electrified-transport.yaml | 11 ++++++----- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/Snakefile b/Snakefile index dac207fe..6c621d01 100644 --- a/Snakefile +++ b/Snakefile @@ -69,20 +69,20 @@ onerror: rule all: message: "Generate euro-calliope pre-built models and run tests." input: - #"build/logs/continental/test.success", - #"build/logs/national/test.success", + "build/logs/continental/test.success", + "build/logs/national/test.success", "build/models/continental/example-model.yaml", "build/models/national/example-model.yaml", "build/models/regional/example-model.yaml", "build/models/continental/build-metadata.yaml", "build/models/national/build-metadata.yaml", "build/models/regional/build-metadata.yaml", - # "build/models/regional/summary-of-potentials.nc", - # "build/models/regional/summary-of-potentials.csv", - # "build/models/national/summary-of-potentials.nc", - # "build/models/national/summary-of-potentials.csv", - # "build/models/continental/summary-of-potentials.nc", - # "build/models/continental/summary-of-potentials.csv" + "build/models/regional/summary-of-potentials.nc", + "build/models/regional/summary-of-potentials.csv", + "build/models/national/summary-of-potentials.nc", + "build/models/national/summary-of-potentials.csv", + "build/models/continental/summary-of-potentials.nc", + "build/models/continental/summary-of-potentials.csv" rule all_tests: diff --git a/scripts/transport/extract_ev_installed_capacities.py b/scripts/transport/extract_ev_installed_capacities.py index 3f7f67cc..71dfca71 100644 --- a/scripts/transport/extract_ev_installed_capacities.py +++ b/scripts/transport/extract_ev_installed_capacities.py @@ -122,7 +122,6 @@ def compute_weighted_share_per_vehicle_type(path_to_vehicle_type_distance): battery_sizes, country_codes, ) - breakpoint() if resolution == "continental": df = scale_to_continental_resolution(df) elif resolution == "national": @@ -133,5 +132,4 @@ def compute_weighted_share_per_vehicle_type(path_to_vehicle_type_distance): ) else: raise ValueError("Input resolution is not recognised") - breakpoint() df.T.to_csv(path_to_output, index_label=["id"]) diff --git a/templates/models/techs/demand/electrified-transport.yaml b/templates/models/techs/demand/electrified-transport.yaml index f5bb32d1..a5a72fa1 100644 --- a/templates/models/techs/demand/electrified-transport.yaml +++ b/templates/models/techs/demand/electrified-transport.yaml @@ -28,12 +28,13 @@ techs: road_transport_controlled_dummy: exists: false essentials: - name: 'Dummy tech for controlled road transport demand -- required for max potential charging' + name: 'Dummy tech for controlled road transport demand -- required for max potential charging and demand shape' parent: conversion carrier_in: electricity carrier_out: electricity constraints: energy_eff: 1 + energy_cap_max_time_varying: file=demand/plugin-profiles-ev.csv overrides: keep-historic-electricity-demand-from-road-transport: @@ -60,15 +61,15 @@ overrides: techs: demand_road_transport_electrified_controlled: constraints: - demand_shape_per_month_min: file=demand/demand-shape-min-ev.csv - demand_shape_per_month_max: file=demand/demand-shape-max-ev.csv + demand_shape_per_month_min_time_varying: file=demand/demand-shape-min-ev.csv + demand_shape_per_month_max_time_varying: file=demand/demand-shape-max-ev.csv monthly_transport_demand_equality: techs: demand_road_transport_electrified_controlled: constraints: - demand_shape_per_month_equals: file=demand/demand-shape-equals-ev.csv - + demand_shape_per_month_equals_time_varying: file=demand/demand-shape-equals-ev.csv + {% for year in locations.columns %} {% if "charging" in year %} {{ year }}_max_ev_potential: From cffaedb382320e50b90834df225893cfacef3537 Mon Sep 17 00:00:00 2001 From: adrienmellot Date: Fri, 12 Apr 2024 12:41:21 +0200 Subject: [PATCH 10/24] fix force_resource on heat sector Co-Authored-By: Francesco Sanvito <68587472+FraSanvit@users.noreply.github.com> --- templates/models/techs/demand/electrified-heat.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/models/techs/demand/electrified-heat.yaml b/templates/models/techs/demand/electrified-heat.yaml index 4f0ad4f7..40467372 100644 --- a/templates/models/techs/demand/electrified-heat.yaml +++ b/templates/models/techs/demand/electrified-heat.yaml @@ -14,7 +14,7 @@ techs: carrier: electricity constraints: resource: file=demand/heat-demand-historic-electrification.csv - resource_min_use: 1 + force_resource: true overrides: keep-historic-electricity-demand-from-heat: From db9ed2adab97b9a6f3ac92adf9c80d07031f5b78 Mon Sep 17 00:00:00 2001 From: adrienmellot Date: Fri, 12 Apr 2024 12:45:03 +0200 Subject: [PATCH 11/24] Create run.py to include custom constraints Co-Authored-By: Francesco Sanvito <68587472+FraSanvit@users.noreply.github.com> --- config/run.py | 215 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 215 insertions(+) create mode 100644 config/run.py diff --git a/config/run.py b/config/run.py new file mode 100644 index 00000000..fb15f151 --- /dev/null +++ b/config/run.py @@ -0,0 +1,215 @@ +import sys + +import calliope +import numpy as np +import pyomo.core as po +from calliope.backend.pyomo.util import ( + get_param, + invalid, + split_comma_list, +) + +""" +This file allows to run a Calliope model with custom constraints + +To run it, use the following command: + python run.py path_to_model scenario_string path_to_output + + where: + - path_to_model is the path to the Calliope model to run, + - scenario_string is the scenario to run, + - and path_to_output is the path to the output .nc file. + +Make sure to have a Calliope environment which can run the model, e.g. the test environment. +""" + +args = sys.argv[1:] +path_to_model = args[0] +scenario_string = args[1] +path_to_output = args[2] + + +def build_model(path_to_model, scenario, path_to_temp_output): + # Build model without custom constraints. Does not run the model + calliope.set_log_verbosity( + "info", include_solver_output=True, capture_warnings=True + ) + model = calliope.Model(path_to_model, scenario=scenario) + + model._model_data.attrs["scenario"] = scenario + + model.to_netcdf(path_to_temp_output) + + +def run_model(path_to_temp_model, path_to_output): + # Run model with custom constraints + calliope.set_log_verbosity( + "info", include_solver_output=True, capture_warnings=True + ) + model = calliope.read_netcdf(path_to_temp_model) + model.run(build_only=True) + add_eurocalliope_constraints(model) + new_model = model.backend.rerun() + new_model.to_netcdf(path_to_output) + + +def add_eurocalliope_constraints(model): + backend_model = model._backend_model + if "energy_cap_max_time_varying" in model._model_data.data_vars: + print("Building production_max_time_varying constraint") + add_production_max_time_varying_constraint(model, backend_model) + if any( + var.startswith("demand_shape_per_month") for var in model._model_data.data_vars + ): + print("Building demand_shape_per_month constraint") + add_carrier_prod_per_month_constraints(model, backend_model) + + +def equalizer(lhs, rhs, sign): + if sign == "max": + return lhs <= rhs + elif sign == "min": + return lhs >= rhs + elif sign == "equals": + return lhs == rhs + else: + raise ValueError(f"Invalid sign: {sign}") + + +def add_production_max_time_varying_constraint(model, backend_model): + def _carrier_production_max_time_varying_constraint_rule( + backend_model, loc_tech, timestep + ): + """ + Set maximum carrier production for technologies with time varying maximum capacity + """ + energy_cap_max = backend_model.energy_cap_max_time_varying[loc_tech, timestep] + if invalid(energy_cap_max): + return po.Constraint.Skip + model_data_dict = backend_model.__calliope_model_data["data"] + timestep_resolution = backend_model.timestep_resolution[timestep] + loc_tech_carriers_out = split_comma_list( + model_data_dict["lookup_loc_techs_conversion"]["out", loc_tech] + ) + + carrier_prod = sum( + backend_model.carrier_prod[loc_tech_carrier, timestep] + for loc_tech_carrier in loc_tech_carriers_out + ) + return carrier_prod <= ( + backend_model.energy_cap[loc_tech] * timestep_resolution * energy_cap_max + ) + + backend_model.loc_tech_carrier_production_max_time_varying_constraint = po.Set( + initialize=[ + loc_tech + for loc_tech in backend_model.loc_techs_conversion + if model.inputs.energy_cap_max_time_varying.loc[{"loc_techs": loc_tech}] + .notnull() + .all() + ], + ordered=True, + ) + model.backend.add_constraint( + "carrier_production_max_time_varying_constraint", + ["loc_tech_carrier_production_max_time_varying_constraint", "timesteps"], + _carrier_production_max_time_varying_constraint_rule, + ) + + +def add_carrier_prod_per_month_constraints(model, backend_model): + def _carrier_prod_per_month_constraint_rule_generator(sense): + def __carrier_prod_per_month_constraint_rule(backend_model, loc_tech, month): + """ + Set the min/max amount of carrier consumption (relative to annual consumption) + for a specific loc tech that must take place in a given calender month in the model + """ + model_data_dict = backend_model.__calliope_model_data + loc_tech_carrier = model_data_dict["data"]["lookup_loc_techs_conversion"][ + ("out", loc_tech) + ] + + prod = backend_model.carrier_prod + prod_total = sum( + prod[loc_tech_carrier, timestep] for timestep in backend_model.timesteps + ) + prod_month = sum( + prod[loc_tech_carrier, timestep] + for timestep in backend_model.timesteps + if backend_model.month_numbers[timestep].value == month + ) + if "timesteps" in [ + i.name + for i in getattr( + backend_model, f"carrier_prod_per_month_{sense}_time_varying" + )._index.subsets() + ]: + prod_fraction = sum( + get_param( + backend_model, + f"carrier_prod_per_month_{sense}_time_varying", + (loc_tech, timestep), + ) + * backend_model.timestep_resolution[timestep] + for timestep in backend_model.timesteps + if backend_model.month_numbers[timestep].value == month + ) + else: + prod_fraction = get_param( + backend_model, f"carrier_prod_per_month_{sense}", (loc_tech) + ) + + return equalizer(prod_month, prod_total * prod_fraction, sense) + + return __carrier_prod_per_month_constraint_rule + + backend_model.months = po.Set( + initialize=np.unique(model._model_data.timesteps.dt.month.values), ordered=True + ) + month_numbers = model._model_data.timesteps.dt.month.to_series() + month_numbers.index = month_numbers.index.strftime("%Y-%m-%d %H:%M") + + backend_model.month_numbers = po.Param( + backend_model.timesteps, + initialize=month_numbers.to_dict(), + mutable=True, + within=po.Reals, + ) + backend_model.__calliope_datetime_data.add(("data_vars", "month_numbers")) + + for sense in ["min", "max", "equals"]: + if ( + f"carrier_prod_per_month_{sense}_time_varying" + in model._model_data.data_vars + ): + setattr( + backend_model, + f"loc_techs_carrier_prod_per_month_{sense}", + po.Set( + initialize=[ + loc_tech + for loc_tech in backend_model.loc_techs + if ( + model._model_data[ + f"carrier_prod_per_month_{sense}_time_varying" + ] + .loc[{"loc_techs": loc_tech}] + .notnull() + .all() + ) + ], + ordered=True, + ), + ) + model.backend.add_constraint( + f"carrier_prod_per_month_{sense}_constraint", + [f"loc_techs_carrier_prod_per_month_{sense}", "months"], + _carrier_prod_per_month_constraint_rule_generator(sense), + ) + + +build_model(path_to_model, scenario_string, path_to_output) +run_model(path_to_output, path_to_output) + +model = calliope.read_netcdf(path_to_output) +model.plot.timeseries(subset={"locs": ["IRL"]}) From f066a5f10582f7a0ae7d937e068680683b0c5150 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 12 Apr 2024 10:49:06 +0000 Subject: [PATCH 12/24] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- rules/transport.smk | 2 +- templates/models/techs/demand/electrified-transport.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/rules/transport.smk b/rules/transport.smk index 15f3b5b2..dbac4df7 100644 --- a/rules/transport.smk +++ b/rules/transport.smk @@ -7,7 +7,7 @@ rule download_transport_timeseries: params: url = config["data-sources"]["controlled-ev-data"] conda: "../envs/shell.yaml" - output: + output: protected("data/automatic/ramp-ev-{dataset}.csv.gz") wildcard_constraints: dataset = "consumption-profiles|plugin-profiles" diff --git a/templates/models/techs/demand/electrified-transport.yaml b/templates/models/techs/demand/electrified-transport.yaml index a5a72fa1..1bd3e0bf 100644 --- a/templates/models/techs/demand/electrified-transport.yaml +++ b/templates/models/techs/demand/electrified-transport.yaml @@ -63,7 +63,7 @@ overrides: constraints: demand_shape_per_month_min_time_varying: file=demand/demand-shape-min-ev.csv demand_shape_per_month_max_time_varying: file=demand/demand-shape-max-ev.csv - + monthly_transport_demand_equality: techs: demand_road_transport_electrified_controlled: @@ -78,7 +78,7 @@ overrides: locations: {% for location in locations.index %} {{ location }}: - techs: + techs: road_transport_controlled_dummy: constraints: energy_cap_max: {{ locations.loc[location, year] }} # {{ (1 / scaling_factors.transport) | unit("Mio km") }} From f153a0de6197a35730db73d6ecbacbdcae45cd7f Mon Sep 17 00:00:00 2001 From: adrienmellot Date: Fri, 12 Apr 2024 12:54:33 +0200 Subject: [PATCH 13/24] delete unnecessary script which was merged in another Co-Authored-By: Francesco Sanvito <68587472+FraSanvit@users.noreply.github.com> --- .../extract_ev_installed_capacities.py | 135 ------------------ 1 file changed, 135 deletions(-) delete mode 100644 scripts/transport/extract_ev_installed_capacities.py diff --git a/scripts/transport/extract_ev_installed_capacities.py b/scripts/transport/extract_ev_installed_capacities.py deleted file mode 100644 index 71dfca71..00000000 --- a/scripts/transport/extract_ev_installed_capacities.py +++ /dev/null @@ -1,135 +0,0 @@ -import pandas as pd -import pycountry - - -def scale_to_regional_resolution(df, region_country_mapping, populations): - """ - Create regional electricity demand for controlled charging. - ASSUME all road transport is subnationally distributed in proportion to population. - """ - df_population_share = ( - populations.loc[:, "population_sum"] - .reindex(region_country_mapping.keys()) - .groupby(by=region_country_mapping) - .transform(lambda df: df / df.sum()) - ) - regional_df = ( - pd.DataFrame( - index=df.index, - data={ - id: df[country_code] - for id, country_code in region_country_mapping.items() - }, - ) - .mul(df_population_share) - .rename(columns=lambda col_name: col_name.replace(".", "-")) - ) - pd.testing.assert_series_equal(regional_df.sum(axis=1), df.sum(axis=1)) - return regional_df - - -def scale_to_national_resolution(df): - df.columns.name = None - return df - - -def scale_to_continental_resolution(df): - return df.sum(axis=1).to_frame("EUR") - - -def extract_national_ev_numbers( - path_to_ev_numbers: str, - transport_scaling_factor: float, - first_year: int, - final_year: int, - conversion_factors: dict[str, float], - battery_sizes: dict[str, float], - country_codes: list[str], -): - # Extract number of EVs per vehicle type - df_ev_numbers = ( - pd.read_csv(path_to_ev_numbers, index_col=[0, 1, 2, 3, 4]) - .squeeze() - .droplevel(["vehicle_subtype", "section"]) - ) - assert df_ev_numbers.isnull().values.any() == False - # Compute max. distance travelled per full battery for one EV [in Mio km / vehicle] - battery_size = pd.DataFrame.from_dict( - { - vehicle: battery_sizes[vehicle] / conversion_factors[vehicle] - for vehicle in battery_sizes - }, - orient="index", - columns=["value"], - ) - - # Compute available chargeable distance per vehicle type [in transport scaling unit km] - df_ev_chargeable_distance = ( - df_ev_numbers.align(battery_size, level="vehicle_type")[1] - .squeeze() - .mul(df_ev_numbers) - .groupby(level=["country_code", "year"]) - .sum() - .loc[country_codes] - .unstack("year") - .mul(transport_scaling_factor) - ) - df_ev_chargeable_distance = df_ev_chargeable_distance.assign(**{ - str(year): df_ev_chargeable_distance[2015] for year in range(2016, 2019) - }) - - df_ev_chargeable_distance.columns = df_ev_chargeable_distance.columns.astype(int) - df_ev_chargeable_distance.index.name = None - - return df_ev_chargeable_distance[range(first_year, final_year + 1)].T - - -def compute_weighted_share_per_vehicle_type(path_to_vehicle_type_distance): - return ( - pd.read_csv(path_to_vehicle_type_distance, index_col=[0, 1, 2, 3, 4]) - .droplevel(["vehicle_subtype", "section"]) - .groupby(level=["country_code", "year"]) - .transform(lambda x: (x / x.sum())) - ) - - -if __name__ == "__main__": - resolution = snakemake.wildcards.resolution - - path_to_ev_numbers = snakemake.input.ev_vehicle_number - transport_scaling_factor = snakemake.params.transport_scaling_factor - first_year = snakemake.params.first_year - final_year = snakemake.params.final_year - conversion_factors = snakemake.params.conversion_factors - battery_sizes = snakemake.params.battery_sizes - path_to_output = snakemake.output[0] - country_codes = [ - pycountry.countries.lookup(c).alpha_3 for c in snakemake.params.countries - ] - region_country_mapping = ( - pd.read_csv(snakemake.input.locations, index_col=0) - .loc[:, "country_code"] - .to_dict() - ) - populations = pd.read_csv(snakemake.input.populations, index_col=0) - - df = extract_national_ev_numbers( - path_to_ev_numbers, - transport_scaling_factor, - first_year, - final_year, - conversion_factors, - battery_sizes, - country_codes, - ) - if resolution == "continental": - df = scale_to_continental_resolution(df) - elif resolution == "national": - df = scale_to_national_resolution(df) - elif resolution == "regional": - df = scale_to_regional_resolution( - df, region_country_mapping=region_country_mapping, populations=populations - ) - else: - raise ValueError("Input resolution is not recognised") - df.T.to_csv(path_to_output, index_label=["id"]) From b13103bf664ea25f790323691dbffc9f7c31bd68 Mon Sep 17 00:00:00 2001 From: adrienmellot Date: Fri, 12 Apr 2024 13:00:20 +0200 Subject: [PATCH 14/24] Respect ruff formatting Co-Authored-By: Francesco Sanvito <68587472+FraSanvit@users.noreply.github.com> --- scripts/transport/road_transport_controlled_charging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/transport/road_transport_controlled_charging.py b/scripts/transport/road_transport_controlled_charging.py index b95c018e..d53e54be 100644 --- a/scripts/transport/road_transport_controlled_charging.py +++ b/scripts/transport/road_transport_controlled_charging.py @@ -80,7 +80,7 @@ def extract_national_ev_charging_potentials( .squeeze() .droplevel(["vehicle_subtype", "section"]) ) - assert df_ev_numbers.isnull().values.any() == False + assert not df_ev_numbers.isnull().values.any() # Compute max. distance travelled per full battery for one EV [in Mio km / vehicle] battery_size = pd.DataFrame.from_dict( { From aedecc83b74eefea447bf9fd1a4167a1ce3e8fa6 Mon Sep 17 00:00:00 2001 From: adrienmellot Date: Tue, 14 May 2024 13:58:45 +0200 Subject: [PATCH 15/24] Rename monthly demand parameters Co-Authored-By: Francesco Sanvito <68587472+FraSanvit@users.noreply.github.com> --- config/default.yaml | 2 +- config/schema.yaml | 4 ++-- rules/transport.smk | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/config/default.yaml b/config/default.yaml index 6a6aacb9..9b246adb 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -170,7 +170,7 @@ parameters: coaches-and-buses: 0.2 # own assumption based on HDVs from [EUCAR_2019] passenger-cars: 0.08 # average from [EUCAR_2019] uncontrolled-ev-charging-share: 0.5 - monthly_demand_ranges: + monthly-demand-bound-fraction: min: 0.9 max: 1.1 equals: 1 diff --git a/config/schema.yaml b/config/schema.yaml index 463ff18d..9c6c1f42 100644 --- a/config/schema.yaml +++ b/config/schema.yaml @@ -374,9 +374,9 @@ properties: uncontrolled-ev-charging-share: type: number description: Share of uncontrolled charging. - monthly_demand_ranges: + monthly-demand-bound-fraction: type: object - description: Charging flexibility parameters. + description: Charging flexibility parameter constraining the amount of monthly demand to be met. additionalProperties: false properties: min: diff --git a/rules/transport.smk b/rules/transport.smk index dbac4df7..6f9cd7eb 100644 --- a/rules/transport.smk +++ b/rules/transport.smk @@ -77,7 +77,7 @@ rule create_controlled_ev_charging_parameters: ev_profiles = lambda wildcards: "data/automatic/ramp-ev-consumption-profiles.csv.gz" if "demand" in wildcards.dataset_name else f"data/automatic/ramp-ev-{wildcards.dataset_name}.csv.gz", populations = "build/data/regional/population.csv", params: - demand_range = config["parameters"]["transport"]["monthly_demand_ranges"], + demand_range = config["parameters"]["transport"]["monthly-demand-bound-fraction"], first_year = config["scope"]["temporal"]["first-year"], final_year = config["scope"]["temporal"]["final-year"], country_neighbour_dict = config["data-pre-processing"]["fill-missing-values"]["ramp"], From f398760ec5b2fe607c89ec1a69d02afd277b7223 Mon Sep 17 00:00:00 2001 From: adrienmellot Date: Tue, 14 May 2024 14:28:40 +0200 Subject: [PATCH 16/24] Add dataset wildcards Co-Authored-By: Francesco Sanvito <68587472+FraSanvit@users.noreply.github.com> --- rules/transport.smk | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rules/transport.smk b/rules/transport.smk index 6f9cd7eb..955d2c82 100644 --- a/rules/transport.smk +++ b/rules/transport.smk @@ -71,7 +71,7 @@ rule create_controlled_road_transport_annual_demand_and_installed_capacities: script: "../scripts/transport/road_transport_controlled_charging.py" rule create_controlled_ev_charging_parameters: - message: "Create timeseries parameters for controlled EV charging" + message: "Create timeseries parameters {wildcards.dataset_name} for controlled EV charging at {wildcards.resolution} resolution" input: locations = "build/data/regional/units.csv", ev_profiles = lambda wildcards: "data/automatic/ramp-ev-consumption-profiles.csv.gz" if "demand" in wildcards.dataset_name else f"data/automatic/ramp-ev-{wildcards.dataset_name}.csv.gz", @@ -82,6 +82,8 @@ rule create_controlled_ev_charging_parameters: final_year = config["scope"]["temporal"]["final-year"], country_neighbour_dict = config["data-pre-processing"]["fill-missing-values"]["ramp"], countries = config["scope"]["spatial"]["countries"], + wildcard_constraints: + dataset_name = "demand-shape-equals|demand-shape-max|demand-shape-min|plugin-profiles" conda: "../envs/default.yaml" output: "build/models/{resolution}/timeseries/demand/{dataset_name}-ev.csv" script: "../scripts/transport/road_transport_controlled_constraints.py" From d2e3dd0bc4a359615d2b5d6164c2401f1ffb28e4 Mon Sep 17 00:00:00 2001 From: adrienmellot Date: Tue, 14 May 2024 14:58:23 +0200 Subject: [PATCH 17/24] Update road_transport_controlled_charging.py Co-Authored-By: Francesco Sanvito <68587472+FraSanvit@users.noreply.github.com> --- .../transport/road_transport_controlled_charging.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/scripts/transport/road_transport_controlled_charging.py b/scripts/transport/road_transport_controlled_charging.py index d53e54be..d632687a 100644 --- a/scripts/transport/road_transport_controlled_charging.py +++ b/scripts/transport/road_transport_controlled_charging.py @@ -102,12 +102,15 @@ def extract_national_ev_charging_potentials( .unstack("year") .mul(transport_scaling_factor) ) - df_ev_chargeable_distance = df_ev_chargeable_distance.assign(**{ - str(year): df_ev_chargeable_distance[2015] for year in range(2016, 2019) - }) + + if final_year > 2015: + # ASSUME 2015 data is used for all years after 2015 + df_ev_chargeable_distance = df_ev_chargeable_distance.assign(**{ + str(year): df_ev_chargeable_distance[2015] + for year in range(2016, final_year + 1) + }) df_ev_chargeable_distance.columns = df_ev_chargeable_distance.columns.astype(int) - # df_ev_chargeable_distance.index.name = None return df_ev_chargeable_distance[range(first_year, final_year + 1)].T From 014085f39b967887e191be335e2cefb833848eb3 Mon Sep 17 00:00:00 2001 From: adrienmellot Date: Tue, 14 May 2024 15:07:07 +0200 Subject: [PATCH 18/24] Small fixes post merge Co-Authored-By: Francesco Sanvito <68587472+FraSanvit@users.noreply.github.com> --- config/default.yaml | 1 - rules/transport.smk | 2 -- 2 files changed, 3 deletions(-) diff --git a/config/default.yaml b/config/default.yaml index cf0c862a..ed1f396a 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -20,7 +20,6 @@ data-sources: swiss-energy-balance: https://www.bfe.admin.ch/bfe/en/home/versorgung/statistik-und-geodaten/energiestatistiken/gesamtenergiestatistik.exturl.html/aHR0cHM6Ly9wdWJkYi5iZmUuYWRtaW4uY2gvZGUvcHVibGljYX/Rpb24vZG93bmxvYWQvNzUxOQ==.html swiss-industry-energy-balance: https://www.bfe.admin.ch/bfe/en/home/versorgung/statistik-und-geodaten/energiestatistiken/teilstatistiken.exturl.html/aHR0cHM6Ly9wdWJkYi5iZmUuYWRtaW4uY2gvZGUvcHVibGljYX/Rpb24vZG93bmxvYWQvODc4OA==.html controlled-ev-data: https://zenodo.org/record/6579421/files/ramp-ev-{dataset}.csv.gz?download=1 - ev-data: https://zenodo.org/record/6579421/files/ramp-ev-consumption-profiles.csv.gz?download=1 gridded-temperature-data: https://zenodo.org/records/6557643/files/temperature.nc?download=1 gridded-10m-windspeed-data: https://zenodo.org/records/6557643/files/wind10m.nc?download=1 when2heat-params: https://zenodo.org/records/10965295/files/{dataset}?download=1 diff --git a/rules/transport.smk b/rules/transport.smk index 4e10b31f..5545157e 100644 --- a/rules/transport.smk +++ b/rules/transport.smk @@ -38,8 +38,6 @@ rule create_controlled_road_transport_annual_demand_and_installed_capacities: annual_controlled_demand = "build/data/transport/annual-road-transport-distance-demand-controlled.csv", ev_vehicle_number = "build/data/jrc-idees/transport/processed-road-vehicles.csv", jrc_road_distance = "build/data/jrc-idees/transport/processed-road-distance.csv", - locations = "build/data/regional/units.csv", - populations = "build/data/regional/population.csv", locations = "build/data/{resolution}/units.csv", populations = "build/data/{resolution}/population.csv", params: From 0832b0e7009c73500220db5367faede3709903a5 Mon Sep 17 00:00:00 2001 From: adrienmellot Date: Tue, 14 May 2024 16:33:29 +0200 Subject: [PATCH 19/24] Fix compatibility with ehighways Co-Authored-By: Francesco Sanvito <68587472+FraSanvit@users.noreply.github.com> --- rules/transport.smk | 4 ++-- scripts/transport/road_transport_controlled_constraints.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/rules/transport.smk b/rules/transport.smk index 5545157e..8733067d 100644 --- a/rules/transport.smk +++ b/rules/transport.smk @@ -57,9 +57,9 @@ rule create_controlled_road_transport_annual_demand_and_installed_capacities: rule create_controlled_ev_charging_parameters: message: "Create timeseries parameters {wildcards.dataset_name} for controlled EV charging at {wildcards.resolution} resolution" input: - locations = "build/data/regional/units.csv", ev_profiles = lambda wildcards: "data/automatic/ramp-ev-consumption-profiles.csv.gz" if "demand" in wildcards.dataset_name else f"data/automatic/ramp-ev-{wildcards.dataset_name}.csv.gz", - populations = "build/data/regional/population.csv", + locations = "build/data/{resolution}/units.csv", + populations = "build/data/{resolution}/population.csv", params: demand_range = config["parameters"]["transport"]["monthly-demand-bound-fraction"], first_year = config["scope"]["temporal"]["first-year"], diff --git a/scripts/transport/road_transport_controlled_constraints.py b/scripts/transport/road_transport_controlled_constraints.py index b4c4b15d..535bafb5 100644 --- a/scripts/transport/road_transport_controlled_constraints.py +++ b/scripts/transport/road_transport_controlled_constraints.py @@ -9,7 +9,7 @@ def scale_to_resolution_and_create_file( df = df elif resolution == "continental": df = df.sum(axis=1).to_frame("EUR") - elif resolution == "regional": + elif resolution in ["regional", "ehighways"]: df = scale_national_to_regional(df, region_country_mapping, populations) else: raise ValueError(f"Resolution {resolution} is not supported") From 5d5b4eb2b54c9381c569a2230e302e9014173a00 Mon Sep 17 00:00:00 2001 From: adrienmellot Date: Tue, 14 May 2024 16:39:26 +0200 Subject: [PATCH 20/24] style: add return types of functions Co-Authored-By: Francesco Sanvito <68587472+FraSanvit@users.noreply.github.com> --- scripts/transport/road_transport_controlled_charging.py | 4 ++-- scripts/transport/road_transport_controlled_constraints.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/transport/road_transport_controlled_charging.py b/scripts/transport/road_transport_controlled_charging.py index 55e009dc..e91a847a 100644 --- a/scripts/transport/road_transport_controlled_charging.py +++ b/scripts/transport/road_transport_controlled_charging.py @@ -45,7 +45,7 @@ def convert_annual_distance_to_electricity_demand( final_year: int, conversion_factors: dict[str, float], country_codes: list[str], -): +) -> pd.DataFrame: """ Convert annual distance driven demand to electricity demand for controlled charging accounting for conversion factors. @@ -73,7 +73,7 @@ def extract_national_ev_charging_potentials( conversion_factors: dict[str, float], battery_sizes: dict[str, float], country_codes: list[str], -): +) -> pd.DataFrame: # Extract number of EVs per vehicle type df_ev_numbers = ( pd.read_csv(path_to_ev_numbers, index_col=[0, 1, 2, 3, 4]) diff --git a/scripts/transport/road_transport_controlled_constraints.py b/scripts/transport/road_transport_controlled_constraints.py index 535bafb5..17bf3b9e 100644 --- a/scripts/transport/road_transport_controlled_constraints.py +++ b/scripts/transport/road_transport_controlled_constraints.py @@ -47,7 +47,7 @@ def get_national_ev_profiles( final_year: int, country_neighbour_dict: dict[str, list[str]], country_codes: list[str], -): +) -> pd.DataFrame: df_timeseries = ( pd.read_csv(ev_profiles_path, index_col=[0, 1, 2], parse_dates=[0]) .xs(slice(first_year, final_year), level="year") From 0b05d53c347bb8b2532fa9e27f91dd6ed98a822c Mon Sep 17 00:00:00 2001 From: adrienmellot Date: Tue, 14 May 2024 16:42:15 +0200 Subject: [PATCH 21/24] rename 'data' to 'profiles' for RAMP Co-Authored-By: Francesco Sanvito <68587472+FraSanvit@users.noreply.github.com> --- config/default.yaml | 2 +- config/schema.yaml | 2 +- rules/transport.smk | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/config/default.yaml b/config/default.yaml index ed1f396a..e909539f 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -19,7 +19,7 @@ data-sources: swiss-end-use: https://www.bfe.admin.ch/bfe/en/home/versorgung/statistik-und-geodaten/energiestatistiken/energieverbrauch-nach-verwendungszweck.exturl.html/aHR0cHM6Ly9wdWJkYi5iZmUuYWRtaW4uY2gvZGUvcHVibGljYX/Rpb24vZG93bmxvYWQvOTg1NA==.html swiss-energy-balance: https://www.bfe.admin.ch/bfe/en/home/versorgung/statistik-und-geodaten/energiestatistiken/gesamtenergiestatistik.exturl.html/aHR0cHM6Ly9wdWJkYi5iZmUuYWRtaW4uY2gvZGUvcHVibGljYX/Rpb24vZG93bmxvYWQvNzUxOQ==.html swiss-industry-energy-balance: https://www.bfe.admin.ch/bfe/en/home/versorgung/statistik-und-geodaten/energiestatistiken/teilstatistiken.exturl.html/aHR0cHM6Ly9wdWJkYi5iZmUuYWRtaW4uY2gvZGUvcHVibGljYX/Rpb24vZG93bmxvYWQvODc4OA==.html - controlled-ev-data: https://zenodo.org/record/6579421/files/ramp-ev-{dataset}.csv.gz?download=1 + controlled-ev-profiles: https://zenodo.org/record/6579421/files/ramp-ev-{dataset}.csv.gz?download=1 gridded-temperature-data: https://zenodo.org/records/6557643/files/temperature.nc?download=1 gridded-10m-windspeed-data: https://zenodo.org/records/6557643/files/wind10m.nc?download=1 when2heat-params: https://zenodo.org/records/10965295/files/{dataset}?download=1 diff --git a/config/schema.yaml b/config/schema.yaml index d5295449..730f2b1b 100644 --- a/config/schema.yaml +++ b/config/schema.yaml @@ -103,7 +103,7 @@ properties: type: string pattern: ^(https?|http?):\/\/.+ description: Web address of Swiss industry energy balance data. - controlled-ev-data: + controlled-ev-profiles: type: string pattern: ^(https?|http?):\/\/.+ description: Web address of electric vehicle data for controlled charging. diff --git a/rules/transport.smk b/rules/transport.smk index 8733067d..32740a11 100644 --- a/rules/transport.smk +++ b/rules/transport.smk @@ -5,7 +5,7 @@ rule download_transport_timeseries: # TODO have correct timeseries data once RAMP has generated the new charging profile and it's been put on Zenodo message: "Get EV data from RAMP" params: - url = config["data-sources"]["controlled-ev-data"] + url = config["data-sources"]["controlled-ev-profiles"] conda: "../envs/shell.yaml" output: protected("data/automatic/ramp-ev-{dataset}.csv.gz") From 92f0c7d7aafd33da9e35c04301e5fa7c6e4022c9 Mon Sep 17 00:00:00 2001 From: adrienmellot Date: Thu, 30 May 2024 16:26:52 +0200 Subject: [PATCH 22/24] delete run.py following #385 --- config/run.py | 215 -------------------------------------------------- 1 file changed, 215 deletions(-) delete mode 100644 config/run.py diff --git a/config/run.py b/config/run.py deleted file mode 100644 index fb15f151..00000000 --- a/config/run.py +++ /dev/null @@ -1,215 +0,0 @@ -import sys - -import calliope -import numpy as np -import pyomo.core as po -from calliope.backend.pyomo.util import ( - get_param, - invalid, - split_comma_list, -) - -""" -This file allows to run a Calliope model with custom constraints - -To run it, use the following command: - python run.py path_to_model scenario_string path_to_output - - where: - - path_to_model is the path to the Calliope model to run, - - scenario_string is the scenario to run, - - and path_to_output is the path to the output .nc file. - -Make sure to have a Calliope environment which can run the model, e.g. the test environment. -""" - -args = sys.argv[1:] -path_to_model = args[0] -scenario_string = args[1] -path_to_output = args[2] - - -def build_model(path_to_model, scenario, path_to_temp_output): - # Build model without custom constraints. Does not run the model - calliope.set_log_verbosity( - "info", include_solver_output=True, capture_warnings=True - ) - model = calliope.Model(path_to_model, scenario=scenario) - - model._model_data.attrs["scenario"] = scenario - - model.to_netcdf(path_to_temp_output) - - -def run_model(path_to_temp_model, path_to_output): - # Run model with custom constraints - calliope.set_log_verbosity( - "info", include_solver_output=True, capture_warnings=True - ) - model = calliope.read_netcdf(path_to_temp_model) - model.run(build_only=True) - add_eurocalliope_constraints(model) - new_model = model.backend.rerun() - new_model.to_netcdf(path_to_output) - - -def add_eurocalliope_constraints(model): - backend_model = model._backend_model - if "energy_cap_max_time_varying" in model._model_data.data_vars: - print("Building production_max_time_varying constraint") - add_production_max_time_varying_constraint(model, backend_model) - if any( - var.startswith("demand_shape_per_month") for var in model._model_data.data_vars - ): - print("Building demand_shape_per_month constraint") - add_carrier_prod_per_month_constraints(model, backend_model) - - -def equalizer(lhs, rhs, sign): - if sign == "max": - return lhs <= rhs - elif sign == "min": - return lhs >= rhs - elif sign == "equals": - return lhs == rhs - else: - raise ValueError(f"Invalid sign: {sign}") - - -def add_production_max_time_varying_constraint(model, backend_model): - def _carrier_production_max_time_varying_constraint_rule( - backend_model, loc_tech, timestep - ): - """ - Set maximum carrier production for technologies with time varying maximum capacity - """ - energy_cap_max = backend_model.energy_cap_max_time_varying[loc_tech, timestep] - if invalid(energy_cap_max): - return po.Constraint.Skip - model_data_dict = backend_model.__calliope_model_data["data"] - timestep_resolution = backend_model.timestep_resolution[timestep] - loc_tech_carriers_out = split_comma_list( - model_data_dict["lookup_loc_techs_conversion"]["out", loc_tech] - ) - - carrier_prod = sum( - backend_model.carrier_prod[loc_tech_carrier, timestep] - for loc_tech_carrier in loc_tech_carriers_out - ) - return carrier_prod <= ( - backend_model.energy_cap[loc_tech] * timestep_resolution * energy_cap_max - ) - - backend_model.loc_tech_carrier_production_max_time_varying_constraint = po.Set( - initialize=[ - loc_tech - for loc_tech in backend_model.loc_techs_conversion - if model.inputs.energy_cap_max_time_varying.loc[{"loc_techs": loc_tech}] - .notnull() - .all() - ], - ordered=True, - ) - model.backend.add_constraint( - "carrier_production_max_time_varying_constraint", - ["loc_tech_carrier_production_max_time_varying_constraint", "timesteps"], - _carrier_production_max_time_varying_constraint_rule, - ) - - -def add_carrier_prod_per_month_constraints(model, backend_model): - def _carrier_prod_per_month_constraint_rule_generator(sense): - def __carrier_prod_per_month_constraint_rule(backend_model, loc_tech, month): - """ - Set the min/max amount of carrier consumption (relative to annual consumption) - for a specific loc tech that must take place in a given calender month in the model - """ - model_data_dict = backend_model.__calliope_model_data - loc_tech_carrier = model_data_dict["data"]["lookup_loc_techs_conversion"][ - ("out", loc_tech) - ] - - prod = backend_model.carrier_prod - prod_total = sum( - prod[loc_tech_carrier, timestep] for timestep in backend_model.timesteps - ) - prod_month = sum( - prod[loc_tech_carrier, timestep] - for timestep in backend_model.timesteps - if backend_model.month_numbers[timestep].value == month - ) - if "timesteps" in [ - i.name - for i in getattr( - backend_model, f"carrier_prod_per_month_{sense}_time_varying" - )._index.subsets() - ]: - prod_fraction = sum( - get_param( - backend_model, - f"carrier_prod_per_month_{sense}_time_varying", - (loc_tech, timestep), - ) - * backend_model.timestep_resolution[timestep] - for timestep in backend_model.timesteps - if backend_model.month_numbers[timestep].value == month - ) - else: - prod_fraction = get_param( - backend_model, f"carrier_prod_per_month_{sense}", (loc_tech) - ) - - return equalizer(prod_month, prod_total * prod_fraction, sense) - - return __carrier_prod_per_month_constraint_rule - - backend_model.months = po.Set( - initialize=np.unique(model._model_data.timesteps.dt.month.values), ordered=True - ) - month_numbers = model._model_data.timesteps.dt.month.to_series() - month_numbers.index = month_numbers.index.strftime("%Y-%m-%d %H:%M") - - backend_model.month_numbers = po.Param( - backend_model.timesteps, - initialize=month_numbers.to_dict(), - mutable=True, - within=po.Reals, - ) - backend_model.__calliope_datetime_data.add(("data_vars", "month_numbers")) - - for sense in ["min", "max", "equals"]: - if ( - f"carrier_prod_per_month_{sense}_time_varying" - in model._model_data.data_vars - ): - setattr( - backend_model, - f"loc_techs_carrier_prod_per_month_{sense}", - po.Set( - initialize=[ - loc_tech - for loc_tech in backend_model.loc_techs - if ( - model._model_data[ - f"carrier_prod_per_month_{sense}_time_varying" - ] - .loc[{"loc_techs": loc_tech}] - .notnull() - .all() - ) - ], - ordered=True, - ), - ) - model.backend.add_constraint( - f"carrier_prod_per_month_{sense}_constraint", - [f"loc_techs_carrier_prod_per_month_{sense}", "months"], - _carrier_prod_per_month_constraint_rule_generator(sense), - ) - - -build_model(path_to_model, scenario_string, path_to_output) -run_model(path_to_output, path_to_output) - -model = calliope.read_netcdf(path_to_output) -model.plot.timeseries(subset={"locs": ["IRL"]}) From 905e331af543abf7f0fbab26ac4c289dd7b25af9 Mon Sep 17 00:00:00 2001 From: adrienmellot Date: Fri, 31 May 2024 15:07:45 +0200 Subject: [PATCH 23/24] uncontrolled charging share to 1 by default --- config/default.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/default.yaml b/config/default.yaml index 2959c1c2..e1483159 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -174,7 +174,7 @@ parameters: motorcycles: 0.01 # own assumption coaches-and-buses: 0.2 # own assumption based on HDVs from [EUCAR_2019] passenger-cars: 0.08 # average from [EUCAR_2019] - uncontrolled-ev-charging-share: 0.5 + uncontrolled-ev-charging-share: 1 monthly-demand-bound-fraction: min: 0.9 max: 1.1 From 1a0db451aa169305a5eac96877d4c6eadd496c87 Mon Sep 17 00:00:00 2001 From: adrienmellot Date: Fri, 31 May 2024 17:54:42 +0200 Subject: [PATCH 24/24] Edits based on review --- CHANGELOG.md | 2 +- config/default.yaml | 2 +- config/schema.yaml | 2 +- .../transport/road_transport_controlled_constraints.py | 8 ++++---- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 83cd8bd4..92e98129 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ * **ADD** fully-electrified heat demand (#284). -* **ADD** fully-electrified road transportation (#270, #271, #358). A parameter allows to define the share of uncontrolled (timeseries) vs controlled charging (optimised) by the solver (#338). +* **ADD** fully-electrified road transportation (#270, #271, #358). A parameter allows to define the share of uncontrolled (timeseries) vs controlled charging (optimised) by the solver (#338). Data for controlled charging constraints is readily available (#356), but corresponding constraints are not yet implemented (#385). * **ADD** nuclear power plant technology with capacity limits. Capacity limits can be equal to today or be bound by a minimum and maximum capacity to represent an available range in future. In either case, capacities are allocated at a subnational resolution based on linear scaling from current capacity geolocations, using the JRC power plant database (#78). diff --git a/config/default.yaml b/config/default.yaml index e1483159..d4450bf4 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -168,7 +168,7 @@ parameters: coaches-and-buses: Motor coaches, buses and trolley buses passenger-cars: Passenger cars motorcycles: Powered 2-wheelers - ev-battery-sizes: # MWh + ev-battery-sizes: heavy-duty-vehicles: 0.2 # average from [EUCAR_2019] light-duty-vehicles: 0.1 # own assumption based on passenger cars from [EUCAR_2019] motorcycles: 0.01 # own assumption diff --git a/config/schema.yaml b/config/schema.yaml index ed1a2b8f..443045a1 100644 --- a/config/schema.yaml +++ b/config/schema.yaml @@ -373,7 +373,7 @@ properties: description: JRC-IDEES name of motorcycles. ev-battery-sizes: type: object - description: EV battery size per vehicle type. + description: EV battery size per vehicle type in MWh. additionalProperties: false properties: light-duty-vehicles: diff --git a/scripts/transport/road_transport_controlled_constraints.py b/scripts/transport/road_transport_controlled_constraints.py index 17bf3b9e..5761eff1 100644 --- a/scripts/transport/road_transport_controlled_constraints.py +++ b/scripts/transport/road_transport_controlled_constraints.py @@ -1,5 +1,5 @@ import pandas as pd -import pycountry +from eurocalliopelib import utils def scale_to_resolution_and_create_file( @@ -89,9 +89,9 @@ def fill_empty_country(df, country_neighbour_dict): first_year=snakemake.params.first_year, final_year=snakemake.params.final_year, country_neighbour_dict=snakemake.params.country_neighbour_dict, - country_codes=([ - pycountry.countries.lookup(c).alpha_3 for c in snakemake.params.countries - ]), + country_codes=utils.convert_valid_countries( + snakemake.params.countries + ).values(), ) scale_to_resolution_and_create_file(