diff --git a/explorer.py b/explorer.py index 9d7885a..993c2fb 100644 --- a/explorer.py +++ b/explorer.py @@ -22,6 +22,7 @@ "#efefef", ] + # =========================================================================== # Functions to be cached # =========================================================================== @@ -354,17 +355,6 @@ def gen_rank_plt(input_col, options, **kwargs): return fig -chart_config = { - "displaylogo": False, - "modeBarButtonsToRemove": ["hoverClosestPie"], - "toImageButtonOptions": { - "width": 700, - "height": 450, - "scale": (210 / 25.4) / (700 / 300), - }, -} - - def print_total(number): st.write(f"**{number}** respondents answered the question with the current filter") @@ -377,6 +367,17 @@ def answered_by(group): st.caption(f"This question was only answered by {text}") +chart_config = { + "displaylogo": False, + "modeBarButtonsToRemove": ["hoverClosestPie"], + "toImageButtonOptions": { + "width": 700, + "height": 450, + "scale": (210 / 25.4) / (700 / 300), + }, +} + + # =========================================================================== # Import data from stored pickle # =========================================================================== @@ -405,6 +406,9 @@ def callback(): "Resources // FOI", "Resources // Appreciation", "Media Reporting", + "Public Campaigning", + "Policy Advocacy", + "Strategic Litigation", "Protection", "Constraints", "Attitudes", @@ -652,6 +656,19 @@ def callback(): config=chart_config, ) + st.write("### Predominant activity of CSO professionals `[CSpreselection]`") + CSpreselection_counts = df[filter]["CSpreselection"].value_counts() + print_total(CSpreselection_counts.sum()) + st.plotly_chart( + gen_px_pie( + df[filter], + values=CSpreselection_counts, + names=CSpreselection_counts.index, + ), + use_container_width=True, + config=chart_config, + ) + st.write("### Gender `[gender]`") gender_counts = df[filter]["gender"].value_counts() print_total(gender_counts.sum()) @@ -675,10 +692,9 @@ def callback(): # TODO Privacy notice # =========================================================================== -# Resources +# Resources // HR # =========================================================================== - if selected_section == "Resources // HR": st.write("# Resources // HR") @@ -703,6 +719,7 @@ def callback(): config=chart_config, ) + # ======================================================================= st.write( "### How many days per month do you work on surveillance by intelligence agencies? `[hr2]`" ) @@ -760,10 +777,11 @@ def callback(): config=chart_config, ) - st.write("### Which type of medium do you work for? `[hr3ms]`") - hr3ms_df = pd.DataFrame(columns=("option", "count", "country")) + # ======================================================================= + st.write("### Which type of medium do you work for? `[MShr3]`") + MShr3_df = pd.DataFrame(columns=("option", "count", "country")) answered_by("media") - hr3ms_options = [ + MShr3_options = [ "daily_newspaper", "weekly_newspaper", "magazine", @@ -773,7 +791,7 @@ def callback(): "online_stand_alone", "online_of_offline", ] - hr3ms_options_clean = [ + MShr3_options_clean = [ "Daily newspaper", "Weekly newspaper", "Magazine", @@ -783,29 +801,29 @@ def callback(): "Online outlet
(standalone)", "Online outlet
(of an offline publication)", ] - for option, option_clean in zip(hr3ms_options, hr3ms_options_clean): - hr3ms_data = df[filter]["country"][df[f"hr3ms[{option}]"] == 1].tolist() - for i in hr3ms_data: - hr3ms_df = hr3ms_df.append( - {"option": option_clean, "count": hr3ms_data.count(i), "country": i}, + for option, option_clean in zip(MShr3_options, MShr3_options_clean): + MShr3_data = df[filter]["country"][df[f"MShr3[{option}]"] == 1].tolist() + for i in MShr3_data: + MShr3_df = MShr3_df.append( + {"option": option_clean, "count": MShr3_data.count(i), "country": i}, ignore_index=True, ) - hr3ms_df = hr3ms_df.drop_duplicates() + MShr3_df = MShr3_df.drop_duplicates() if filters["field"] == "CSO Professionals": print_total(0) else: # If one respondent chose at least one medium it counts towards the total - hr3ms_col_list = [col for col in df[filter].columns if col.startswith("hr3ms")] - hr3ms_df_total = df[filter][hr3ms_col_list] - hr3ms_df_total["answered"] = [ - "Y" if x > 0 else "N" for x in np.sum(hr3ms_df_total.values == True, 1) + MShr3_col_list = [col for col in df[filter].columns if col.startswith("MShr3")] + MShr3_df_total = df[filter][MShr3_col_list] + MShr3_df_total["answered"] = [ + "Y" if x > 0 else "N" for x in np.sum(MShr3_df_total.values == True, 1) ] - print_total(hr3ms_df_total["answered"].value_counts().sort_index()[1]) + print_total(MShr3_df_total["answered"].value_counts().sort_index()[1]) st.plotly_chart( gen_px_histogram( - hr3ms_df, + MShr3_df, x="option", y="count", nbins=None, @@ -816,25 +834,32 @@ def callback(): "France": colors[4], }, labels={"count": "people who work
for this medium"}, + marginal=None, ), use_container_width=True, config=chart_config, ) + # ======================================================================= st.write( - "### Within the past year, did you have enough time to cover surveillance by intelligence agencies? `[hr4ms]`" + "### Within the past year, did you have enough time to cover surveillance by intelligence agencies? `[MShr4]`" ) answered_by("media") - hr4ms_counts = df[filter]["hr4ms"].value_counts().sort_index() + MShr4_counts = df[filter]["MShr4"].value_counts().sort_index() + print_total(MShr4_counts.sum()) st.plotly_chart( gen_go_pie( - labels=hr4ms_counts.sort_index().index, - values=hr4ms_counts.sort_index().values, + labels=MShr4_counts.sort_index().index, + values=MShr4_counts.sort_index().values, ), use_container_width=True, config=chart_config, ) +# =========================================================================== +# Resources // Expertise +# =========================================================================== + if selected_section == "Resources // Expertise": st.write("# Resources // Expertise") @@ -860,7 +885,6 @@ def callback(): use_container_width=True, config=chart_config, ) - st.plotly_chart( gen_px_box( df=df[filter], @@ -879,6 +903,7 @@ def callback(): config=chart_config, ) + # ======================================================================= st.write( "### How do you assess your level of expertise concerning the **legal** aspects of surveillance by intelligence agencies? `[expertise2]`" ) @@ -893,6 +918,7 @@ def callback(): config=chart_config, ) + # ======================================================================= st.write( "### How do you assess your level of expertise concerning the **political** aspects of surveillance by intelligence agencies `[expertise3]`?" ) @@ -907,6 +933,7 @@ def callback(): config=chart_config, ) + # ======================================================================= st.write( "### How do you assess your level of expertise concerning the **technical** aspects of surveillance by intelligence agencies? `[expertise4]`" ) @@ -921,6 +948,10 @@ def callback(): config=chart_config, ) +# =========================================================================== +# Resources // Finance +# =========================================================================== + if selected_section == "Resources // Finance": st.write("# Resources // Finance") @@ -938,18 +969,19 @@ def callback(): config=chart_config, ) + # ======================================================================= st.write( - "### If you wanted to conduct investigative research into surveillance by intelligence agencies, could you access extra funding for this research? (For example, a special budget or a stipend) `[finance2ms]`" + "### If you wanted to conduct investigative research into surveillance by intelligence agencies, could you access extra funding for this research? (For example, a special budget or a stipend) `[MSfinance2]`" ) - finance2ms_counts = df[filter]["finance2ms"].value_counts() + MSfinance2_counts = df[filter]["MSfinance2"].value_counts() answered_by("media") - print_total(finance2ms_counts.sum()) + print_total(MSfinance2_counts.sum()) st.plotly_chart( gen_px_pie( - finance2ms_counts, - values=finance2ms_counts, - names=finance2ms_counts.index, - color=finance2ms_counts.index, + MSfinance2_counts, + values=MSfinance2_counts, + names=MSfinance2_counts.index, + color=MSfinance2_counts.index, color_discrete_map={ "No": colors[0], "Yes": colors[2], @@ -961,10 +993,11 @@ def callback(): config=chart_config, ) + # ======================================================================= st.write( - "### How important are the following funding categories for your organisation's work on intelligence-related issues? `[finance2cs]`" + "### How important are the following funding categories for your organisation's work on intelligence-related issues? `[CSfinance2]`" ) - finance2cs_options = [ + CSfinance2_options = [ "private_foundations", "donations", "national_public_funds", @@ -972,7 +1005,7 @@ def callback(): "international_public_funds", "other", ] - finance2cs_options_clean = [ + CSfinance2_options_clean = [ "Private foundations", "Donations", "National public funds", @@ -980,12 +1013,12 @@ def callback(): "International public funds", "Other", ] - finance2cs_very_important = [] - finance2cs_somewhat_important = [] - finance2cs_important = [] - finance2cs_slightly_important = [] - finance2cs_not_important = [] - finance2cs_prefer_not_to_say = [] + CSfinance2_very_important = [] + CSfinance2_somewhat_important = [] + CSfinance2_important = [] + CSfinance2_slightly_important = [] + CSfinance2_not_important = [] + CSfinance2_prefer_not_to_say = [] for importance in [ "Very important", "Somewhat important", @@ -994,28 +1027,28 @@ def callback(): "Not important at all", "I prefer not to say", ]: - for option in finance2cs_options: + for option in CSfinance2_options: try: - count = df[filter][f"finance2cs[{option}]"].value_counts()[importance] + count = df[filter][f"CSfinance2[{option}]"].value_counts()[importance] except KeyError: count = 0 if importance == "Very important": - finance2cs_very_important.append(count) + CSfinance2_very_important.append(count) elif importance == "Somewhat important": - finance2cs_somewhat_important.append(count) + CSfinance2_somewhat_important.append(count) elif importance == "Important": - finance2cs_important.append(count) + CSfinance2_important.append(count) elif importance == "Slightly important": - finance2cs_slightly_important.append(count) + CSfinance2_slightly_important.append(count) elif importance == "Not important at all": - finance2cs_not_important.append(count) + CSfinance2_not_important.append(count) elif importance == "I prefer not to say": - finance2cs_prefer_not_to_say.append(count) + CSfinance2_prefer_not_to_say.append(count) else: continue totals = [ - df[filter][f"finance2cs[{option}]"].value_counts().sum() - for option in finance2cs_options + df[filter][f"CSfinance2[{option}]"].value_counts().sum() + for option in CSfinance2_options ] answered_by("cso") print_total(max(totals)) @@ -1024,38 +1057,38 @@ def callback(): data=[ go.Bar( name="Very important", - x=finance2cs_options_clean, - y=finance2cs_very_important, + x=CSfinance2_options_clean, + y=CSfinance2_very_important, marker_color=colors[0], ), go.Bar( name="Somewhat important", - x=finance2cs_options_clean, - y=finance2cs_somewhat_important, + x=CSfinance2_options_clean, + y=CSfinance2_somewhat_important, marker_color=colors[1], ), go.Bar( name="Important", - x=finance2cs_options_clean, - y=finance2cs_important, + x=CSfinance2_options_clean, + y=CSfinance2_important, marker_color=colors[2], ), go.Bar( name="Slightly important", - x=finance2cs_options_clean, - y=finance2cs_slightly_important, + x=CSfinance2_options_clean, + y=CSfinance2_slightly_important, marker_color=colors[3], ), go.Bar( name="Not important at all", - x=finance2cs_options_clean, - y=finance2cs_not_important, + x=CSfinance2_options_clean, + y=CSfinance2_not_important, marker_color=colors[4], ), go.Bar( name="I prefer not to say", - x=finance2cs_options_clean, - y=finance2cs_prefer_not_to_say, + x=CSfinance2_options_clean, + y=CSfinance2_prefer_not_to_say, marker_color=colors[5], ), ], @@ -1067,7 +1100,7 @@ def callback(): st.write("# Resources // FOI") st.write( - "### Have you requested information under the national FOI† law when you worked on intelligence-related issues over the past 5 years?" + "### Have you requested information under the national FOI† law when you worked on intelligence-related issues over the past 5 years? `[foi1]`" ) st.caption("†Freedom of Information") foi1_counts = df[filter]["foi1"].value_counts() @@ -1088,8 +1121,9 @@ def callback(): use_container_width=True, ) - st.write("### How often did you request information?") - foi2_counts = df[filter]["foi3"].value_counts() + # ======================================================================= + st.write("### How often did you request information? `[foi2]`") + foi2_counts = df[filter]["foi2"].value_counts() print_total(foi2_counts.sum()) st.plotly_chart( gen_px_histogram( @@ -1107,7 +1141,6 @@ def callback(): ), use_container_width=True, ) - st.plotly_chart( gen_px_box( df=df[filter], @@ -1125,8 +1158,9 @@ def callback(): use_container_width=True, ) + # ======================================================================= st.write( - "### Over the past 5 years, did you receive a response to your FOI request(s) in a timely manner?" + "### Over the past 5 years, did you receive a response to your FOI request(s) in a timely manner? `[foi3]`" ) foi3_counts = df[filter]["foi3"].value_counts() print_total(foi3_counts.sum()) @@ -1139,8 +1173,9 @@ def callback(): config=chart_config, ) + # ======================================================================= st.write( - "### How helpful have Freedom of Information requests been for your work on intelligence-related issues?" + "### How helpful have Freedom of Information requests been for your work on intelligence-related issues? `[foi4]`" ) foi4_counts = df[filter]["foi4"].value_counts() print_total(foi4_counts.sum()) @@ -1153,11 +1188,11 @@ def callback(): config=chart_config, ) + # ======================================================================= st.write( - "### Why haven’t you requested information under the national FOI law when you reported on intelligence-related issues over the past 5 years?" + "### Why haven’t you requested information under the national FOI law when you reported on intelligence-related issues over the past 5 years? `[foi5]`" ) foi5_df = pd.DataFrame(columns=("option", "count", "country")) - # TODO Map proper labels for option in [ "not_aware", "not_covered", @@ -1203,6 +1238,7 @@ def callback(): "United Kingdom": colors[5], }, labels={"count": "people who answered 'Yes'"}, + marginal=None, ), use_container_width=True, config=chart_config, @@ -1277,7 +1313,6 @@ def callback(): st.write( """### Please estimate: how many journalistic pieces have you produced on intelligence-related topics in the past year? `[MSsoc1]`""" ) - answered_by("media") MSsoc1_df = df[filter][["country", "MSsoc1"]] MSsoc1_df = MSsoc1_df.dropna(subset=["MSsoc1"]) @@ -1299,7 +1334,6 @@ def callback(): use_container_width=True, config=chart_config, ) - st.plotly_chart( gen_px_box( df=df[filter], @@ -1322,7 +1356,6 @@ def callback(): st.write( "### Please estimate: how many of these pieces focused on surveillance by intelligence agencies? `[MSsoc2]`" ) - answered_by("media") MSsoc2_df = df[filter][["country", "MSsoc2"]] MSsoc2_df = MSsoc2_df.dropna(subset=["MSsoc2"]) @@ -1346,7 +1379,6 @@ def callback(): use_container_width=True, config=chart_config, ) - st.plotly_chart( gen_px_box( df=df[filter], @@ -1415,7 +1447,6 @@ def callback(): st.write( "### How regularly do you report on surveillance by intelligence agencies? `[MSsoc3]`" ) - answered_by("media") MSsoc3_counts = df[filter]["MSsoc3"].value_counts() print_total(MSsoc3_counts.sum()) @@ -1432,8 +1463,6 @@ def callback(): st.write( "### When covering surveillance by intelligence agencies, which topics usually prompt you to write an article? `[MSsoc4]`" ) - - answered_by("media") MSsoc4_df = pd.DataFrame(columns=("option", "count", "country")) MSsoc4_options = [ "follow_up_on_other_media", @@ -1455,6 +1484,19 @@ def callback(): "I prefer not to say", "Other", ] + answered_by("media") + if filters["field"] == "CSO Professionals": + print_total(0) + else: + # If one respondent chose at least one medium it counts towards the total + MSsoc4_col_list = [ + col for col in df[filter].columns if col.startswith("MSsoc4") + ] + MSsoc4_df_total = df[filter][MSsoc4_col_list] + MSsoc4_df_total["answered"] = [ + "Y" if x > 0 else "N" for x in np.sum(MSsoc4_df_total.values == True, 1) + ] + print_total(MSsoc4_df_total["answered"].value_counts().sort_index()[1]) for option, option_clean in zip(MSsoc4_options, MSsoc4_options_clean): MSsoc4_data = df[filter]["country"][df[f"MSsoc4[{option}]"] == 1].tolist() for i in MSsoc4_data: @@ -1485,7 +1527,7 @@ def callback(): st.write( "### When covering surveillance by intelligence agencies, which of the following topics to you report on frequently? `[MSsoc5]`" ) - + answered_by("media") MSsoc5_df = pd.DataFrame(columns=("option", "count", "country")) MSsoc5_options = [ "national_security_risks", @@ -1512,6 +1554,18 @@ def callback(): {"option": option_clean, "count": MSsoc5_data.count(i), "country": i}, ignore_index=True, ) + if filters["field"] == "CSO Professionals": + print_total(0) + else: + # If one respondent chose at least one medium it counts towards the total + MSsoc5_col_list = [ + col for col in df[filter].columns if col.startswith("MSsoc5") + ] + MSsoc5_df_total = df[filter][MSsoc5_col_list] + MSsoc5_df_total["answered"] = [ + "Y" if x > 0 else "N" for x in np.sum(MSsoc5_df_total.values == True, 1) + ] + print_total(MSsoc5_df_total["answered"].value_counts().sort_index()[1]) MSsoc5_df = MSsoc5_df.drop_duplicates() st.plotly_chart( gen_px_histogram( @@ -1587,6 +1641,7 @@ def callback(): }, ), use_container_width=True, + config=chart_config, ) # ======================================================================= @@ -1619,7 +1674,6 @@ def callback(): "I don't know", "I prefer not to say", ] - print(df[filter]["country"][df["MSimpact1[above_avg_comments]"]]) for option, option_clean in zip(options, options_clean): MSimpact1_data = df[filter]["country"][df[f"MSimpact1[{option}]"] == 1].tolist() for i in MSimpact1_data: @@ -1631,6 +1685,19 @@ def callback(): }, ignore_index=True, ) + answered_by("media") + if filters["field"] == "CSO Professionals": + print_total(0) + else: + # If one respondent chose at least one medium it counts towards the total + MSimpact1_col_list = [ + col for col in df[filter].columns if col.startswith("MSimpact1") + ] + MSimpact1_df_total = df[filter][MSimpact1_col_list] + MSimpact1_df_total["answered"] = [ + "Y" if x > 0 else "N" for x in np.sum(MSimpact1_df_total.values == True, 1) + ] + print_total(MSimpact1_df_total["answered"].value_counts().sort_index()[1]) MSimpact1_df = MSimpact1_df.drop_duplicates() st.plotly_chart( gen_px_histogram( @@ -1648,6 +1715,7 @@ def callback(): marginal=None, ), use_container_width=True, + config=chart_config, ) # ======================================================================= @@ -1689,6 +1757,19 @@ def callback(): }, ignore_index=True, ) + answered_by("media") + if filters["field"] == "CSO Professionals": + print_total(0) + else: + # If one respondent chose at least one medium it counts towards the total + MSimpact1_col_list = [ + col for col in df[filter].columns if col.startswith("MSimpact1") + ] + MSimpact1_df_total = df[filter][MSimpact1_col_list] + MSimpact1_df_total["answered"] = [ + "Y" if x > 0 else "N" for x in np.sum(MSimpact1_df_total.values == True, 1) + ] + print_total(MSimpact1_df_total["answered"].value_counts().sort_index()[1]) MSimpact2_df = MSimpact2_df.drop_duplicates() st.plotly_chart( gen_px_histogram( @@ -1707,6 +1788,853 @@ def callback(): font_size=11, ), use_container_width=True, + config=chart_config, + ) + +# =========================================================================== +# Public Campaigning +# =========================================================================== + +if selected_section == "Public Campaigning": + st.caption( + "__NB__: All questions in this section have only been presented to civil society professionals." + ) + + st.write("# Activity") + + st.write( + "### How important are the following campaigning tools for your work concerning intelligence-related issues? `[CScampact2]`" + ) + answered_by("cso") + options = [ + "media_contributions", + "own_publications", + "petitions_open_letters", + "public_events", + "collaborations", + "demonstrations", + "social_media", + "advertising", + "volunteer_activities", + "providing_technical_tools", + "support_for_eu_campaigns", + "other", + ] + options_clean = [ + "Media contributions", + "Own publications", + "Petitions and open letters", + "Public events", + "Collaborations with celebrities and influencers", + "Demonstrations and rallies", + "Social media communications", + "Advertising", + "Volunteer activities", + "Providing technical tools", + "Support for campaign activities in other countries", + "Other", + ] + CScampact2_very_important = [] + CScampact2_somewhat_important = [] + CScampact2_important = [] + CScampact2_slightly_important = [] + CScampact2_not_important = [] + for importance in [ + "Very important", + "Somewhat important", + "Important", + "Slightly important", + "Not important at all", + ]: + for option in options: + try: + count = df[filter][f"CScampact2[{option}]"].value_counts()[importance] + except KeyError: + count = 0 + if importance == "Very important": + CScampact2_very_important.append(count) + elif importance == "Somewhat important": + CScampact2_somewhat_important.append(count) + elif importance == "Important": + CScampact2_important.append(count) + elif importance == "Slightly important": + CScampact2_slightly_important.append(count) + elif importance == "Not important at all": + CScampact2_not_important.append(count) + else: + continue + + if filters["field"] == "Media Professionals": + print_total(0) + else: + # If one respondent chose at least one medium it counts towards the total + CScampact2_col_list = [ + col for col in df[filter].columns if col.startswith("CScampact2") + ] + CScampact2_df_total = df[filter][CScampact2_col_list] + # Make all NaNs numeric zeroes + CScampact2_df_total = CScampact2_df_total.fillna(0) + # Now replace everything that is not a number with "Y" + for col in CScampact2_df_total.columns: + CScampact2_df_total[col] = ( + pd.to_numeric(CScampact2_df_total[col], errors="coerce") + .fillna("Y") + .astype("string") + ) + # Make a column to count "Y" + CScampact2_df_total["answered"] = [ + "Y" if x > 0 else "N" for x in np.sum(CScampact2_df_total.values == "Y", 1) + ] + print_total(CScampact2_df_total["answered"].value_counts().sort_index()[1]) + + st.plotly_chart( + gen_go_bar_stack( + data=[ + go.Bar( + name="Very important", + x=options_clean, + y=CScampact2_very_important, + marker_color=colors[0], + ), + go.Bar( + name="Somewhat important", + x=options_clean, + y=CScampact2_somewhat_important, + marker_color=colors[1], + ), + go.Bar( + name="Important", + x=options_clean, + y=CScampact2_important, + marker_color=colors[2], + ), + go.Bar( + name="Slightly important", + x=options_clean, + y=CScampact2_slightly_important, + marker_color=colors[3], + ), + go.Bar( + name="Not important at all", + x=options_clean, + y=CScampact2_not_important, + marker_color=colors[5], + ), + ], + ), + use_container_width=True, + config=chart_config, + ) + + # ======================================================================= + st.write("# Transnational Scope") + + st.write( + "### How frequently do your public campaigns address transnational issues of surveillance by intelligence agencies? `[CScamptrans1]`" + ) + answered_by("cso") + CScamptrans1_counts = df[filter]["CScamptrans1"].value_counts() + print_total(CScamptrans1_counts.sum()) + st.plotly_chart( + gen_go_pie( + labels=CScamptrans1_counts.sort_index().index, + values=CScamptrans1_counts.sort_index().values, + ), + use_container_width=True, + config=chart_config, + ) + + # ======================================================================= + st.write( + "### When conducting public campaigns on surveillance by intelligence agencies, how often do you collaborate with civil society actors in other countries? `[CScamptrans2]`" + ) + answered_by("cso") + CScamptrans2_counts = df[filter]["CScamptrans2"].value_counts() + print_total(CScamptrans2_counts.sum()) + st.plotly_chart( + gen_go_pie( + labels=CScamptrans2_counts.sort_index().index, + values=CScamptrans2_counts.sort_index().values, + ), + use_container_width=True, + config=chart_config, + ) + + # ======================================================================= + st.write("# Perceived Impact") + + st.write( + "### How much do you agree with the following statements concerning the effectiveness of your campaigning activities regarding intelligence-related issues over the past 5 years? `[CScampimpact1]`" + ) + answered_by("cso") + options = [ + "increased_awareness", + "policies_reflect_demands", + "created_media_attention", + "achieved_goals", + ] + options_clean = [ + "Helped increase public awareness", + "Our demands have been reflected in politics", + "Created media attention", + "Achieved defined goals", + ] + CScampimpact1_agree_completely = [] + CScampimpact1_agree_to_great_extent = [] + CScampimpact1_agree_somewhat = [] + CScampimpact1_agree_slightly = [] + CScampimpact1_not_agree_at_all = [] + for agreement in [ + "Agree completely", + "Agree to a great extent", + "Agree somewhat", + "Agree slightly", + "Not agree at all", + ]: + for option in options: + try: + count = df[filter][f"CScampimpact1[{option}]"].value_counts()[agreement] + except KeyError: + count = 0 + if agreement == "Agree completely": + CScampimpact1_agree_completely.append(count) + elif agreement == "Agree to a great extent": + CScampimpact1_agree_to_great_extent.append(count) + elif agreement == "Agree somewhat": + CScampimpact1_agree_somewhat.append(count) + elif agreement == "Agree slightly": + CScampimpact1_agree_slightly.append(count) + elif agreement == "Not agree at all": + CScampimpact1_not_agree_at_all.append(count) + else: + continue + if filters["field"] == "Media Professionals": + print_total(0) + else: + # If one respondent chose at least one medium it counts towards the total + CScampimpact1_col_list = [ + col for col in df[filter].columns if col.startswith("CScampimpact1") + ] + CScampimpact1_df_total = df[filter][CScampimpact1_col_list] + # Make all NaNs numeric zeroes + CScampimpact1_df_total = CScampimpact1_df_total.fillna(0) + # Now replace everything that is not a number with "Y" + for col in CScampimpact1_df_total.columns: + CScampimpact1_df_total[col] = ( + pd.to_numeric(CScampimpact1_df_total[col], errors="coerce") + .fillna("Y") + .astype("string") + ) + # Make a column to count "Y" + CScampimpact1_df_total["answered"] = [ + "Y" if x > 0 else "N" + for x in np.sum(CScampimpact1_df_total.values == "Y", 1) + ] + print_total(CScampimpact1_df_total["answered"].value_counts().sort_index()[1]) + st.plotly_chart( + gen_go_bar_stack( + data=[ + go.Bar( + name="Agree completely", + x=options_clean, + y=CScampimpact1_agree_completely, + marker_color=colors[0], + ), + go.Bar( + name="Agree to a great extent", + x=options_clean, + y=CScampimpact1_agree_to_great_extent, + marker_color=colors[1], + ), + go.Bar( + name="Agree somewhat", + x=options_clean, + y=CScampimpact1_agree_somewhat, + marker_color=colors[2], + ), + go.Bar( + name="Agree slightly", + x=options_clean, + y=CScampimpact1_agree_slightly, + marker_color=colors[3], + ), + go.Bar( + name="Not agree at all", + x=options_clean, + y=CScampimpact1_not_agree_at_all, + marker_color=colors[5], + ), + ], + ), + use_container_width=True, + config=chart_config, + ) + +# =========================================================================== +# Policy Advocacy +# =========================================================================== + +if selected_section == "Policy Advocacy": + st.caption( + "__NB__: All questions in this section have only been presented to civil society organisation professionals." + ) + st.write("# Activity") + + st.write( + "### How important are the following policy advocacy tools for your work on intelligence-related issues? `[CSadvocact2]`" + ) + answered_by("cso") + options = [ + "research", + "consultations", + "briefings", + "expert_events", + "participation_in_fora", + "legal_opinions", + "informal_encounters", + "other", + ] + options_clean = [ + "Research & analysis", + "Contributing to consultations", + "Briefing of policy makers", + "Expert events", + "Participation in fora or bodies", + "Legal opinions", + "Informal encounters", + "Other", + ] + CSadvocact2_very_important = [] + CSadvocact2_somewhat_important = [] + CSadvocact2_important = [] + CSadvocact2_slightly_important = [] + CSadvocact2_not_important = [] + for importance in [ + "Very important", + "Somewhat important", + "Important", + "Slightly important", + "Not important at all", + ]: + for option in options: + try: + count = df[filter][f"CSadvocact2[{option}]"].value_counts()[importance] + except KeyError: + count = 0 + if importance == "Very important": + CSadvocact2_very_important.append(count) + elif importance == "Somewhat important": + CSadvocact2_somewhat_important.append(count) + elif importance == "Important": + CSadvocact2_important.append(count) + elif importance == "Slightly important": + CSadvocact2_slightly_important.append(count) + elif importance == "Not important at all": + CSadvocact2_not_important.append(count) + else: + continue + try: + # If one respondent chose at least one medium it counts towards the total + CSadvocact2_col_list = [ + col for col in df[filter].columns if col.startswith("CSadvocact2") + ] + CSadvocact2_df_total = df[filter][CSadvocact2_col_list] + # Make all NaNs numeric zeroes + CSadvocact2_df_total = CSadvocact2_df_total.fillna(0) + # Now replace everything that is not a number with "Y" + for col in CSadvocact2_df_total.columns: + CSadvocact2_df_total[col] = ( + pd.to_numeric(CSadvocact2_df_total[col], errors="coerce") + .fillna("Y") + .astype("string") + ) + # Make a column to count "Y" + CSadvocact2_df_total["answered"] = [ + "Y" if x > 0 else "N" for x in np.sum(CSadvocact2_df_total.values == "Y", 1) + ] + print_total(CSadvocact2_df_total["answered"].value_counts().sort_index()[1]) + except IndexError: + print_total(0) + st.plotly_chart( + gen_go_bar_stack( + data=[ + go.Bar( + name="Very important", + x=options_clean, + y=CSadvocact2_very_important, + marker_color=colors[0], + ), + go.Bar( + name="Somewhat important", + x=options_clean, + y=CSadvocact2_somewhat_important, + marker_color=colors[1], + ), + go.Bar( + name="Important", + x=options_clean, + y=CSadvocact2_important, + marker_color=colors[2], + ), + go.Bar( + name="Slightly important", + x=options_clean, + y=CSadvocact2_slightly_important, + marker_color=colors[3], + ), + go.Bar( + name="Not important at all", + x=options_clean, + y=CSadvocact2_not_important, + marker_color=colors[5], + ), + ], + ), + use_container_width=True, + config=chart_config, + ) + + # ======================================================================= + st.write("# Transnational Scope") + + st.write( + "### How frequently does your policy advocacy address transnational issues of surveillance by intelligence agencies? `[CSadvoctrans1]`" + ) + answered_by("cso") + CSadvoctrans1_counts = df[filter]["CSadvoctrans1"].value_counts() + print_total(CSadvoctrans1_counts.sum()) + st.plotly_chart( + gen_go_pie( + labels=CSadvoctrans1_counts.sort_index().index, + values=CSadvoctrans1_counts.sort_index().values, + ), + use_container_width=True, + config=chart_config, + ) + + # ======================================================================= + st.write( + "### When performing policy advocacy concerning surveillance by intelligence agencies, how often do you collaborate with civil society actors in other countries? `[CSadvoctrans2]`" + ) + answered_by("cso") + CSadvoctrans2_counts = df[filter]["CSadvoctrans2"].value_counts() + print_total(CSadvoctrans2_counts.sum()) + st.plotly_chart( + gen_go_pie( + labels=CSadvoctrans2_counts.sort_index().index, + values=CSadvoctrans2_counts.sort_index().values, + ), + use_container_width=True, + config=chart_config, + ) + + # ======================================================================= + st.write("# Perceived Impact") + + st.write( + "### How much do you agree with the following statements concerning the effectiveness of your advocacy activities regarding intelligence-related issues over the past 5 years? `[CSadvocimpact1]`" + ) + options = [ + "increased_awareness", + "policies_reflect_recommendations", + "more_informed_debates", + "achieved_goals", + ] + options_clean = [ + "Helped increase public awareness", + "Our policy recommendations have been reflected in politics", + "Contributed to more informed debate", + "Achieved defined goals", + ] + CSadvocimpact1_agree_completely = [] + CSadvocimpact1_agree_to_great_extent = [] + CSadvocimpact1_agree_somewhat = [] + CSadvocimpact1_agree_slightly = [] + CSadvocimpact1_not_agree_at_all = [] + for agreement in [ + "Agree completely", + "Agree to a great extent", + "Agree somewhat", + "Agree slightly", + "Not agree at all", + ]: + for option in options: + try: + count = df[filter][f"CSadvocimpact1[{option}]"].value_counts()[ + agreement + ] + except KeyError: + count = 0 + if agreement == "Agree completely": + CSadvocimpact1_agree_completely.append(count) + elif agreement == "Agree to a great extent": + CSadvocimpact1_agree_to_great_extent.append(count) + elif agreement == "Agree somewhat": + CSadvocimpact1_agree_somewhat.append(count) + elif agreement == "Agree slightly": + CSadvocimpact1_agree_slightly.append(count) + elif agreement == "Not agree at all": + CSadvocimpact1_not_agree_at_all.append(count) + else: + continue + st.plotly_chart( + gen_go_bar_stack( + data=[ + go.Bar( + name="Agree completely", + x=options_clean, + y=CSadvocimpact1_agree_completely, + marker_color=colors[0], + ), + go.Bar( + name="Agree to a great extent", + x=options_clean, + y=CSadvocimpact1_agree_to_great_extent, + marker_color=colors[1], + ), + go.Bar( + name="Agree somewhat", + x=options_clean, + y=CSadvocimpact1_agree_somewhat, + marker_color=colors[2], + ), + go.Bar( + name="Agree slightly", + x=options_clean, + y=CSadvocimpact1_agree_slightly, + marker_color=colors[3], + ), + go.Bar( + name="Not agree at all", + x=options_clean, + y=CSadvocimpact1_not_agree_at_all, + marker_color=colors[5], + ), + ], + ), + use_container_width=True, + config=chart_config, + ) + +# =========================================================================== +# Strategic Litigation +# =========================================================================== + +if selected_section == "Strategic Litigation": + st.caption( + "__NB__: All questions in this section have only been presented to civil society organisation professionals." + ) + st.write("# Activity") + + st.write( + "### How important are the following policy litigateacy tools for your work on intelligence-related issues? `[CSlitigateact2]`" + ) + answered_by("cso") + options = [ + "initiating_lawsuit", + "initiating_complaint", + "supporting_existing_legislation", + "other", + ] + options_clean = [ + "Initiating and/or coordinating a lawsuit", + "Initiating a legal complaint", + "Supporting existing legislation", + "Other", + ] + CSlitigateact2_very_important = [] + CSlitigateact2_somewhat_important = [] + CSlitigateact2_important = [] + CSlitigateact2_slightly_important = [] + CSlitigateact2_not_important = [] + for importance in [ + "Very important", + "Somewhat important", + "Important", + "Slightly important", + "Not important at all", + ]: + for option in options: + try: + count = df[filter][f"CSlitigateact2[{option}]"].value_counts()[ + importance + ] + except KeyError: + count = 0 + if importance == "Very important": + CSlitigateact2_very_important.append(count) + elif importance == "Somewhat important": + CSlitigateact2_somewhat_important.append(count) + elif importance == "Important": + CSlitigateact2_important.append(count) + elif importance == "Slightly important": + CSlitigateact2_slightly_important.append(count) + elif importance == "Not important at all": + CSlitigateact2_not_important.append(count) + else: + continue + try: + # If one respondent chose at least one medium it counts towards the total + CSlitigateact2_col_list = [ + col for col in df[filter].columns if col.startswith("CSlitigateact2") + ] + CSlitigateact2_df_total = df[filter][CSlitigateact2_col_list] + # Make all NaNs numeric zeroes + CSlitigateact2_df_total = CSlitigateact2_df_total.fillna(0) + # Now replace everything that is not a number with "Y" + for col in CSlitigateact2_df_total.columns: + CSlitigateact2_df_total[col] = ( + pd.to_numeric(CSlitigateact2_df_total[col], errors="coerce") + .fillna("Y") + .astype("string") + ) + # Make a column to count "Y" + CSlitigateact2_df_total["answered"] = [ + "Y" if x > 0 else "N" + for x in np.sum(CSlitigateact2_df_total.values == "Y", 1) + ] + print_total(CSlitigateact2_df_total["answered"].value_counts().sort_index()[1]) + except IndexError: + print_total(0) + + st.plotly_chart( + gen_go_bar_stack( + data=[ + go.Bar( + name="Very important", + x=options_clean, + y=CSlitigateact2_very_important, + marker_color=colors[0], + ), + go.Bar( + name="Somewhat important", + x=options_clean, + y=CSlitigateact2_somewhat_important, + marker_color=colors[1], + ), + go.Bar( + name="Important", + x=options_clean, + y=CSlitigateact2_important, + marker_color=colors[2], + ), + go.Bar( + name="Slightly important", + x=options_clean, + y=CSlitigateact2_slightly_important, + marker_color=colors[3], + ), + go.Bar( + name="Not important at all", + x=options_clean, + y=CSlitigateact2_not_important, + marker_color=colors[5], + ), + ], + ), + use_container_width=True, + config=chart_config, + ) + + # ======================================================================= + st.write("# Costs") + + st.write( + "### How frequently did the costs (e.g. court fees, loser pays principles, lawyers fees) prevent your organisation from starting a strategic litigation process? `[CSlitigatecost1]`" + ) + answered_by("cso") + CSlitigatecost1_counts = df[filter]["CSlitigatecost1"].value_counts().sort_index() + print_total(CSlitigatecost1_counts.sum()) + st.plotly_chart( + gen_go_pie( + labels=CSlitigatecost1_counts.sort_index().index, + values=CSlitigatecost1_counts.sort_index().values, + ), + use_container_width=True, + config=chart_config, + ) + + # ======================================================================= + st.write( + "### How frequently did your organisation benefit from pro bono support? `[CSlitigatecost2]`" + ) + answered_by("cso") + CSlitigatecost2_counts = df[filter]["CSlitigatecost2"].value_counts() + print_total(CSlitigatecost2_counts.sum()) + st.plotly_chart( + gen_go_pie( + labels=CSlitigatecost2_counts.sort_index().index, + values=CSlitigatecost2_counts.sort_index().values, + ), + use_container_width=True, + config=chart_config, + ) + + # ======================================================================= + st.write( + "### Imagine your organisation lost a strategic litigation case concerning surveillance by intelligence agencies. How financially risky would it be for the organisation to be defeated in court? `[CSlitigatecost3]`" + ) + answered_by("cso") + CSlitigatecost3_counts = df[filter]["CSlitigatecost3"].value_counts() + print_total(CSlitigatecost3_counts.sum()) + st.plotly_chart( + gen_go_pie( + labels=CSlitigatecost3_counts.sort_index().index, + values=CSlitigatecost3_counts.sort_index().values, + ), + use_container_width=True, + config=chart_config, + ) + + # ======================================================================= + st.write("# Transnational Scope") + + st.write( + "### How frequently do your strategic litigation cases address transnational issues of surveillance by intelligence agencies? `[CSlitigatetrans1]`" + ) + answered_by("cso") + CSlitigatetrans1_counts = df[filter]["CSlitigatetrans1"].value_counts() + print_total(CSlitigatetrans1_counts.sum()) + st.plotly_chart( + gen_go_pie( + labels=CSlitigatetrans1_counts.sort_index().index, + values=CSlitigatetrans1_counts.sort_index().values, + ), + use_container_width=True, + config=chart_config, + ) + + # ======================================================================= + st.write( + "### When performing strategic litigation concerning surveillance by intelligence agencies, how often do you collaborate with civil society actors in other countries? `[CSlitigatetrans2]`" + ) + answered_by("cso") + CSlitigatetrans2_counts = df[filter]["CSlitigatetrans2"].value_counts() + print_total(CSlitigatetrans2_counts.sum()) + st.plotly_chart( + gen_go_pie( + labels=CSlitigatetrans2_counts.sort_index().index, + values=CSlitigatetrans2_counts.sort_index().values, + ), + use_container_width=True, + config=chart_config, + ) + + # ======================================================================= + st.write("# Perceived Impact") + + st.write( + "### How much do you agree with the following statements concerning the effectiveness of your strategic litigation activities regarding surveillance by intelligence agencies over the past 5 years? `[CSlitigateimpact1]`" + ) + answered_by("cso") + options = [ + "increased_awareness", + "changed_the_law", + "amendments_of_the_law", + "revealed_new_information", + "achieved_goals", + ] + options_clean = [ + "Helped increase public awareness", + "Changed the prevalent case law", + "Led to amendments in legislation", + "Revealed new information", + "Achieved defined goals", + ] + CSlitigateimpact1_agree_completely = [] + CSlitigateimpact1_agree_to_great_extent = [] + CSlitigateimpact1_agree_somewhat = [] + CSlitigateimpact1_agree_slightly = [] + CSlitigateimpact1_not_agree_at_all = [] + for agreement in [ + "Agree completely", + "Agree to a great extent", + "Agree somewhat", + "Agree slightly", + "Not agree at all", + ]: + for option in options: + try: + count = df[filter][f"CSlitigateimpact1[{option}]"].value_counts()[ + agreement + ] + except KeyError: + count = 0 + if agreement == "Agree completely": + CSlitigateimpact1_agree_completely.append(count) + elif agreement == "Agree to a great extent": + CSlitigateimpact1_agree_to_great_extent.append(count) + elif agreement == "Agree somewhat": + CSlitigateimpact1_agree_somewhat.append(count) + elif agreement == "Agree slightly": + CSlitigateimpact1_agree_slightly.append(count) + elif agreement == "Not agree at all": + CSlitigateimpact1_not_agree_at_all.append(count) + else: + continue + try: + # If one respondent chose at least one medium it counts towards the total + CSlitigateimpact1_col_list = [ + col for col in df[filter].columns if col.startswith("CSlitigateimpact1") + ] + CSlitigateimpact1_df_total = df[filter][CSlitigateimpact1_col_list] + # Make all NaNs numeric zeroes + CSlitigateimpact1_df_total = CSlitigateimpact1_df_total.fillna(0) + # Now replace everything that is not a number with "Y" + for col in CSlitigateimpact1_df_total.columns: + CSlitigateimpact1_df_total[col] = ( + pd.to_numeric(CSlitigateimpact1_df_total[col], errors="coerce") + .fillna("Y") + .astype("string") + ) + # Make a column to count "Y" + CSlitigateimpact1_df_total["answered"] = [ + "Y" if x > 0 else "N" + for x in np.sum(CSlitigateimpact1_df_total.values == "Y", 1) + ] + print_total( + CSlitigateimpact1_df_total["answered"].value_counts().sort_index()[1] + ) + except IndexError: + print_total(0) + + st.plotly_chart( + gen_go_bar_stack( + data=[ + go.Bar( + name="Agree completely", + x=options_clean, + y=CSlitigateimpact1_agree_completely, + marker_color=colors[0], + ), + go.Bar( + name="Agree to a great extent", + x=options_clean, + y=CSlitigateimpact1_agree_to_great_extent, + marker_color=colors[1], + ), + go.Bar( + name="Agree somewhat", + x=options_clean, + y=CSlitigateimpact1_agree_somewhat, + marker_color=colors[2], + ), + go.Bar( + name="Agree slightly", + x=options_clean, + y=CSlitigateimpact1_agree_slightly, + marker_color=colors[3], + ), + go.Bar( + name="Not agree at all", + x=options_clean, + y=CSlitigateimpact1_not_agree_at_all, + marker_color=colors[5], + ), + ], + ), + use_container_width=True, + config=chart_config, ) # =========================================================================== diff --git a/scripts/clean.py b/scripts/clean.py index 13ec359..6fbe310 100755 --- a/scripts/clean.py +++ b/scripts/clean.py @@ -21,12 +21,12 @@ def construct_cs_df(): columns={ "startlanguage": "XXcountry", "lastpage": "XXlastpage", - "CSfinance2[SQ01]": "CSfinance2cs[private_foundations]", - "CSfinance2[SQ02]": "CSfinance2cs[donations]", - "CSfinance2[SQ03]": "CSfinance2cs[national_public_funds]", - "CSfinance2[SQ04]": "CSfinance2cs[corporate_sponsorship]", - "CSfinance2[SQ05]": "CSfinance2cs[international_public_funds]", - "CSfinance2[SQ06]": "CSfinance2cs[other]", + "CSfinance2[SQ01]": "CSCSfinance2[private_foundations]", + "CSfinance2[SQ02]": "CSCSfinance2[donations]", + "CSfinance2[SQ03]": "CSCSfinance2[national_public_funds]", + "CSfinance2[SQ04]": "CSCSfinance2[corporate_sponsorship]", + "CSfinance2[SQ05]": "CSCSfinance2[international_public_funds]", + "CSfinance2[SQ06]": "CSCSfinance2[other]", "CSfoi5[SQ01]": "CSfoi5[not_aware]", "CSfoi5[SQ02]": "CSfoi5[not_covered]", "CSfoi5[SQ03]": "CSfoi5[too_expensive]", @@ -36,6 +36,55 @@ def construct_cs_df(): "CSfoi5[SQ07]": "CSfoi5[other]", "CSfoi5[SQ08]": "CSfoi5[dont_know]", "CSfoi5[SQ09]": "CSfoi5[prefer_not_to_say]", + "CSpreselection": "CSCSpreselection", + "CScampact2[SQ01]": "CSCScampact2[media_contributions]", + "CScampact2[SQ02]": "CSCScampact2[own_publications]", + "CScampact2[SQ03]": "CSCScampact2[petitions_open_letters]", + "CScampact2[SQ04]": "CSCScampact2[public_events]", + "CScampact2[SQ05]": "CSCScampact2[collaborations]", + "CScampact2[SQ06]": "CSCScampact2[demonstrations]", + "CScampact2[SQ07]": "CSCScampact2[social_media]", + "CScampact2[SQ08]": "CSCScampact2[advertising]", + "CScampact2[SQ09]": "CSCScampact2[volunteer_activities]", + "CScampact2[SQ10]": "CSCScampact2[providing_technical_tools]", + "CScampact2[SQ11]": "CSCScampact2[support_for_eu_campaigns]", + "CScampact2[SQ12]": "CSCScampact2[other]", + "CScampimpact1[SQ01]": "CSCScampimpact1[increased_awareness]", + "CScampimpact1[SQ02]": "CSCScampimpact1[policies_reflect_demands]", + "CScampimpact1[SQ03]": "CSCScampimpact1[created_media_attention]", + "CScampimpact1[SQ04]": "CSCScampimpact1[achieved_goals]", + "CScampimpact2": "CSCScampimpact2", + "CScamptrans1": "CSCScamptrans1", + "CScamptrans2": "CSCScamptrans2", + "CSadvocact2[SQ01]": "CSCSadvocact2[research]", + "CSadvocact2[SQ02]": "CSCSadvocact2[consultations]", + "CSadvocact2[SQ03]": "CSCSadvocact2[briefings]", + "CSadvocact2[SQ04]": "CSCSadvocact2[expert_events]", + "CSadvocact2[SQ05]": "CSCSadvocact2[participation_in_fora]", + "CSadvocact2[SQ06]": "CSCSadvocact2[legal_opinions]", + "CSadvocact2[SQ07]": "CSCSadvocact2[informal_encounters]", + "CSadvocact2[SQ08]": "CSCSadvocact2[other]", + "CSadvoctrans1": "CSCSadvoctrans1", + "CSadvoctrans2": "CSCSadvoctrans2", + "CSadvocimpact1[SQ01]": "CSCSadvocimpact1[increased_awareness]", + "CSadvocimpact1[SQ02]": "CSCSadvocimpact1[policies_reflect_recommendations]", + "CSadvocimpact1[SQ03]": "CSCSadvocimpact1[more_informed_debates]", + "CSadvocimpact1[SQ04]": "CSCSadvocimpact1[achieved_goals]", + "CSlitigateact2[SQ01]": "CSCSlitigateact2[initiating_lawsuit]", + "CSlitigateact2[SQ02]": "CSCSlitigateact2[initiating_complaint]", + "CSlitigateact2[SQ03]": "CSCSlitigateact2[supporting_existing_legislation]", + "CSlitigateact2[SQ04]": "CSCSlitigateact2[other]", + "CSlitigateimpact1[SQ01]": "CSCSlitigateimpact1[increased_awareness]", + "CSlitigateimpact1[SQ02]": "CSCSlitigateimpact1[changed_the_law]", + "CSlitigateimpact1[SQ03]": "CSCSlitigateimpact1[amendments_of_the_law]", + "CSlitigateimpact1[SQ04]": "CSCSlitigateimpact1[revealed_new_information]", + "CSlitigateimpact1[SQ05]": "CSCSlitigateimpact1[achieved_goals]", + "CSlitigateimpact2": "CSCSlitigateimpact2", + "CSlitigatecost1": "CSCSlitigatecost1", + "CSlitigatecost2": "CSCSlitigatecost2", + "CSlitigatecost3": "CSCSlitigatecost3", + "CSlitigatetrans1": "CSCSlitigatetrans1", + "CSlitigatetrans2": "CSCSlitigatetrans2", "CSprotectops1[SQ01]": "CSprotectops1[sectraining]", "CSprotectops1[SQ02]": "CSprotectops1[e2e]", "CSprotectops3[SQ01]": "CSprotectops3[encrypted_email]", @@ -62,14 +111,12 @@ def construct_cs_df(): "CSconstraintinter5[SQ01]": "CSconstraintinter5[unsolicited_information]", "CSconstraintinter5[SQ02]": "CSconstraintinter5[invitations]", "CSconstraintinter5[SQ03]": "CSconstraintinter5[other]", - # "CSconstraintinter5ot": "CSconstraintinter5other", "CSconstraintinter6[SQ01]": "CSconstraintinter6[gender]", "CSconstraintinter6[SQ02]": "CSconstraintinter6[ethnicity]", "CSconstraintinter6[SQ03]": "CSconstraintinter6[political]", "CSconstraintinter6[SQ04]": "CSconstraintinter6[sexual]", "CSconstraintinter6[SQ05]": "CSconstraintinter6[religious]", "CSconstraintinter6[SQ06]": "CSconstraintinter6[other]", - # "CSconstraintinter6ot": "CSconstraintinter6other", "CSattitude3[SQ01]": "CSattitude3[rule_of_law]", "CSattitude3[SQ02]": "CSattitude3[civil_liberties]", "CSattitude3[SQ03]": "CSattitude3[effectiveness_of_intel]", @@ -99,13 +146,12 @@ def construct_cs_df(): "CSexpertise3", "CSexpertise4", "CSfinance1", - "CSfinance2cs[private_foundations]", - "CSfinance2cs[donations]", - "CSfinance2cs[national_public_funds]", - "CSfinance2cs[corporate_sponsorship]", - "CSfinance2cs[international_public_funds]", - "CSfinance2cs[other]", - # "CSfinance2other", + "CSCSfinance2[private_foundations]", + "CSCSfinance2[donations]", + "CSCSfinance2[national_public_funds]", + "CSCSfinance2[corporate_sponsorship]", + "CSCSfinance2[international_public_funds]", + "CSCSfinance2[other]", "CSfinance3", "CSfinance4", "CSfoi1", @@ -121,7 +167,54 @@ def construct_cs_df(): "CSfoi5[other]", "CSfoi5[dont_know]", "CSfoi5[prefer_not_to_say]", - # "CSfoi5other", + "CSCSpreselection", + "CSCScampact2[media_contributions]", + "CSCScampact2[own_publications]", + "CSCScampact2[petitions_open_letters]", + "CSCScampact2[public_events]", + "CSCScampact2[collaborations]", + "CSCScampact2[demonstrations]", + "CSCScampact2[social_media]", + "CSCScampact2[advertising]", + "CSCScampact2[volunteer_activities]", + "CSCScampact2[providing_technical_tools]", + "CSCScampact2[support_for_eu_campaigns]", + "CSCScampact2[other]", + "CSCScamptrans1", + "CSCScamptrans2", + "CSCScampimpact1[increased_awareness]", + "CSCScampimpact1[policies_reflect_demands]", + "CSCScampimpact1[created_media_attention]", + "CSCScampimpact1[achieved_goals]", + "CSCSadvoctrans1", + "CSCSadvoctrans2", + "CSCSadvocact2[research]", + "CSCSadvocact2[consultations]", + "CSCSadvocact2[briefings]", + "CSCSadvocact2[expert_events]", + "CSCSadvocact2[participation_in_fora]", + "CSCSadvocact2[legal_opinions]", + "CSCSadvocact2[informal_encounters]", + "CSCSadvocact2[other]", + "CSCSadvocimpact1[increased_awareness]", + "CSCSadvocimpact1[policies_reflect_recommendations]", + "CSCSadvocimpact1[more_informed_debates]", + "CSCSadvocimpact1[achieved_goals]", + "CSCSlitigateact2[initiating_lawsuit]", + "CSCSlitigateact2[initiating_complaint]", + "CSCSlitigateact2[supporting_existing_legislation]", + "CSCSlitigateact2[other]", + "CSCSlitigatecost1", + "CSCSlitigatecost2", + "CSCSlitigatecost3", + "CSCSlitigatetrans1", + "CSCSlitigatetrans2", + "CSCSlitigateimpact1[increased_awareness]", + "CSCSlitigateimpact1[changed_the_law]", + "CSCSlitigateimpact1[amendments_of_the_law]", + "CSCSlitigateimpact1[revealed_new_information]", + "CSCSlitigateimpact1[achieved_goals]", + "CSCSlitigateimpact2", "CSprotectops1[sectraining]", "CSprotectops1[e2e]", "CSprotectops2", @@ -132,15 +225,12 @@ def construct_cs_df(): "CSprotectops3[encrypted_hardware]", "CSprotectops3[2fa]", "CSprotectops3[other]", - # "CSprotectops3other", "CSprotectops4", "CSprotectleg1", "CSprotectleg2", - # "CSprotectleg2no", "CSprotectleg3[free_counsel]", "CSprotectleg3[cost_insurance]", "CSprotectleg3[other]", - # "CSprotectleg3other", "CSconstraintinter1", "CSconstraintinter2", "CSconstraintinter3", @@ -158,14 +248,12 @@ def construct_cs_df(): "CSconstraintinter5[unsolicited_information]", "CSconstraintinter5[invitations]", "CSconstraintinter5[other]", - # "CSconstraintinter5other", "CSconstraintinter6[gender]", "CSconstraintinter6[ethnicity]", "CSconstraintinter6[political]", "CSconstraintinter6[sexual]", "CSconstraintinter6[religious]", "CSconstraintinter6[other]", - # "CSconstraintinter6other", "CSattitude1", "CSattitude2", "CSattitude3[rule_of_law]", @@ -221,17 +309,17 @@ def construct_ms_df(): columns={ "startlanguage": "XXcountry", "lastpage": "XXlastpage", - "MSfinance2": "MSfinance2ms", + "MSfinance2": "MSMSfinance2", "MFfoi2": "MSfoi2", - "MShr3[SQ01]": "MShr3ms[daily_newspaper]", - "MShr3[SQ02]": "MShr3ms[weekly_newspaper]", - "MShr3[SQ03]": "MShr3ms[magazine]", - "MShr3[SQ04]": "MShr3ms[tv]", - "MShr3[SQ05]": "MShr3ms[radio]", - "MShr3[SQ06]": "MShr3ms[news_agency]", - "MShr3[SQ07]": "MShr3ms[online_stand_alone]", - "MShr3[SQ08]": "MShr3ms[online_of_offline]", - "MShr4": "MShr4ms", + "MShr3[SQ01]": "MSMShr3[daily_newspaper]", + "MShr3[SQ02]": "MSMShr3[weekly_newspaper]", + "MShr3[SQ03]": "MSMShr3[magazine]", + "MShr3[SQ04]": "MSMShr3[tv]", + "MShr3[SQ05]": "MSMShr3[radio]", + "MShr3[SQ06]": "MSMShr3[news_agency]", + "MShr3[SQ07]": "MSMShr3[online_stand_alone]", + "MShr3[SQ08]": "MSMShr3[online_of_offline]", + "MShr4": "MSMShr4", "MSfoi5[SQ01]": "MSfoi5[not_aware]", "MSfoi5[SQ02]": "MSfoi5[not_covered]", "MSfoi5[SQ03]": "MSfoi5[too_expensive]", @@ -340,22 +428,22 @@ def construct_ms_df(): "XXlastpage", "MShr1", "MShr2", - "MShr3ms[daily_newspaper]", - "MShr3ms[weekly_newspaper]", - "MShr3ms[magazine]", - "MShr3ms[tv]", - "MShr3ms[radio]", - "MShr3ms[news_agency]", - "MShr3ms[online_stand_alone]", - "MShr3ms[online_of_offline]", - "MShr4ms", + "MSMShr3[daily_newspaper]", + "MSMShr3[weekly_newspaper]", + "MSMShr3[magazine]", + "MSMShr3[tv]", + "MSMShr3[radio]", + "MSMShr3[news_agency]", + "MSMShr3[online_stand_alone]", + "MSMShr3[online_of_offline]", + "MSMShr4", "MSgender", "MSexpertise1", "MSexpertise2", "MSexpertise3", "MSexpertise4", "MSfinance1", - "MSfinance2ms", + "MSMSfinance2", "MSfoi1", "MSfoi2", "MSfoi3", @@ -532,7 +620,7 @@ def construct_ms_df(): } ) -df["hr4ms"] = df["hr4ms"].replace( +df["MShr4"] = df["MShr4"].replace( { "AO01": "A1: I had enough time", "AO02": "A2: I had some time", @@ -602,7 +690,7 @@ def construct_ms_df(): } ) -df["finance2ms"] = df["finance2ms"].replace( +df["MSfinance2"] = df["MSfinance2"].replace( { "AO01": "Yes", "AO02": "No", @@ -611,7 +699,7 @@ def construct_ms_df(): } ) -finance2cs_options = [ +CSfinance2_options = [ "private_foundations", "donations", "national_public_funds", @@ -619,8 +707,8 @@ def construct_ms_df(): "international_public_funds", "other", ] -for label in finance2cs_options: - df[f"finance2cs[{label}]"] = df[f"finance2cs[{label}]"].replace( +for label in CSfinance2_options: + df[f"CSfinance2[{label}]"] = df[f"CSfinance2[{label}]"].replace( { "AO01": "Very important", "AO02": "Important", @@ -731,6 +819,275 @@ def construct_ms_df(): "AO04": "I prefer not to say", } ) + +df["CSpreselection"] = df["CSpreselection"].replace( + { + "AO01": "Public Campaigning", + "AO02": "Policy Advocacy", + "AO03": "Strategic Litigation", + } +) + +CScampact2_options = [ + "media_contributions", + "own_publications", + "petitions_open_letters", + "public_events", + "collaborations", + "demonstrations", + "social_media", + "advertising", + "volunteer_activities", + "providing_technical_tools", + "support_for_eu_campaigns", + "other", +] +for label in CScampact2_options: + df[f"CScampact2[{label}]"] = df[f"CScampact2[{label}]"].replace( + { + "AO01": "Very important", + "AO02": "Important", + "AO03": "Somewhat important", + "AO04": "Slightly important", + "AO05": "Not important at all", + "AO06": "I don't know", + "AO07": "I prefer not to say", + } + ) + +df["CScamptrans1"] = df["CScamptrans1"].replace( + { + "AO01": "A1: Always", + "AO02": "A2: Often (75% of the time)", + "AO03": "A3: Sometimes (50% of the time)", + "AO04": "A4: Rarely (25% of the time)", + "AO05": "A5: Never", + "AO06": "A6: I don't know", + "AO07": "A7: I prefer not to say", + } +) + +df["CScamptrans2"] = df["CScamptrans2"].replace( + { + "AO01": "A1: Always", + "AO02": "A2: Often (75% of the time)", + "AO03": "A3: Sometimes (50% of the time)", + "AO04": "A4: Rarely (25% of the time)", + "AO05": "A5: Never", + "AO06": "A6: I don't know", + "AO07": "A7: I prefer not to say", + } +) + +CScampimpact1_options = [ + "increased_awareness", + "policies_reflect_demands", + "created_media_attention", + "achieved_goals", +] +for label in CScampimpact1_options: + df[f"CScampimpact1[{label}]"] = df[f"CScampimpact1[{label}]"].replace( + { + "AO01": "Agree completely", + "AO02": "Agree to a great extent", + "AO03": "Agree somewhat", + "AO04": "Agree sligthly", + "AO05": "Not agree at all", + "AO06": "I don't know", + "AO07": "I prefer not to say", + } + ) + +CSadvocact2_options = [ + "research", + "consultations", + "briefings", + "expert_events", + "participation_in_fora", + "legal_opinions", + "informal_encounters", + "other", +] +for label in CSadvocact2_options: + df[f"CSadvocact2[{label}]"] = df[f"CSadvocact2[{label}]"].replace( + { + "AO01": "Very important", + "AO02": "Important", + "AO03": "Somewhat important", + "AO04": "Slightly important", + "AO07": "Not important at all", + "AO09": "I don't know", + "AO11": "I prefer not to say", + } + ) + # Coding in LimeSurvey differs for UK + df.loc[is_uk, f"CSadvocact2[{label}]"] = df[f"CSadvocact2[{label}]"].replace( + { + "AO01": "Very important", + "AO02": "Important", + "AO03": "Somewhat important", + "AO04": "Slightly important", + "AO05": "Not important at all", + "AO06": "I don't know", + "AO07": "I prefer not to say", + } + ) + + +df["CSadvoctrans1"] = df["CSadvoctrans1"].replace( + { + "AO01": "A1: Always", + "AO02": "A2: Often (75% of the time)", + "AO03": "A3: Sometimes (50% of the time)", + "AO04": "A4: Rarely (25% of the time)", + "AO05": "A5: Never", + "AO06": "A6: I don't know", + "AO07": "A7: I prefer not to say", + } +) + +df["CSadvoctrans2"] = df["CSadvoctrans2"].replace( + { + "AO01": "A1: Always", + "AO02": "A2: Often (75% of the time)", + "AO03": "A3: Sometimes (50% of the time)", + "AO04": "A4: Rarely (25% of the time)", + "AO05": "A5: Never", + "AO06": "A6: I don't know", + "AO07": "A7: I prefer not to say", + } +) + +CSadvocimpact1_options = [ + "increased_awareness", + "policies_reflect_recommendations", + "more_informed_debates", + "achieved_goals", +] +for label in CSadvocimpact1_options: + df[f"CSadvocimpact1[{label}]"] = df[f"CSadvocimpact1[{label}]"].replace( + { + "AO01": "Agree completely", + "AO42": "Agree to a great extent", + "AO43": "Agree somewhat", + "AO44": "Agree sligthly", + "AO45": "Not agree at all", + "AO46": "I don't know", + "AO47": "I prefer not to say", + } + ) + # Here only DE survey is differenlty coded + df.loc[is_de, f"CSadvocimpact1[{label}]"] = df[f"CSadvocimpact1[{label}]"].replace( + { + "AO01": "Agree completely", + "AO02": "Agree to a great extent", + "AO03": "Agree somewhat", + "AO04": "Agree sligthly", + "AO05": "Not agree at all", + "AO06": "I don't know", + "AO07": "I prefer not to say", + } + ) + +CSlitigateact2_options = [ + "initiating_lawsuit", + "initiating_complaint", + "supporting_existing_legislation", + "other", +] +for label in CSlitigateact2_options: + df[f"CSlitigateact2[{label}]"] = df[f"CSlitigateact2[{label}]"].replace( + { + "AO01": "Very important", + "AO02": "Important", + "AO03": "Somewhat important", + "AO04": "Slightly important", + "AO05": "Not important at all", + "AO06": "I don't know", + "AO07": "I prefer not to say", + } + ) + +df["CSlitigatetrans1"] = df["CSlitigatetrans1"].replace( + { + "AO01": "A1: Always", + "AO02": "A2: Often (75% of the time)", + "AO03": "A3: Sometimes (50% of the time)", + "AO04": "A4: Rarely (25% of the time)", + "AO05": "A5: Never", + "AO06": "A6: I don't know", + "AO07": "A7: I prefer not to say", + } +) + +df["CSlitigatetrans2"] = df["CSlitigatetrans2"].replace( + { + "AO01": "A1: Always", + "AO02": "A2: Often (75% of the time)", + "AO03": "A3: Sometimes (50% of the time)", + "AO04": "A4: Rarely (25% of the time)", + "AO05": "A5: Never", + "AO06": "A6: I don't know", + "AO07": "A7: I prefer not to say", + } +) + +CSlitigateimpact1_options = [ + "increased_awareness", + "changed_the_law", + "amendments_of_the_law", + "revealed_new_information", + "achieved_goals", +] +for label in CSlitigateimpact1_options: + df[f"CSlitigateimpact1[{label}]"] = df[f"CSlitigateimpact1[{label}]"].replace( + { + "AO01": "Agree completely", + "AO42": "Agree to a great extent", + "AO43": "Agree somewhat", + "AO44": "Agree sligthly", + "AO45": "Not agree at all", + "AO46": "I don't know", + "AO47": "I prefer not to say", + } + ) + + df.loc[is_de, f"CSlitigateimpact1[{label}]"] = df[ + f"CSlitigateimpact1[{label}]" + ].replace( + { + "AO01": "Agree completely", + "AO02": "Agree to a great extent", + "AO03": "Agree somewhat", + "AO04": "Agree sligthly", + "AO05": "Not agree at all", + "AO06": "I don't know", + "AO07": "I prefer not to say", + } + ) + +for i in range(1, 3): + df[f"CSlitigatecost{i}"] = df[f"CSlitigatecost{i}"].replace( + { + "AO01": "A1: Always or very often", + "AO02": "A2: Often (75% of the time)", + "AO03": "A3: Sometimes (50% of the time)", + "AO04": "A4: Rarely (25% of the time)", + "AO05": "A5: Never or rarely", + "AO06": "A6: I don't know", + "AO07": "A7: I prefer not to say", + } + ) + +df["CSlitigatecost3"] = df["CSlitigatecost3"].replace( + { + "AO01": "A1: Not risky at all", + "AO02": "A2: Somewaht risky", + "AO03": "A3: Very risky", + "AO04": "A4: I don't know", + "AO05": "A5: I prefer not to say", + } +) df["protectops1[sectraining]"] = df["protectops1[sectraining]"].replace( { "AO01": "Yes", @@ -770,13 +1127,13 @@ def construct_ms_df(): for label in protectops3_options: df[f"protectops3[{label}]"] = df[f"protectops3[{label}]"].replace( { - "AO01": "Very important", - "AO02": "Important", - "AO03": "Somewhat important", - "AO04": "Slightly important", - "AO05": "Not important at all", - "AO06": "I don't know", - "AO07": "I prefer not to say", + "AO01": "A1: Very important", + "AO02": "A2: Important", + "AO03": "A3: Somewhat important", + "AO04": "A4: Slightly important", + "AO05": "A5: Not important at all", + "AO06": "A6: I don't know", + "AO07": "A7: I prefer not to say", } ) # this was hard to spot. Only the CS survey for DE was coded as below @@ -785,14 +1142,14 @@ def construct_ms_df(): f"protectops3[{label}]" ].replace( { - "AO01": "Very important", - "AO02": "Important", - "AO03": "Somewhat important", - "AO04": "Slightly important", - "AO05": "Not important at all", + "AO01": "A1: Very important", + "AO02": "A2: Important", + "AO03": "A3: Somewhat important", + "AO04": "A4: Slightly important", + "AO05": "A5: Not important at all", # notice the AO09 instead of AO06 as above - "AO09": "I don't know", - "AO11": "I prefer not to say", + "AO09": "A6: I don't know", + "AO11": "A7: I prefer not to say", } ) @@ -980,7 +1337,7 @@ def construct_ms_df(): if ( col.startswith("foi5[") or col.startswith("attitude3") - or col.startswith("hr3") + or col.startswith("MShr3") or col.startswith("MSsoc4") or col.startswith("MSsoc5") or col.startswith("MSimpact1")