From 49b5d579064a7a2c8a0947d4258bd5dbea18572d Mon Sep 17 00:00:00 2001 From: linozen Date: Sun, 29 Aug 2021 18:27:38 +0200 Subject: [PATCH] improved merged --- explorer/merged.py | 1903 +++++++++++++++++++++++--------------------- 1 file changed, 987 insertions(+), 916 deletions(-) diff --git a/explorer/merged.py b/explorer/merged.py index 8d3eea2..cf6d263 100644 --- a/explorer/merged.py +++ b/explorer/merged.py @@ -7,6 +7,7 @@ from lib.figures import ( generate_pie_chart, + generate_boxplot, generate_histogram, generate_overlaid_histogram, generate_stacked_bar_chart, @@ -29,11 +30,13 @@ def render_pie_chart( df, values, names, - labels=None, color=None, color_discrete_sequence=px.colors.qualitative.Prism, + hover_data=None, + custom_data=None, color_discrete_map=None, hover_name=None, + labels=None, ): return generate_pie_chart( df, @@ -41,12 +44,19 @@ def render_pie_chart( names, hover_name, color, + hover_data, + custom_data, color_discrete_sequence, color_discrete_map, labels, ) +@st.cache +def render_boxplot(df, x, y, color=None, points="all", color_discrete_map=None): + return generate_boxplot(df, x, y, points, color, color_discrete_map) + + @st.cache def render_histogram(df, x, y, nbins, color, color_discrete_map, labels): return generate_histogram(df, x, y, nbins, color, color_discrete_map, labels) @@ -92,7 +102,7 @@ def get_significance_matrix(df): st.set_page_config( - page_title="IOI Survey Data Explorer (CS & MS)", + page_title="IOI Survey Data Explorer", ) @@ -878,10 +888,22 @@ def get_merged_ms_df(): ############################################################################### -# Filter logic +# Sidebar | Filter logic ############################################################################### +section = st.sidebar.radio( + "Choose section", + [ + "Overview", + "Resources", + "Protection", + "Constraints", + "Attitudes", + "Appendix", + ], +) + filters = { "surveytype": st.sidebar.selectbox( "Survey type", ["All", "Civil Society Scrutiny", "Media Scrutiny"] @@ -904,9 +926,9 @@ def get_merged_ms_df(): ############################################################################### -# df.to_pickle("./data/merged.pkl") -# df.to_excel("./data/merged.xlsx") -# df.to_csv("./data/merged.csv") +df.to_pickle("./data/merged.pkl") +df.to_excel("./data/merged.xlsx") +df.to_csv("./data/merged.csv") ############################################################################### @@ -925,1023 +947,1072 @@ def get_merged_ms_df(): ############################################################################### -st.title("IOI Survey Data Explorer (CS & MS)") +st.title("IOI Survey Data Explorer") +st.write( + "... of the responses given by __media__ and __civil society organisation__ representatives" +) -st.write("# General") +if section == "Overview": + st.write("# Overview") -merged_markdown = read_markdown_file("explorer/markdown/merged.md") -st.markdown(merged_markdown, unsafe_allow_html=True) + merged_markdown = read_markdown_file("explorer/markdown/merged.md") + st.markdown(merged_markdown, unsafe_allow_html=True) -st.write("### Country `[country]`") -country_counts = df[filter]["country"].value_counts() -st.plotly_chart( - render_pie_chart( - df[filter], - values=country_counts, - names=country_counts.index, + st.write("### Country `[country]`") + country_counts = df[filter]["country"].value_counts() + st.plotly_chart( + render_pie_chart( + df[filter], + values=country_counts, + names=country_counts.index, + ) ) -) -st.write("### Surveytype `[surveytype]`") -surveytype_counts = df[filter]["surveytype"].value_counts() -st.plotly_chart( - render_pie_chart( - df[filter], - values=surveytype_counts, - names=surveytype_counts.index, + st.write("### Surveytype `[surveytype]`") + surveytype_counts = df[filter]["surveytype"].value_counts() + st.plotly_chart( + render_pie_chart( + df[filter], + values=surveytype_counts, + names=surveytype_counts.index, + ) ) -) -st.write("# Resources") - -st.write("## Human Resources") - -st.write("### What is your employment status `[hr1]`") -hr1_counts = df[filter]["hr1"].value_counts() -st.plotly_chart( - render_pie_chart( - hr1_counts, - values=hr1_counts, - names=hr1_counts.index, - color=hr1_counts.index, - color_discrete_map={ - "Full-time": px.colors.qualitative.Prism[0], - "Part-time (>50%)": px.colors.qualitative.Prism[1], - "Part-time (<50%)": px.colors.qualitative.Prism[2], - "Freelance": px.colors.qualitative.Prism[4], - "Other": px.colors.qualitative.Prism[10], - }, +if section == "Resources": + st.write("# Resources") + + st.write("## Human Resources") + + st.write("### What is your employment status `[hr1]`") + hr1_counts = df[filter]["hr1"].value_counts() + st.plotly_chart( + render_pie_chart( + hr1_counts, + values=hr1_counts, + names=hr1_counts.index, + color=hr1_counts.index, + color_discrete_map={ + "Full-time": px.colors.qualitative.Prism[0], + "Part-time (>50%)": px.colors.qualitative.Prism[1], + "Part-time (<50%)": px.colors.qualitative.Prism[2], + "Freelance": px.colors.qualitative.Prism[4], + "Other": px.colors.qualitative.Prism[10], + }, + ) ) -) -st.write( - "### How many days per month do you work on surveillance by intelligence agencies? `[hr2]`" -) -st.plotly_chart( - render_histogram( - df=df[filter], - x="hr2", - y=None, - nbins=None, - color="country", - color_discrete_map={ - "Germany": px.colors.qualitative.Prism[5], - "France": px.colors.qualitative.Prism[1], - "United Kingdom": px.colors.qualitative.Prism[7], - }, - labels={"hr2": "days per month"}, + st.write( + "### How many days per month do you work on surveillance by intelligence agencies? `[hr2]`" + ) + st.plotly_chart( + render_histogram( + df=df[filter], + x="hr2", + y=None, + nbins=None, + color="country", + color_discrete_map={ + "Germany": px.colors.qualitative.Prism[5], + "France": px.colors.qualitative.Prism[1], + "United Kingdom": px.colors.qualitative.Prism[7], + }, + labels={"hr2": "days per month"}, + ) ) -) -st.write( - "### Respondents working more than 5 days on surveillance by intelligence agencies? `[hr2]`" -) -df["hr2_more_than_five"] = np.where(df["hr2"] > 5, True, False) -df["hr2_more_than_five"] = df["hr2_more_than_five"].replace( - {True: ">5 days", False: "5 days or less"} -) -hr2_more_than_five_counts = df[filter]["hr2_more_than_five"].value_counts() -print(hr2_more_than_five_counts) -st.plotly_chart( - render_pie_chart( - hr2_more_than_five_counts, - values=hr2_more_than_five_counts, - names=hr2_more_than_five_counts.index, - color=hr2_more_than_five_counts.index, + st.plotly_chart( + render_boxplot( + df=df[filter], + points="all", + x="country", + y="hr2", + color="country", + color_discrete_map={ + "Germany": px.colors.qualitative.Prism[5], + "France": px.colors.qualitative.Prism[1], + "United Kingdom": px.colors.qualitative.Prism[7], + }, + ) ) -) + df["hr2_more_than_five"] = np.where(df["hr2"] > 5, True, False) + df["hr2_more_than_five"] = df["hr2_more_than_five"].replace( + {True: ">5 days", False: "5 days or less"} + ) + hr2_more_than_five_counts = df[filter]["hr2_more_than_five"].value_counts() + st.plotly_chart( + render_pie_chart( + hr2_more_than_five_counts, + values=hr2_more_than_five_counts, + names=hr2_more_than_five_counts.index, + color=hr2_more_than_five_counts.index, + ) + ) -st.write("## Expertise") + st.write("## Expertise") -# TODO Show distribution/average for the three countries -st.write( - "### How many years have you spent working on surveillance by intelligence agencies? `[expertise1]`" -) -st.plotly_chart( - render_histogram( - df[filter], - x="expertise1", - y=None, - nbins=20, - color="country", - color_discrete_map={ - "Germany": px.colors.qualitative.Prism[5], - "France": px.colors.qualitative.Prism[1], - "United Kingdom": px.colors.qualitative.Prism[7], - }, - labels={"expertise1": "years"}, + st.write( + "### How many years have you spent working on surveillance by intelligence agencies? `[expertise1]`" ) -) - -st.write( - "### How do you assess your level of expertise concerning the **legal** aspects of surveillance by intelligence agencies? For example, knowledge of intelligence law, case law. `[expertise2]`" -) -expertise2_counts = df[filter]["expertise2"].value_counts() -st.plotly_chart( - render_pie_chart( - df[filter], - values=expertise2_counts, - names=expertise2_counts.index, - color_discrete_sequence=None, - color=expertise2_counts.index, - color_discrete_map={ - "Expert knowledge": px.colors.qualitative.Prism[9], - "Advanced knowledge": px.colors.qualitative.Prism[8], - "Some knowledge": px.colors.qualitative.Prism[7], - "Basic knowledge": px.colors.qualitative.Prism[6], - "No knowledge": px.colors.qualitative.Prism[5], - "I don't know": px.colors.qualitative.Prism[10], - "I prefer not to say": px.colors.qualitative.Prism[10], - }, + st.plotly_chart( + render_histogram( + df[filter], + x="expertise1", + y=None, + nbins=20, + color="country", + color_discrete_map={ + "Germany": px.colors.qualitative.Prism[5], + "France": px.colors.qualitative.Prism[1], + "United Kingdom": px.colors.qualitative.Prism[7], + }, + labels={"expertise1": "years"}, + ) ) -) -st.write( - "### How do you assess your level of expertise concerning the **political** aspects of surveillance by intelligence agencies?` [expertise3]`" -) -expertise3_counts = df[filter]["expertise3"].value_counts() -st.plotly_chart( - render_pie_chart( - df[filter], - values=expertise3_counts, - names=expertise3_counts.index, - color_discrete_sequence=None, - color=expertise3_counts.index, - color_discrete_map={ - "Expert knowledge": px.colors.qualitative.Prism[9], - "Advanced knowledge": px.colors.qualitative.Prism[8], - "Some knowledge": px.colors.qualitative.Prism[7], - "Basic knowledge": px.colors.qualitative.Prism[6], - "No knowledge": px.colors.qualitative.Prism[5], - "I don't know": px.colors.qualitative.Prism[10], - "I prefer not to say": px.colors.qualitative.Prism[10], - }, + st.plotly_chart( + render_boxplot( + df=df[filter], + points="all", + x="country", + y="expertise1", + color="country", + color_discrete_map={ + "Germany": px.colors.qualitative.Prism[5], + "France": px.colors.qualitative.Prism[1], + "United Kingdom": px.colors.qualitative.Prism[7], + }, + ) ) -) -st.write( - "### How do you assess your level of expertise concerning the **technical** aspects of surveillance by intelligence agencies?` [expertise4]`" -) -expertise4_counts = df[filter]["expertise4"].value_counts() -st.plotly_chart( - render_pie_chart( - df[filter], - values=expertise4_counts, - names=expertise4_counts.index, - color_discrete_sequence=None, - color=expertise4_counts.index, - color_discrete_map={ - "Expert knowledge": px.colors.qualitative.Prism[9], - "Advanced knowledge": px.colors.qualitative.Prism[8], - "Some knowledge": px.colors.qualitative.Prism[7], - "Basic knowledge": px.colors.qualitative.Prism[6], - "No knowledge": px.colors.qualitative.Prism[5], - "I don't know": px.colors.qualitative.Prism[10], - "I prefer not to say": px.colors.qualitative.Prism[10], - }, + st.write( + "### How do you assess your level of expertise concerning the **legal** aspects of surveillance by intelligence agencies? For example, knowledge of intelligence law, case law. `[expertise2]`" ) -) - -st.write("## Financial Resources") - -st.write( - "### How do you assess the financial resources that have been available for your work on intelligence over the past 5 years? `[finance1]`" -) -finance1_counts = df[filter]["finance1"].value_counts() -st.plotly_chart( - render_pie_chart( - df[filter], - values=finance1_counts, - names=finance1_counts.index, - color_discrete_sequence=None, - color=finance1_counts.index, - color_discrete_map={ - "A great deal of funding": px.colors.qualitative.Prism[9], - "Sufficient funding": px.colors.qualitative.Prism[8], - "Some funding": px.colors.qualitative.Prism[7], - "Little funding": px.colors.qualitative.Prism[6], - "No funding": px.colors.qualitative.Prism[5], - "I don't know": px.colors.qualitative.Prism[10], - "I prefer not to say": px.colors.qualitative.Prism[10], - }, + expertise2_counts = df[filter]["expertise2"].value_counts() + st.plotly_chart( + render_pie_chart( + df[filter], + values=expertise2_counts, + names=expertise2_counts.index, + color_discrete_sequence=None, + color=expertise2_counts.index, + color_discrete_map={ + "Expert knowledge": px.colors.qualitative.Prism[9], + "Advanced knowledge": px.colors.qualitative.Prism[8], + "Some knowledge": px.colors.qualitative.Prism[7], + "Basic knowledge": px.colors.qualitative.Prism[6], + "No knowledge": px.colors.qualitative.Prism[5], + "I don't know": px.colors.qualitative.Prism[10], + "I prefer not to say": px.colors.qualitative.Prism[10], + }, + ) ) -) - -st.write("## Freedom of Information") -st.write( - "### Have you requested information under the national FOI law when you worked on intelligence-related issues over the past 5 years? `[foi1]`" -) -foi1_counts = df[filter]["foi1"].value_counts() -st.plotly_chart( - render_pie_chart( - foi1_counts, - values=foi1_counts, - names=foi1_counts.index, - color=foi1_counts.index, - color_discrete_map={ - "No": px.colors.qualitative.Prism[8], - "Yes": px.colors.qualitative.Prism[2], - "I don't know": px.colors.qualitative.Prism[10], - "I prefer not to say": px.colors.qualitative.Prism[10], - }, + st.write( + "### How do you assess your level of expertise concerning the **political** aspects of surveillance by intelligence agencies?` [expertise3]`" ) -) - -st.write("### How often did you request information? `[foi2]`") -st.plotly_chart( - render_histogram( - df[filter], - x="foi2", - y=None, - nbins=10, - color="country", - color_discrete_map={ - "Germany": px.colors.qualitative.Prism[5], - "France": px.colors.qualitative.Prism[1], - "United Kingdom": px.colors.qualitative.Prism[7], - }, - labels={"foi2": "Number of requests"}, + expertise3_counts = df[filter]["expertise3"].value_counts() + st.plotly_chart( + render_pie_chart( + df[filter], + values=expertise3_counts, + names=expertise3_counts.index, + color_discrete_sequence=None, + color=expertise3_counts.index, + color_discrete_map={ + "Expert knowledge": px.colors.qualitative.Prism[9], + "Advanced knowledge": px.colors.qualitative.Prism[8], + "Some knowledge": px.colors.qualitative.Prism[7], + "Basic knowledge": px.colors.qualitative.Prism[6], + "No knowledge": px.colors.qualitative.Prism[5], + "I don't know": px.colors.qualitative.Prism[10], + "I prefer not to say": px.colors.qualitative.Prism[10], + }, + ) ) -) -st.write( - "### Over the past 5 years, did you receive a response to your FOI request(s) in a timely manner? `[foi3]`" -) -foi3_counts = df[filter]["foi3"].value_counts() -st.plotly_chart( - render_pie_chart( - foi3_counts, - values=foi3_counts, - names=foi3_counts.index, - color=foi3_counts.index, - color_discrete_map={ - "Never": px.colors.qualitative.Prism[9], - "No, usually longer than 30 days": px.colors.qualitative.Prism[8], - "Yes, within 30 days": px.colors.qualitative.Prism[2], - "I don't know": px.colors.qualitative.Prism[10], - "I prefer not to say": px.colors.qualitative.Prism[10], - }, + st.write( + "### How do you assess your level of expertise concerning the **technical** aspects of surveillance by intelligence agencies?` [expertise4]`" + ) + expertise4_counts = df[filter]["expertise4"].value_counts() + st.plotly_chart( + render_pie_chart( + df[filter], + values=expertise4_counts, + names=expertise4_counts.index, + color_discrete_sequence=None, + color=expertise4_counts.index, + color_discrete_map={ + "Expert knowledge": px.colors.qualitative.Prism[9], + "Advanced knowledge": px.colors.qualitative.Prism[8], + "Some knowledge": px.colors.qualitative.Prism[7], + "Basic knowledge": px.colors.qualitative.Prism[6], + "No knowledge": px.colors.qualitative.Prism[5], + "I don't know": px.colors.qualitative.Prism[10], + "I prefer not to say": px.colors.qualitative.Prism[10], + }, + ) ) -) -st.write( - "### How helpful have Freedom of Information requests been for your work on intelligence-related issues? `[foi4]`" -) -protectops2_counts = df[filter]["foi4"].value_counts() + st.write("## Financial Resources") -st.plotly_chart( - render_pie_chart( - df[filter], - values=protectops2_counts, - names=protectops2_counts.index, + st.write( + "### How do you assess the financial resources that have been available for your work on intelligence over the past 5 years? `[finance1]`" ) -) - -st.write( - "### Why haven’t you requested information under the national FOI law when you reported on intelligence-related issues over the past 5 years? `[foi5]`" -) -foi5_df = pd.DataFrame(columns=("option", "count", "country")) -# TODO Map proper labels -for label in [ - "not_aware", - "not_covered", - "too_expensive", - "too_time_consuming", - "afraid_of_data_destruction", - "afraid_of_discrimination", - "other", - "dont_know", - "prefer_not_to_say", -]: - foi5_data = df[filter]["country"][df[f"foi5[{label}]"] == 1].tolist() - for i in foi5_data: - foi5_df = foi5_df.append( - {"option": label, "count": foi5_data.count(i), "country": i}, - ignore_index=True, + finance1_counts = df[filter]["finance1"].value_counts() + st.plotly_chart( + render_pie_chart( + df[filter], + values=finance1_counts, + names=finance1_counts.index, + color_discrete_sequence=None, + color=finance1_counts.index, + color_discrete_map={ + "A great deal of funding": px.colors.qualitative.Prism[9], + "Sufficient funding": px.colors.qualitative.Prism[8], + "Some funding": px.colors.qualitative.Prism[7], + "Little funding": px.colors.qualitative.Prism[6], + "No funding": px.colors.qualitative.Prism[5], + "I don't know": px.colors.qualitative.Prism[10], + "I prefer not to say": px.colors.qualitative.Prism[10], + }, ) -foi5_df = foi5_df.drop_duplicates() -st.plotly_chart( - render_histogram( - foi5_df, - x="option", - y="count", - nbins=None, - color="country", - color_discrete_map={ - "Germany": px.colors.qualitative.Prism[5], - "France": px.colors.qualitative.Prism[1], - "United Kingdom": px.colors.qualitative.Prism[7], - }, - labels={"count": "people who answered 'Yes'"}, ) -) -st.write("### If you selected ‘other’, please specify `[foi5other]`") -for i in df[filter]["foi5other"].to_list(): - if type(i) != float: - st.write("- " + i) + st.write("## Freedom of Information") -st.write("# Protection") + st.write( + "### Have you requested information under the national FOI law when you worked on intelligence-related issues over the past 5 years? `[foi1]`" + ) + foi1_counts = df[filter]["foi1"].value_counts() + st.plotly_chart( + render_pie_chart( + foi1_counts, + values=foi1_counts, + names=foi1_counts.index, + color=foi1_counts.index, + color_discrete_map={ + "No": px.colors.qualitative.Prism[8], + "Yes": px.colors.qualitative.Prism[2], + "I don't know": px.colors.qualitative.Prism[10], + "I prefer not to say": px.colors.qualitative.Prism[10], + }, + ) + ) -st.write("## Operational Protection") + st.write("### How often did you request information? `[foi2]`") + st.plotly_chart( + render_histogram( + df[filter], + x="foi2", + y=None, + nbins=10, + color="country", + color_discrete_map={ + "Germany": px.colors.qualitative.Prism[5], + "France": px.colors.qualitative.Prism[1], + "United Kingdom": px.colors.qualitative.Prism[7], + }, + labels={"foi2": "Number of requests"}, + ) + ) -st.write( - "### Have you taken any of the following measures to protect your datas from attacks and surveillance? `[protectops1]`" -) -protectops1_options = [ - "Participation in digital security training", - "Use of E2E encrypted communication channels", -] + st.plotly_chart( + render_boxplot( + df=df[filter], + points="all", + x="country", + y="foi2", + color="country", + color_discrete_map={ + "Germany": px.colors.qualitative.Prism[5], + "France": px.colors.qualitative.Prism[1], + "United Kingdom": px.colors.qualitative.Prism[7], + }, + ) + ) -protectops1_yes = [] -protectops1_no = [] -protectops1_dont_know = [] -protectops1_prefer_not_to_say = [] -for answer in [ - "Yes", - "No", - "I don't know", - "I prefer not to say", -]: - for label in ["sectraining", "e2e"]: - try: - count = df[filter][f"protectops1[{label}]"].value_counts()[answer] - except KeyError: - count = 0 - if answer == "Yes": - protectops1_yes.append(count) - elif answer == "No": - protectops1_no.append(count) - elif answer == "I don't know": - protectops1_dont_know.append(count) - elif answer == "I prefer not to say": - protectops1_prefer_not_to_say.append(count) - else: - continue - -st.plotly_chart( - generate_stacked_bar_chart( - data=[ - go.Bar( - name="Yes", - x=protectops1_options, - y=protectops1_yes, - marker_color=px.colors.qualitative.Prism[2], - ), - go.Bar( - name="No", - x=protectops1_options, - y=protectops1_no, - marker_color=px.colors.qualitative.Prism[8], - ), - go.Bar( - name="I don't know", - x=protectops1_options, - y=protectops1_dont_know, - marker_color=px.colors.qualitative.Prism[10], - ), - go.Bar( - name="I prefer not to say", - x=protectops1_options, - y=protectops1_prefer_not_to_say, - marker_color=px.colors.qualitative.Prism[10], - ), - ], + st.write( + "### Over the past 5 years, did you receive a response to your FOI request(s) in a timely manner? `[foi3]`" + ) + foi3_counts = df[filter]["foi3"].value_counts() + st.plotly_chart( + render_pie_chart( + foi3_counts, + values=foi3_counts, + names=foi3_counts.index, + color=foi3_counts.index, + color_discrete_map={ + "Never": px.colors.qualitative.Prism[9], + "No, usually longer than 30 days": px.colors.qualitative.Prism[8], + "Yes, within 30 days": px.colors.qualitative.Prism[2], + "I don't know": px.colors.qualitative.Prism[10], + "I prefer not to say": px.colors.qualitative.Prism[10], + }, + ) ) -) -st.write("### Were any of these measures provided by your employer? `[protectops2]`") -protectops2_counts = df[filter]["protectops2"].value_counts() -st.plotly_chart( - render_pie_chart( - df[filter], - values=protectops2_counts, - names=protectops2_counts.index, - color_discrete_sequence=px.colors.qualitative.Prism, + st.write( + "### How helpful have Freedom of Information requests been for your work on intelligence-related issues? `[foi4]`" ) -) + foi4_counts = df[filter]["foi4"].value_counts() -st.write( - "### How important is the use of the following technical tools for you to protect your communications, your online activities and the data you handle? `[protectops3]`" -) -protectops3_options = [ - "Encrypted Email", - "VPN", - "Tor", - "E2E Messengers", - "Encrpyted hardware", - "Two-Factor authentication", - "Other", -] + st.plotly_chart( + render_pie_chart( + df[filter], + values=foi4_counts, + names=foi4_counts.index, + ) + ) -protectops3_very_important = [] -protectops3_somewhat_important = [] -protectops3_important = [] -protectops3_slightly_important = [] -protectops3_not_important = [] -for importance in [ - "Very important", - "Somewhat important", - "Important", - "Slightly important", - "Not important at all", -]: + st.write( + "### Why haven’t you requested information under the national FOI law when you reported on intelligence-related issues over the past 5 years? `[foi5]`" + ) + foi5_df = pd.DataFrame(columns=("option", "count", "country")) + # TODO Map proper labels for label in [ - "encrypted_email", - "vpn", - "tor", - "e2e_chat", - "encrypted_hardware", - "2fa", + "not_aware", + "not_covered", + "too_expensive", + "too_time_consuming", + "afraid_of_data_destruction", + "afraid_of_discrimination", "other", + "dont_know", + "prefer_not_to_say", ]: - try: - count = df[filter][f"protectops3[{label}]"].value_counts()[importance] - except KeyError: - count = 0 - if importance == "Very important": - protectops3_very_important.append(count) - elif importance == "Somewhat important": - protectops3_somewhat_important.append(count) - elif importance == "Important": - protectops3_important.append(count) - elif importance == "Slightly important": - protectops3_slightly_important.append(count) - elif importance == "Not important at all": - protectops3_not_important.append(count) - else: - continue - -st.plotly_chart( - generate_stacked_bar_chart( - data=[ - go.Bar( - name="Very important", - x=protectops3_options, - y=protectops3_very_important, - marker_color="#581845", - ), - go.Bar( - name="Somewhat important", - x=protectops3_options, - y=protectops3_somewhat_important, - marker_color="#900C3F", - ), - go.Bar( - name="Important", - x=protectops3_options, - y=protectops3_important, - marker_color="#C70039", - ), - go.Bar( - name="Slightly important", - x=protectops3_options, - y=protectops3_slightly_important, - marker_color="#FF5733", - ), - go.Bar( - name="Not important at all", - x=protectops3_options, - y=protectops3_not_important, - marker_color="#FFC300", - ), - ], + foi5_data = df[filter]["country"][df[f"foi5[{label}]"] == 1].tolist() + for i in foi5_data: + foi5_df = foi5_df.append( + {"option": label, "count": foi5_data.count(i), "country": i}, + ignore_index=True, + ) + foi5_df = foi5_df.drop_duplicates() + st.plotly_chart( + render_histogram( + foi5_df, + x="option", + y="count", + nbins=None, + color="country", + color_discrete_map={ + "Germany": px.colors.qualitative.Prism[5], + "France": px.colors.qualitative.Prism[1], + "United Kingdom": px.colors.qualitative.Prism[7], + }, + labels={"count": "people who answered 'Yes'"}, + ) ) -) -st.write("### If you selected ‘other’, please specify `[protectops3other]`") -for i in df[filter]["protectops3other"].to_list(): - if type(i) != float: - st.write("- " + i) + st.write("### If you selected ‘other’, please specify `[foi5other]`") + for i in df[filter]["foi5other"].to_list(): + if type(i) != float: + st.write("- " + i) -st.write( - "### Which of the following statements best describes your level of confidence in the protection offered by technological tools? `[protectops4]`" -) -protectops4_counts = df[filter]["protectops4"].value_counts() -st.plotly_chart( - render_pie_chart( - protectops4_counts, - values=protectops4_counts, - names=protectops4_counts.index, - color=protectops4_counts.index, - color_discrete_map={ - "I have full confidence that the right tools
will protect my communication from surveillance": px.colors.qualitative.Prism[ - 4 - ], - "Technological tools help to protect my identity
to some extent, but an attacker with sufficient power
may eventually be able to bypass my technological
safeguards": px.colors.qualitative.Prism[ - 5 - ], - "Under the current conditions of communications
surveillance, technological solutions cannot offer
sufficient protection for the data I handle": px.colors.qualitative.Prism[ - 6 - ], - "I have no confidence in the protection offered by
technological tools": px.colors.qualitative.Prism[ - 7 - ], - "I try to avoid technology-based communication whenever
possible when I work on intelligence-related issues": px.colors.qualitative.Prism[ - 8 +if section == "Protection": + st.write("# Protection") + + st.write("## Operational Protection") + + st.write( + "### Have you taken any of the following measures to protect your datas from attacks and surveillance? `[protectops1]`" + ) + protectops1_options = [ + "Participation in digital security training", + "Use of E2E encrypted communication channels", + ] + + protectops1_yes = [] + protectops1_no = [] + protectops1_dont_know = [] + protectops1_prefer_not_to_say = [] + for answer in [ + "Yes", + "No", + "I don't know", + "I prefer not to say", + ]: + for label in ["sectraining", "e2e"]: + try: + count = df[filter][f"protectops1[{label}]"].value_counts()[answer] + except KeyError: + count = 0 + if answer == "Yes": + protectops1_yes.append(count) + elif answer == "No": + protectops1_no.append(count) + elif answer == "I don't know": + protectops1_dont_know.append(count) + elif answer == "I prefer not to say": + protectops1_prefer_not_to_say.append(count) + else: + continue + + st.plotly_chart( + generate_stacked_bar_chart( + data=[ + go.Bar( + name="Yes", + x=protectops1_options, + y=protectops1_yes, + marker_color=px.colors.qualitative.Prism[2], + ), + go.Bar( + name="No", + x=protectops1_options, + y=protectops1_no, + marker_color=px.colors.qualitative.Prism[8], + ), + go.Bar( + name="I don't know", + x=protectops1_options, + y=protectops1_dont_know, + marker_color=px.colors.qualitative.Prism[10], + ), + go.Bar( + name="I prefer not to say", + x=protectops1_options, + y=protectops1_prefer_not_to_say, + marker_color=px.colors.qualitative.Prism[10], + ), ], - "I don't know": px.colors.qualitative.Prism[10], - "I prefer not to say": px.colors.qualitative.Prism[10], - }, + ) ) -) -st.write("## Legal Protection") + st.write( + "### Were any of these measures provided by your employer? `[protectops2]`" + ) + protectops2_counts = df[filter]["protectops2"].value_counts() + st.plotly_chart( + render_pie_chart( + df[filter], + values=protectops2_counts, + names=protectops2_counts.index, + color_discrete_sequence=px.colors.qualitative.Prism, + ) + ) -# TODO Clarify that in MS it's about source protection (also for protectleg2) -st.write( - "### When working on intelligence-related issues, do you feel you have reason to be concerned about surveillance of your activities `[protectleg1]`" -) + st.write( + "### How important is the use of the following technical tools for you to protect your communications, your online activities and the data you handle? `[protectops3]`" + ) + protectops3_options = [ + "Encrypted Email", + "VPN", + "Tor", + "E2E Messengers", + "Encrpyted hardware", + "Two-Factor authentication", + "Other", + ] -protectleg1_counts = df[filter]["protectleg1"].value_counts() -st.plotly_chart( - render_pie_chart( - df[filter], - values=protectleg1_counts, - names=protectleg1_counts.index, - color=protectleg1_counts.index, - color_discrete_map={ - "Always": px.colors.qualitative.Prism[9], - "Often": px.colors.qualitative.Prism[8], - "Sometimes": px.colors.qualitative.Prism[7], - "Rarely": px.colors.qualitative.Prism[6], - "Never": px.colors.qualitative.Prism[5], - "I don't know": px.colors.qualitative.Prism[10], - "I prefer not to say": px.colors.qualitative.Prism[10], - }, + protectops3_very_important = [] + protectops3_somewhat_important = [] + protectops3_important = [] + protectops3_slightly_important = [] + protectops3_not_important = [] + for importance in [ + "Very important", + "Somewhat important", + "Important", + "Slightly important", + "Not important at all", + ]: + for label in [ + "encrypted_email", + "vpn", + "tor", + "e2e_chat", + "encrypted_hardware", + "2fa", + "other", + ]: + try: + count = df[filter][f"protectops3[{label}]"].value_counts()[importance] + except KeyError: + count = 0 + if importance == "Very important": + protectops3_very_important.append(count) + elif importance == "Somewhat important": + protectops3_somewhat_important.append(count) + elif importance == "Important": + protectops3_important.append(count) + elif importance == "Slightly important": + protectops3_slightly_important.append(count) + elif importance == "Not important at all": + protectops3_not_important.append(count) + else: + continue + + st.plotly_chart( + generate_stacked_bar_chart( + data=[ + go.Bar( + name="Very important", + x=protectops3_options, + y=protectops3_very_important, + marker_color="#581845", + ), + go.Bar( + name="Somewhat important", + x=protectops3_options, + y=protectops3_somewhat_important, + marker_color="#900C3F", + ), + go.Bar( + name="Important", + x=protectops3_options, + y=protectops3_important, + marker_color="#C70039", + ), + go.Bar( + name="Slightly important", + x=protectops3_options, + y=protectops3_slightly_important, + marker_color="#FF5733", + ), + go.Bar( + name="Not important at all", + x=protectops3_options, + y=protectops3_not_important, + marker_color="#FFC300", + ), + ], + ) ) -) -st.write( - "### Do you regard the existing legal protections against surveillance of your activities in your country as a sufficient safeguard for your work on intelligence-related issues? `[protectleg2]`" -) + st.write("### If you selected ‘other’, please specify `[protectops3other]`") + for i in df[filter]["protectops3other"].to_list(): + if type(i) != float: + st.write("- " + i) -protectleg2_counts = df[filter]["protectleg2"].value_counts() -st.plotly_chart( - render_pie_chart( - protectleg2_counts, - values=protectleg2_counts, - names=protectleg2_counts.index, - color_discrete_sequence=px.colors.qualitative.Prism, - color=protectleg2_counts.index, - color_discrete_map={ - "No": px.colors.qualitative.Prism[8], - "Yes": px.colors.qualitative.Prism[2], - "I don't know": px.colors.qualitative.Prism[10], - "I prefer not to say": px.colors.qualitative.Prism[10], - }, + st.write( + "### Which of the following statements best describes your level of confidence in the protection offered by technological tools? `[protectops4]`" + ) + protectops4_counts = df[filter]["protectops4"].value_counts() + st.plotly_chart( + render_pie_chart( + protectops4_counts, + values=protectops4_counts, + names=protectops4_counts.index, + color=protectops4_counts.index, + color_discrete_map={ + "I have full confidence that the right tools
will protect my communication from surveillance": px.colors.qualitative.Prism[ + 4 + ], + "Technological tools help to protect my identity
to some extent, but an attacker with sufficient power
may eventually be able to bypass my technological
safeguards": px.colors.qualitative.Prism[ + 5 + ], + "Under the current conditions of communications
surveillance, technological solutions cannot offer
sufficient protection for the data I handle": px.colors.qualitative.Prism[ + 6 + ], + "I have no confidence in the protection offered by
technological tools": px.colors.qualitative.Prism[ + 7 + ], + "I try to avoid technology-based communication whenever
possible when I work on intelligence-related issues": px.colors.qualitative.Prism[ + 8 + ], + "I don't know": px.colors.qualitative.Prism[10], + "I prefer not to say": px.colors.qualitative.Prism[10], + }, + ) ) -) -st.write("### If you selected ‘no’, please specify `[protectleg2no]`") -for i in df[filter]["protectleg2no"].to_list(): - if type(i) != float: - st.write("- " + i) + st.write("## Legal Protection") -st.write( - "### Are any of the following forms of institutional support readily available to you? `[protectleg3]`" -) -protectleg3_options = ["Free legal counsel", "Legal cost insurance", "Other"] -protectleg3_yes = [] -protectleg3_no = [] -protectleg3_dont_know = [] -protectleg3_prefer_not_to_say = [] -for answer in ["Yes", "No", "I don't know", "I prefer not to say"]: - for label in ["free_counsel", "cost_insurance", "other"]: - try: - count = df[filter][f"protectleg3[{label}]"].value_counts()[answer] - except KeyError: - count = 0 - if answer == "Yes": - protectleg3_yes.append(count) - elif answer == "No": - protectleg3_no.append(count) - elif answer == "I don't know": - protectleg3_dont_know.append(count) - elif answer == "I prefer not to say": - protectleg3_prefer_not_to_say.append(count) - else: - continue -st.plotly_chart( - generate_stacked_bar_chart( - data=[ - go.Bar( - name="Yes", - x=protectleg3_options, - y=protectleg3_yes, - marker_color=px.colors.qualitative.Prism[2], - ), - go.Bar( - name="No", - x=protectleg3_options, - y=protectleg3_no, - marker_color=px.colors.qualitative.Prism[8], - ), - go.Bar( - name="I don't know", - x=protectleg3_options, - y=protectleg3_dont_know, - marker_color="#7f7f7f", - opacity=0.8, - ), - go.Bar( - name="I prefer not to say", - x=protectleg3_options, - y=protectleg3_prefer_not_to_say, - marker_color="#525252", - opacity=0.8, - ), - ], + # TODO Clarify that in MS it's about source protection (also for protectleg2) + st.write( + "### When working on intelligence-related issues, do you feel you have reason to be concerned about surveillance of your activities `[protectleg1]`" ) -) -st.write("### If you selected ‘other’, please specify `[protectleg3other]`") -for i in df[filter]["protectleg3other"].to_list(): - if type(i) != float: - st.write("- " + i) + protectleg1_counts = df[filter]["protectleg1"].value_counts() + st.plotly_chart( + render_pie_chart( + df[filter], + values=protectleg1_counts, + names=protectleg1_counts.index, + color=protectleg1_counts.index, + color_discrete_map={ + "Always": px.colors.qualitative.Prism[9], + "Often": px.colors.qualitative.Prism[8], + "Sometimes": px.colors.qualitative.Prism[7], + "Rarely": px.colors.qualitative.Prism[6], + "Never": px.colors.qualitative.Prism[5], + "I don't know": px.colors.qualitative.Prism[10], + "I prefer not to say": px.colors.qualitative.Prism[10], + }, + ) + ) -st.write("# Constraints") + st.write( + "### Do you regard the existing legal protections against surveillance of your activities in your country as a sufficient safeguard for your work on intelligence-related issues? `[protectleg2]`" + ) -st.write("## Interferences") + protectleg2_counts = df[filter]["protectleg2"].value_counts() + st.plotly_chart( + render_pie_chart( + protectleg2_counts, + values=protectleg2_counts, + names=protectleg2_counts.index, + color_discrete_sequence=px.colors.qualitative.Prism, + color=protectleg2_counts.index, + color_discrete_map={ + "No": px.colors.qualitative.Prism[8], + "Yes": px.colors.qualitative.Prism[2], + "I don't know": px.colors.qualitative.Prism[10], + "I prefer not to say": px.colors.qualitative.Prism[10], + }, + ) + ) -st.write( - "### Has your institution or have you yourself been subjected to surveillance by intelligence agencies in the past five years? `[constraintinter1]`" -) + st.write("### If you selected ‘no’, please specify `[protectleg2no]`") + for i in df[filter]["protectleg2no"].to_list(): + if type(i) != float: + st.write("- " + i) -constraintinter1_counts = df[filter]["constraintinter1"].value_counts() -st.plotly_chart( - render_pie_chart( - df[filter], - values=constraintinter1_counts, - names=constraintinter1_counts.index, - color=constraintinter1_counts.index, - color_discrete_map={ - "No": px.colors.qualitative.Prism[8], - "Yes, I have evidence": px.colors.qualitative.Prism[1], - "Yes, I suspect": px.colors.qualitative.Prism[2], - "I don't know": px.colors.qualitative.Prism[10], - "I prefer not to say": px.colors.qualitative.Prism[10], - }, + st.write( + "### Are any of the following forms of institutional support readily available to you? `[protectleg3]`" + ) + protectleg3_options = ["Free legal counsel", "Legal cost insurance", "Other"] + protectleg3_yes = [] + protectleg3_no = [] + protectleg3_dont_know = [] + protectleg3_prefer_not_to_say = [] + for answer in ["Yes", "No", "I don't know", "I prefer not to say"]: + for label in ["free_counsel", "cost_insurance", "other"]: + try: + count = df[filter][f"protectleg3[{label}]"].value_counts()[answer] + except KeyError: + count = 0 + if answer == "Yes": + protectleg3_yes.append(count) + elif answer == "No": + protectleg3_no.append(count) + elif answer == "I don't know": + protectleg3_dont_know.append(count) + elif answer == "I prefer not to say": + protectleg3_prefer_not_to_say.append(count) + else: + continue + st.plotly_chart( + generate_stacked_bar_chart( + data=[ + go.Bar( + name="Yes", + x=protectleg3_options, + y=protectleg3_yes, + marker_color=px.colors.qualitative.Prism[2], + ), + go.Bar( + name="No", + x=protectleg3_options, + y=protectleg3_no, + marker_color=px.colors.qualitative.Prism[8], + ), + go.Bar( + name="I don't know", + x=protectleg3_options, + y=protectleg3_dont_know, + marker_color="#7f7f7f", + opacity=0.8, + ), + go.Bar( + name="I prefer not to say", + x=protectleg3_options, + y=protectleg3_prefer_not_to_say, + marker_color="#525252", + opacity=0.8, + ), + ], + ) ) -) -st.write( - "### In the past 5 years, have you been threatened with prosecution or have you actually been prosecuted for your work on intelligence-related issues? `[constraintinter2]`" -) + st.write("### If you selected ‘other’, please specify `[protectleg3other]`") + for i in df[filter]["protectleg3other"].to_list(): + if type(i) != float: + st.write("- " + i) -constraintinter2_counts = df[filter]["constraintinter2"].value_counts() -constraintinter2_fig = px.pie( - df[filter], - values=constraintinter2_counts, - names=constraintinter2_counts.index, - color_discrete_sequence=px.colors.qualitative.Prism, -) +if section == "Constraints": + st.write("# Constraints") -st.plotly_chart(constraintinter2_fig) + st.write("## Interference by state authorities") -st.write("### What was the outcome? `[constraintinter3]`") + st.write( + "### Has your institution or have you yourself been subjected to surveillance by intelligence agencies in the past five years? `[constraintinter1]`" + ) -constraintinter3_counts = df[filter]["constraintinter3"].value_counts() -st.plotly_chart( - render_pie_chart( - df[filter], - values=constraintinter3_counts, - names=constraintinter3_counts.index, - color_discrete_sequence=px.colors.qualitative.Prism, + constraintinter1_counts = df[filter]["constraintinter1"].value_counts() + st.plotly_chart( + render_pie_chart( + df[filter], + values=constraintinter1_counts, + names=constraintinter1_counts.index, + color=constraintinter1_counts.index, + color_discrete_map={ + "No": px.colors.qualitative.Prism[8], + "Yes, I have evidence": px.colors.qualitative.Prism[1], + "Yes, I suspect": px.colors.qualitative.Prism[2], + "I don't know": px.colors.qualitative.Prism[10], + "I prefer not to say": px.colors.qualitative.Prism[10], + }, + ) ) -) -st.write( - "### In the past 5 years, have you experienced any of the following interferences by public authorities in relation to your work on intelligence related topics? `[constraintinter4]`" -) -# TODO Map proper labels -constraintinter4_yes = [] -constraintinter4_no = [] -constraintinter4_dont_know = [] -constraintinter4_prefer_not_to_say = [] -for answer in ["Yes", "No", "I don't know", "I prefer not to say"]: - for label in constraintinter4_options: - try: - count = df[filter][f"constraintinter4[{label}]"].value_counts()[answer] - except KeyError: - count = 0 - if answer == "Yes": - constraintinter4_yes.append(count) - elif answer == "No": - constraintinter4_no.append(count) - elif answer == "I don't know": - constraintinter4_dont_know.append(count) - elif answer == "I prefer not to say": - constraintinter4_prefer_not_to_say.append(count) - else: - continue -st.plotly_chart( - generate_stacked_bar_chart( - data=[ - go.Bar( - name="Yes", - x=constraintinter4_options, - y=constraintinter4_yes, - marker_color=px.colors.qualitative.Prism[2], - ), - go.Bar( - name="No", - x=constraintinter4_options, - y=constraintinter4_no, - marker_color=px.colors.qualitative.Prism[8], - ), - go.Bar( - name="I don't know", - x=constraintinter4_options, - y=constraintinter4_dont_know, - marker_color="#7f7f7f", - opacity=0.8, - ), - go.Bar( - name="I prefer not to say", - x=constraintinter4_options, - y=constraintinter4_prefer_not_to_say, - marker_color="#525252", - opacity=0.8, - ), - ], + st.write( + "### In the past 5 years, have you been threatened with prosecution or have you actually been prosecuted for your work on intelligence-related issues? `[constraintinter2]`" ) -) -st.write( - "### In the past 5 years, have you been approached by intelligence officials and received... `[constraintinter5]`" -) -constraintinter5_options = [ - "Unsolicited information", - "Invitations to off-the-record events or meetings", - "Other", -] -constraintinter5_yes = [] -constraintinter5_no = [] -constraintinter5_dont_know = [] -constraintinter5_prefer_not_to_say = [] -for answer in ["Yes", "No", "I don't know", "I prefer not to say"]: - for label in ["unsolicited_information", "invitations", "other"]: - try: - count = df[filter][f"constraintinter5[{label}]"].value_counts()[answer] - except KeyError: - count = 0 - if answer == "Yes": - constraintinter5_yes.append(count) - elif answer == "No": - constraintinter5_no.append(count) - elif answer == "I don't know": - constraintinter5_dont_know.append(count) - elif answer == "I prefer not to say": - constraintinter5_prefer_not_to_say.append(count) - else: - continue -st.plotly_chart( - generate_stacked_bar_chart( - data=[ - go.Bar( - name="Yes", - x=constraintinter5_options, - y=constraintinter5_yes, - marker_color=px.colors.qualitative.Prism[2], - ), - go.Bar( - name="No", - x=constraintinter5_options, - y=constraintinter5_no, - marker_color=px.colors.qualitative.Prism[8], - ), - go.Bar( - name="I don't know", - x=constraintinter5_options, - y=constraintinter5_dont_know, - marker_color="#7f7f7f", - opacity=0.8, - ), - go.Bar( - name="I prefer not to say", - x=constraintinter5_options, - y=constraintinter5_prefer_not_to_say, - marker_color="#525252", - opacity=0.8, - ), - ], + constraintinter2_counts = df[filter]["constraintinter2"].value_counts() + constraintinter2_fig = px.pie( + df[filter], + values=constraintinter2_counts, + names=constraintinter2_counts.index, + color_discrete_sequence=px.colors.qualitative.Prism, ) -) -st.write("### If you selected ‘other’, please specify `[constraintinter5other]`") -for i in df[filter]["constraintinter5other"].to_list(): - if type(i) != float: - st.write("- " + i) + st.plotly_chart(constraintinter2_fig) -st.write( - "### When working on intelligence-related issues have you ever experienced harassment by security agencies or politicians due to your... `[constraintinter6]`" -) -constraintinter6_options = [ - "Gender", - "Ethnicity", - "Political orientation", - "Sexual orientation", - "Religious affiliation", - "Other", -] -constraintinter6_yes = [] -constraintinter6_no = [] -constraintinter6_dont_know = [] -constraintinter6_prefer_not_to_say = [] -for answer in ["Yes", "No", "I don't know", "I prefer not to say"]: - for label in ["gender", "ethnicity", "political", "sexual", "religious", "other"]: - try: - count = df[filter][f"constraintinter6[{label}]"].value_counts()[answer] - except KeyError: - count = 0 - if answer == "Yes": - constraintinter6_yes.append(count) - elif answer == "No": - constraintinter6_no.append(count) - elif answer == "I don't know": - constraintinter6_dont_know.append(count) - elif answer == "I prefer not to say": - constraintinter6_prefer_not_to_say.append(count) - else: - continue -st.plotly_chart( - generate_stacked_bar_chart( - data=[ - go.Bar( - name="Yes", - x=constraintinter6_options, - y=constraintinter6_yes, - marker_color=px.colors.qualitative.Prism[2], - ), - go.Bar( - name="No", - x=constraintinter6_options, - y=constraintinter6_no, - marker_color=px.colors.qualitative.Prism[8], - ), - go.Bar( - name="I don't know", - x=constraintinter6_options, - y=constraintinter6_dont_know, - marker_color="#7f7f7f", - opacity=0.8, - ), - go.Bar( - name="I prefer not to say", - x=constraintinter6_options, - y=constraintinter6_prefer_not_to_say, - marker_color="#525252", - opacity=0.8, - ), - ], + st.write("### What was the outcome? `[constraintinter3]`") + + constraintinter3_counts = df[filter]["constraintinter3"].value_counts() + st.plotly_chart( + render_pie_chart( + df[filter], + values=constraintinter3_counts, + names=constraintinter3_counts.index, + color_discrete_sequence=px.colors.qualitative.Prism, + ) ) -) -st.write("### If you selected ‘other’, please specify `[constraintinter6other]`") -for i in df[filter]["constraintinter6other"].to_list(): - if type(i) != float: - st.write("- " + i) + st.write( + "### In the past 5 years, have you experienced any of the following interferences by public authorities in relation to your work on intelligence related topics? `[constraintinter4]`" + ) + # TODO Map proper labels + constraintinter4_yes = [] + constraintinter4_no = [] + constraintinter4_dont_know = [] + constraintinter4_prefer_not_to_say = [] + for answer in ["Yes", "No", "I don't know", "I prefer not to say"]: + for label in constraintinter4_options: + try: + count = df[filter][f"constraintinter4[{label}]"].value_counts()[answer] + except KeyError: + count = 0 + if answer == "Yes": + constraintinter4_yes.append(count) + elif answer == "No": + constraintinter4_no.append(count) + elif answer == "I don't know": + constraintinter4_dont_know.append(count) + elif answer == "I prefer not to say": + constraintinter4_prefer_not_to_say.append(count) + else: + continue + st.plotly_chart( + generate_stacked_bar_chart( + data=[ + go.Bar( + name="Yes", + x=constraintinter4_options, + y=constraintinter4_yes, + marker_color=px.colors.qualitative.Prism[2], + ), + go.Bar( + name="No", + x=constraintinter4_options, + y=constraintinter4_no, + marker_color=px.colors.qualitative.Prism[8], + ), + go.Bar( + name="I don't know", + x=constraintinter4_options, + y=constraintinter4_dont_know, + marker_color="#7f7f7f", + opacity=0.8, + ), + go.Bar( + name="I prefer not to say", + x=constraintinter4_options, + y=constraintinter4_prefer_not_to_say, + marker_color="#525252", + opacity=0.8, + ), + ], + ) + ) -st.write("# Attitude") + st.write( + "### In the past 5 years, have you been approached by intelligence officials and received... `[constraintinter5]`" + ) + constraintinter5_options = [ + "Unsolicited information", + "Invitations to off-the-record events or meetings", + "Other", + ] + constraintinter5_yes = [] + constraintinter5_no = [] + constraintinter5_dont_know = [] + constraintinter5_prefer_not_to_say = [] + for answer in ["Yes", "No", "I don't know", "I prefer not to say"]: + for label in ["unsolicited_information", "invitations", "other"]: + try: + count = df[filter][f"constraintinter5[{label}]"].value_counts()[answer] + except KeyError: + count = 0 + if answer == "Yes": + constraintinter5_yes.append(count) + elif answer == "No": + constraintinter5_no.append(count) + elif answer == "I don't know": + constraintinter5_dont_know.append(count) + elif answer == "I prefer not to say": + constraintinter5_prefer_not_to_say.append(count) + else: + continue + st.plotly_chart( + generate_stacked_bar_chart( + data=[ + go.Bar( + name="Yes", + x=constraintinter5_options, + y=constraintinter5_yes, + marker_color=px.colors.qualitative.Prism[2], + ), + go.Bar( + name="No", + x=constraintinter5_options, + y=constraintinter5_no, + marker_color=px.colors.qualitative.Prism[8], + ), + go.Bar( + name="I don't know", + x=constraintinter5_options, + y=constraintinter5_dont_know, + marker_color="#7f7f7f", + opacity=0.8, + ), + go.Bar( + name="I prefer not to say", + x=constraintinter5_options, + y=constraintinter5_prefer_not_to_say, + marker_color="#525252", + opacity=0.8, + ), + ], + ) + ) -st.write( - "### The following four statements are about **intelligence agencies**. Please select the statement you most agree with, based on your national context. `[attitude1]`" -) + st.write("### If you selected ‘other’, please specify `[constraintinter5other]`") + for i in df[filter]["constraintinter5other"].to_list(): + if type(i) != float: + st.write("- " + i) -attitude1_counts = df[filter]["attitude1"].value_counts() -st.plotly_chart( - render_pie_chart( - df[filter], - values=attitude1_counts, - names=attitude1_counts.index, - color_discrete_sequence=px.colors.qualitative.Prism, + st.write( + "### When working on intelligence-related issues have you ever experienced harassment by security agencies or politicians due to your... `[constraintinter6]`" + ) + constraintinter6_options = [ + "Gender", + "Ethnicity", + "Political orientation", + "Sexual orientation", + "Religious affiliation", + "Other", + ] + constraintinter6_yes = [] + constraintinter6_no = [] + constraintinter6_dont_know = [] + constraintinter6_prefer_not_to_say = [] + for answer in ["Yes", "No", "I don't know", "I prefer not to say"]: + for label in [ + "gender", + "ethnicity", + "political", + "sexual", + "religious", + "other", + ]: + try: + count = df[filter][f"constraintinter6[{label}]"].value_counts()[answer] + except KeyError: + count = 0 + if answer == "Yes": + constraintinter6_yes.append(count) + elif answer == "No": + constraintinter6_no.append(count) + elif answer == "I don't know": + constraintinter6_dont_know.append(count) + elif answer == "I prefer not to say": + constraintinter6_prefer_not_to_say.append(count) + else: + continue + st.plotly_chart( + generate_stacked_bar_chart( + data=[ + go.Bar( + name="Yes", + x=constraintinter6_options, + y=constraintinter6_yes, + marker_color=px.colors.qualitative.Prism[2], + ), + go.Bar( + name="No", + x=constraintinter6_options, + y=constraintinter6_no, + marker_color=px.colors.qualitative.Prism[8], + ), + go.Bar( + name="I don't know", + x=constraintinter6_options, + y=constraintinter6_dont_know, + marker_color="#7f7f7f", + opacity=0.8, + ), + go.Bar( + name="I prefer not to say", + x=constraintinter6_options, + y=constraintinter6_prefer_not_to_say, + marker_color="#525252", + opacity=0.8, + ), + ], + ) ) -) -# Pie chart attitudes (attitude2) -st.write( - "### The following four statements are about **intelligence oversight**. Please select the statement you most agree with, based on your national context. `[attitude2]`" -) + st.write("### If you selected ‘other’, please specify `[constraintinter6other]`") + for i in df[filter]["constraintinter6other"].to_list(): + if type(i) != float: + st.write("- " + i) -attitude2_counts = df[filter]["attitude2"].value_counts() -st.plotly_chart( - render_pie_chart( - df[filter], - values=attitude2_counts, - names=attitude2_counts.index, - color_discrete_sequence=px.colors.qualitative.Prism, - ) -) +if section == "Attitudes": + st.write("# Attitudes") -# Histogram (attitude3) + st.write( + "### The following four statements are about **intelligence agencies**. Please select the statement you most agree with, based on your national context. `[attitude1]`" + ) -st.write( - "### In your personal view, what are the goals of intelligence oversight? Please select the three goals of oversight you subscribe to the most. `[attitude3]`" -) + attitude1_counts = df[filter]["attitude1"].value_counts() + st.plotly_chart( + render_pie_chart( + df[filter], + values=attitude1_counts, + names=attitude1_counts.index, + color_discrete_sequence=px.colors.qualitative.Prism, + ) + ) -attitude3_options = [ - "rule_of_law", - "civil_liberties", - "effectiveness_of_intel", - "legitimacy_of_intel", - "trust_in_intel", - "critique_of_intel", - "prefer_not_to_say", -] + st.write( + "### The following four statements are about **intelligence oversight**. Please select the statement you most agree with, based on your national context. `[attitude2]`" + ) -attitude3_df = pd.DataFrame(columns=("option", "count", "country")) -for label in attitude3_options: - attitude3_data = df[filter]["country"][df[f"attitude3[{label}]"] == 1].tolist() - for i in attitude3_data: - attitude3_df = attitude3_df.append( - {"option": label, "count": attitude3_data.count(i), "country": i}, - ignore_index=True, + attitude2_counts = df[filter]["attitude2"].value_counts() + st.plotly_chart( + render_pie_chart( + df[filter], + values=attitude2_counts, + names=attitude2_counts.index, + color_discrete_sequence=px.colors.qualitative.Prism, ) -attitude3_df = attitude3_df.drop_duplicates() - -st.plotly_chart( - generate_histogram( - df=attitude3_df, - x="option", - y="count", - nbins=None, - color="country", - color_discrete_map={ - "Germany": px.colors.qualitative.Prism[5], - "France": px.colors.qualitative.Prism[1], - "United Kingdom": px.colors.qualitative.Prism[7], - }, - labels={"count": "people who answered 'Yes'"}, ) -) + st.write( + "### In your personal view, what are the goals of intelligence oversight? Please select the three goals of oversight you subscribe to the most. `[attitude3]`" + ) -scoring = {1: 6, 2: 5, 3: 4, 4: 3, 5: 2, 6: 1} -bodies = [ - "Parliamentary oversight bodies", - "Judicial oversight bodies", - "Independent expert bodies", - "Data protection authorities", - "Audit courts", - "CSOs | The media", -] + # TODO Map proper labels + attitude3_options = [ + "rule_of_law", + "civil_liberties", + "effectiveness_of_intel", + "legitimacy_of_intel", + "trust_in_intel", + "critique_of_intel", + "prefer_not_to_say", + ] -st.write( - "### Which of the following actors do you trust the most to **enable public debate** on surveillance by intelligence agencies? `[attitude4]`" -) -st.plotly_chart(render_ranking_plot("attitude4")) + attitude3_df = pd.DataFrame(columns=("option", "count", "country")) + for label in attitude3_options: + attitude3_data = df[filter]["country"][df[f"attitude3[{label}]"] == 1].tolist() + for i in attitude3_data: + attitude3_df = attitude3_df.append( + {"option": label, "count": attitude3_data.count(i), "country": i}, + ignore_index=True, + ) + attitude3_df = attitude3_df.drop_duplicates() + + st.plotly_chart( + generate_histogram( + df=attitude3_df, + x="option", + y="count", + nbins=None, + color="country", + color_discrete_map={ + "Germany": px.colors.qualitative.Prism[5], + "France": px.colors.qualitative.Prism[1], + "United Kingdom": px.colors.qualitative.Prism[7], + }, + labels={"count": "people who answered 'Yes'"}, + ) + ) -st.write( - "### Which of the following actors do you trust the most to **contest surveillance** by intelligence agencies? `[attitude5]`" -) -st.plotly_chart(render_ranking_plot("attitude5")) + scoring = {1: 6, 2: 5, 3: 4, 4: 3, 5: 2, 6: 1} + bodies = [ + "Parliamentary oversight bodies", + "Judicial oversight bodies", + "Independent expert bodies", + "Data protection authorities", + "Audit courts", + "CSOs | The media", + ] -st.write( - "### Which of the following actors do you trust the most to **enforce compliance** regarding surveillance by intelligence agencies? `[attitude6]`" -) -st.plotly_chart(render_ranking_plot("attitude6")) + st.write( + "### Which of the following actors do you trust the most to **enable public debate** on surveillance by intelligence agencies? `[attitude4]`" + ) + st.plotly_chart(render_ranking_plot("attitude4")) + st.write( + "### Which of the following actors do you trust the most to **contest surveillance** by intelligence agencies? `[attitude5]`" + ) + st.plotly_chart(render_ranking_plot("attitude5")) -############################################################################### -# Appendix -############################################################################### + st.write( + "### Which of the following actors do you trust the most to **enforce compliance** regarding surveillance by intelligence agencies? `[attitude6]`" + ) + st.plotly_chart(render_ranking_plot("attitude6")) -st.write("# Appendix") +if section == "Appendix": + st.write("# Appendix") -st.write("## Raw data") + st.write("## Raw data") -st.write(get_csv_download_link(df, "merged"), unsafe_allow_html=True) -st.write(get_excel_download_link(df, "merged"), unsafe_allow_html=True) + st.write(get_csv_download_link(df, "merged"), unsafe_allow_html=True) + st.write(get_excel_download_link(df, "merged"), unsafe_allow_html=True) -table = st.checkbox("Show data as table") -if table: - st.dataframe(df[filter]) + table = st.checkbox("Show data as table") + if table: + st.dataframe(df[filter]) -st.write("## Correlation Matrix (Phik `φK`)") + st.write("## Correlation Matrix (Phik `φK`)") -st.write( - "Phik (φk) is a new and practical correlation coefficient that works consistently between categorical, ordinal and interval variables, captures non-linear dependency and reverts to the Pearson correlation coefficient in case of a bivariate normal input distribution. There is extensive documentation available [here](https://phik.readthedocs.io/en/latest/index.html)" -) + st.write( + "Phik (φk) is a new and practical correlation coefficient that works consistently between categorical, ordinal and interval variables, captures non-linear dependency and reverts to the Pearson correlation coefficient in case of a bivariate normal input distribution. There is extensive documentation available [here](https://phik.readthedocs.io/en/latest/index.html)" + ) -show_corr = st.checkbox("Show correlation matrix") -if show_corr: - fig_corr = get_corr_matrix(df) - st.plotly_chart(fig_corr, use_container_width=True) + show_corr = st.checkbox("Show correlation matrix") + if show_corr: + fig_corr = get_corr_matrix(df) + st.plotly_chart(fig_corr, use_container_width=True) -st.write("## Significance Matrix") + st.write("## Significance Matrix") -st.markdown( - body="When assessing correlations it is good practise to evaluate both the correlation and the significance of the correlation: a large correlation may be statistically insignificant, and vice versa a small correlation may be very significant. For instance, scipy.stats.pearsonr returns both the pearson correlation and the p-value. Similarly, the phik package offers functionality the calculate a significance matrix. Significance is defined as: " -) -st.markdown( - body="$Z=\Phi^{-1}(1-p); \Phi(z)=\\frac{1}{\\sqrt{2\pi}}\int_{-\infty}^{z} e^{-t^{2}/2}\,dt$" -) + st.markdown( + body="When assessing correlations it is good practise to evaluate both the correlation and the significance of the correlation: a large correlation may be statistically insignificant, and vice versa a small correlation may be very significant. For instance, scipy.stats.pearsonr returns both the pearson correlation and the p-value. Similarly, the phik package offers functionality the calculate a significance matrix. Significance is defined as: " + ) + st.markdown( + body="$Z=\Phi^{-1}(1-p); \Phi(z)=\\frac{1}{\\sqrt{2\pi}}\int_{-\infty}^{z} e^{-t^{2}/2}\,dt$" + ) -show_sig = st.checkbox("Show significance matrix") -if show_sig: - fig_sig = get_significance_matrix(df) - st.plotly_chart(fig_sig, use_container_width=True) + show_sig = st.checkbox("Show significance matrix") + if show_sig: + fig_sig = get_significance_matrix(df) + st.plotly_chart(fig_sig, use_container_width=True)