From c3dccb95df8d41b345ce7c88a95bc52f33ceeb17 Mon Sep 17 00:00:00 2001 From: Matt Date: Tue, 24 Sep 2024 23:00:32 +0200 Subject: [PATCH] update example notebooks --- examples/dice_rolls_ab_testing.ipynb | 94 ++++++---- examples/goals_scored_ab_testing.ipynb | 66 ++++--- examples/session_data_ab_testing.ipynb | 220 +++++++++++++++++------- examples/session_data_manual_pbbs.ipynb | 86 +++++---- examples/waiting_time_ab_testing.ipynb | 26 +-- 5 files changed, 319 insertions(+), 173 deletions(-) diff --git a/examples/dice_rolls_ab_testing.ipynb b/examples/dice_rolls_ab_testing.ipynb index 62d901f..9be3e6b 100644 --- a/examples/dice_rolls_ab_testing.ipynb +++ b/examples/dice_rolls_ab_testing.ipynb @@ -4,7 +4,9 @@ "cell_type": "code", "execution_count": 1, "id": "45ce22be-8ae0-4b0e-bce5-9e9aab6f105f", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "from IPython.core.interactiveshell import InteractiveShell\n", @@ -19,12 +21,14 @@ "cell_type": "code", "execution_count": 2, "id": "f888f299-69a0-4f3c-bd57-af3a59bedba0", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [ { "data": { "text/plain": [ - "Generator(PCG64) at 0x12A84CBA0" + "Generator(PCG64) at 0x132F469E0" ] }, "execution_count": 2, @@ -42,7 +46,9 @@ "cell_type": "code", "execution_count": 3, "id": "24a15d66-d928-432c-beb3-e25e3be10cc0", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "die_A_rolls = list(np.random.choice(values, 1000, p=[1/6, 1/6, 1/6, 1/6, 1/6, 1/6]))\n", @@ -54,7 +60,9 @@ "cell_type": "code", "execution_count": 4, "id": "35989040-af25-4129-9678-de04c0397c32", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "discrete_test = DiscreteDataTest(values)" @@ -64,7 +72,9 @@ "cell_type": "code", "execution_count": 5, "id": "e902885d-7382-42c8-af7f-1d82fba06bb4", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "discrete_test.add_variant_data('A', die_A_rolls)\n", @@ -76,16 +86,18 @@ "cell_type": "code", "execution_count": 6, "id": "a3ee97a1-d48c-407b-b13c-5cfb11e6591f", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [ { "data": { "text/plain": [ - "{'A': {'concentration': [179.0, 158.0, 174.0, 180.0, 154.0, 155.0],\n", + "{'A': {'concentration': [170.0, 188.0, 159.0, 161.0, 168.0, 154.0],\n", " 'prior': [1, 1, 1, 1, 1, 1]},\n", - " 'B': {'concentration': [230.0, 263.0, 119.0, 119.0, 245.0, 224.0],\n", + " 'B': {'concentration': [243.0, 240.0, 129.0, 124.0, 254.0, 210.0],\n", " 'prior': [1, 1, 1, 1, 1, 1]},\n", - " 'C': {'concentration': [124.0, 43.0, 48.0, 102.0, 74.0, 109.0],\n", + " 'C': {'concentration': [93.0, 53.0, 50.0, 100.0, 105.0, 99.0],\n", " 'prior': [1, 1, 1, 1, 1, 1]}}" ] }, @@ -102,12 +114,14 @@ "cell_type": "code", "execution_count": 7, "id": "23484578-dc84-4325-9aa0-7a1498ee161b", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [ { "data": { "text/plain": [ - "{'A': 0.05845, 'B': 0.1185, 'C': 0.82305}" + "{'A': 0.0006, 'B': 0.00115, 'C': 0.99825}" ] }, "execution_count": 7, @@ -123,12 +137,14 @@ "cell_type": "code", "execution_count": 8, "id": "7001814b-7705-420a-813d-b65393e68288", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [ { "data": { "text/plain": [ - "{'A': 0.1433081, 'B': 0.1150933, 'C': 0.009188}" + "{'A': 0.3012179, 'B': 0.2862335, 'C': 6.09e-05}" ] }, "execution_count": 8, @@ -144,21 +160,23 @@ "cell_type": "code", "execution_count": 9, "id": "86cb2b3b-cc93-489f-ae1d-7becac229c33", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "+-----------+--------------------------------------------------------------+-----------------+-------------------+-----------------+\n", - "| variant | concentration | average_value | prob_being_best | expected_loss |\n", - "+===========+==============================================================+=================+===================+=================+\n", - "| A | {1: 179.0, 2: 158.0, 3: 174.0, 4: 180.0, 5: 154.0, 6: 155.0} | 3.437 | 0.05865 | 0.141573 |\n", - "+-----------+--------------------------------------------------------------+-----------------+-------------------+-----------------+\n", - "| B | {1: 230.0, 2: 263.0, 3: 119.0, 4: 119.0, 5: 245.0, 6: 224.0} | 3.465 | 0.12065 | 0.114587 |\n", - "+-----------+--------------------------------------------------------------+-----------------+-------------------+-----------------+\n", - "| C | {1: 124.0, 2: 43.0, 3: 48.0, 4: 102.0, 5: 74.0, 6: 109.0} | 3.572 | 0.8207 | 0.0092546 |\n", - "+-----------+--------------------------------------------------------------+-----------------+-------------------+-----------------+\n" + "+-----------+--------------------------------------------------------------+-----------------+------------------+-------------------+-----------------+------------------------+\n", + "| variant | concentration | average_value | posterior_mean | prob_being_best | expected_loss | credible_interval |\n", + "+===========+==============================================================+=================+==================+===================+=================+========================+\n", + "| A | {1: 170.0, 2: 188.0, 3: 159.0, 4: 161.0, 5: 168.0, 6: 154.0} | 3.431 | 3.43141 | 0.00065 | 0.301061 | [3.3254478, 3.5351591] |\n", + "+-----------+--------------------------------------------------------------+-----------------+------------------+-------------------+-----------------+------------------------+\n", + "| B | {1: 243.0, 2: 240.0, 3: 129.0, 4: 124.0, 5: 254.0, 6: 210.0} | 3.44667 | 3.44693 | 0.0011 | 0.285589 | [3.3445751, 3.5485946] |\n", + "+-----------+--------------------------------------------------------------+-----------------+------------------+-------------------+-----------------+------------------------+\n", + "| C | {1: 93.0, 2: 53.0, 3: 50.0, 4: 100.0, 5: 105.0, 6: 99.0} | 3.736 | 3.7332 | 0.99825 | 6.19e-05 | [3.575729, 3.8844363] |\n", + "+-----------+--------------------------------------------------------------+-----------------+------------------+-------------------+-----------------+------------------------+\n" ] } ], @@ -171,12 +189,14 @@ "cell_type": "code", "execution_count": 10, "id": "616e35ba-26d3-4d10-ad65-4dc37e5771a6", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [ { "data": { "text/plain": [ - "{'A': 0.6135, 'B': 0.33145, 'C': 0.05505}" + "{'A': 0.5752, 'B': 0.42465, 'C': 0.00015}" ] }, "execution_count": 10, @@ -186,7 +206,7 @@ { "data": { "text/plain": [ - "{'A': 0.0202774, 'B': 0.0484923, 'C': 0.1543976}" + "{'A': 0.0233034, 'B': 0.0382878, 'C': 0.3244604}" ] }, "execution_count": 10, @@ -197,15 +217,15 @@ "name": "stdout", "output_type": "stream", "text": [ - "+-----------+--------------------------------------------------------------+-----------------+-------------------+-----------------+\n", - "| variant | concentration | average_value | prob_being_best | expected_loss |\n", - "+===========+==============================================================+=================+===================+=================+\n", - "| A | {1: 179.0, 2: 158.0, 3: 174.0, 4: 180.0, 5: 154.0, 6: 155.0} | 3.437 | 0.6119 | 0.0203846 |\n", - "+-----------+--------------------------------------------------------------+-----------------+-------------------+-----------------+\n", - "| B | {1: 230.0, 2: 263.0, 3: 119.0, 4: 119.0, 5: 245.0, 6: 224.0} | 3.465 | 0.3346 | 0.048674 |\n", - "+-----------+--------------------------------------------------------------+-----------------+-------------------+-----------------+\n", - "| C | {1: 124.0, 2: 43.0, 3: 48.0, 4: 102.0, 5: 74.0, 6: 109.0} | 3.572 | 0.0535 | 0.154126 |\n", - "+-----------+--------------------------------------------------------------+-----------------+-------------------+-----------------+\n" + "+-----------+--------------------------------------------------------------+-----------------+------------------+-------------------+-----------------+------------------------+\n", + "| variant | concentration | average_value | posterior_mean | prob_being_best | expected_loss | credible_interval |\n", + "+===========+==============================================================+=================+==================+===================+=================+========================+\n", + "| A | {1: 170.0, 2: 188.0, 3: 159.0, 4: 161.0, 5: 168.0, 6: 154.0} | 3.431 | 3.43141 | 0.5783 | 0.0233298 | [3.3252788, 3.5381138] |\n", + "+-----------+--------------------------------------------------------------+-----------------+------------------+-------------------+-----------------+------------------------+\n", + "| B | {1: 243.0, 2: 240.0, 3: 129.0, 4: 124.0, 5: 254.0, 6: 210.0} | 3.44667 | 3.44693 | 0.42165 | 0.0381169 | [3.3426174, 3.5499728] |\n", + "+-----------+--------------------------------------------------------------+-----------------+------------------+-------------------+-----------------+------------------------+\n", + "| C | {1: 93.0, 2: 53.0, 3: 50.0, 4: 100.0, 5: 105.0, 6: 99.0} | 3.736 | 3.7332 | 5e-05 | 0.325504 | [3.5781064, 3.8875444] |\n", + "+-----------+--------------------------------------------------------------+-----------------+------------------+-------------------+-----------------+------------------------+\n" ] } ], @@ -242,7 +262,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.5" + "version": "3.8.17" } }, "nbformat": 4, diff --git a/examples/goals_scored_ab_testing.ipynb b/examples/goals_scored_ab_testing.ipynb index fffbf6b..1122349 100644 --- a/examples/goals_scored_ab_testing.ipynb +++ b/examples/goals_scored_ab_testing.ipynb @@ -4,7 +4,9 @@ "cell_type": "code", "execution_count": 1, "id": "c5f8cedc-94d6-4805-90d4-466d4de6b293", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "from IPython.core.interactiveshell import InteractiveShell\n", @@ -19,12 +21,14 @@ "cell_type": "code", "execution_count": 2, "id": "8e57546e-4b90-4c89-8668-aafe4aff6485", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [ { "data": { "text/plain": [ - "Generator(PCG64) at 0x1283A4BA0" + "Generator(PCG64) at 0x132C389E0" ] }, "execution_count": 2, @@ -40,7 +44,9 @@ "cell_type": "code", "execution_count": 3, "id": "1cc3f939-43a0-4d19-af63-9ae632861dee", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "# goals scored - more is better (duh...)\n", @@ -59,7 +65,9 @@ "cell_type": "code", "execution_count": 4, "id": "fe532f01-6c91-4462-9213-e33379be1f9e", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "# Poisson test for \"goals for\"\n", @@ -82,21 +90,23 @@ "cell_type": "code", "execution_count": 5, "id": "93cd6353-01c6-4873-a62e-9816932679fe", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "+-----------+----------+--------------+--------------------+------------------+-------------------+-----------------+\n", - "| variant | totals | sum_values | observed_average | posterior_mean | prob_being_best | expected_loss |\n", - "+===========+==========+==============+====================+==================+===================+=================+\n", - "| psg | 15 | 43 | 2.86667 | 2.8543 | 0.24375 | 0.509445 |\n", - "+-----------+----------+--------------+--------------------+------------------+-------------------+-----------------+\n", - "| city | 14 | 40 | 2.85714 | 2.5 | 0.05095 | 0.852689 |\n", - "+-----------+----------+--------------+--------------------+------------------+-------------------+-----------------+\n", - "| bayern | 15 | 49 | 3.26667 | 3.25166 | 0.7053 | 0.111539 |\n", - "+-----------+----------+--------------+--------------------+------------------+-------------------+-----------------+\n" + "+-----------+----------+--------------+--------------------+------------------+-------------------+-----------------+------------------------+\n", + "| variant | totals | sum_values | observed_average | posterior_mean | prob_being_best | expected_loss | credible_interval |\n", + "+===========+==========+==============+====================+==================+===================+=================+========================+\n", + "| psg | 15 | 43 | 2.86667 | 2.8543 | 0.2484 | 0.507051 | [2.071789, 3.7911295] |\n", + "+-----------+----------+--------------+--------------------+------------------+-------------------+-----------------+------------------------+\n", + "| city | 14 | 40 | 2.85714 | 2.5 | 0.04775 | 0.866365 | [1.9051119, 3.1609537] |\n", + "+-----------+----------+--------------+--------------------+------------------+-------------------+-----------------+------------------------+\n", + "| bayern | 15 | 49 | 3.26667 | 3.25166 | 0.70385 | 0.112843 | [2.4082776, 4.2095072] |\n", + "+-----------+----------+--------------+--------------------+------------------+-------------------+-----------------+------------------------+\n" ] } ], @@ -111,28 +121,30 @@ "cell_type": "code", "execution_count": 6, "id": "39220217-6553-4f88-b537-064ade561996", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "+-----------+----------+--------------+--------------------+------------------+-------------------+-----------------+\n", - "| variant | totals | sum_values | observed_average | posterior_mean | prob_being_best | expected_loss |\n", - "+===========+==========+==============+====================+==================+===================+=================+\n", - "| psg | 15 | 9 | 0.6 | 0.60265 | 0.7599 | 0.0411113 |\n", - "+-----------+----------+--------------+--------------------+------------------+-------------------+-----------------+\n", - "| city | 14 | 14 | 1 | 1 | 0.07465 | 0.439372 |\n", - "+-----------+----------+--------------+--------------------+------------------+-------------------+-----------------+\n", - "| bayern | 15 | 13 | 0.86667 | 0.86755 | 0.16545 | 0.30863 |\n", - "+-----------+----------+--------------+--------------------+------------------+-------------------+-----------------+\n" + "+-----------+----------+--------------+--------------------+------------------+-------------------+-----------------+------------------------+\n", + "| variant | totals | sum_values | observed_average | posterior_mean | prob_being_best | expected_loss | credible_interval |\n", + "+===========+==========+==============+====================+==================+===================+=================+========================+\n", + "| psg | 15 | 9 | 0.6 | 0.60265 | 0.7468 | 0.0444494 | [0.2171908, 1.2237673] |\n", + "+-----------+----------+--------------+--------------------+------------------+-------------------+-----------------+------------------------+\n", + "| city | 14 | 14 | 1 | 1 | 0.0778 | 0.44033 | [0.4371844, 1.8322232] |\n", + "+-----------+----------+--------------+--------------------+------------------+-------------------+-----------------+------------------------+\n", + "| bayern | 15 | 13 | 0.86667 | 0.86755 | 0.1754 | 0.304051 | [0.3759632, 1.6091403] |\n", + "+-----------+----------+--------------+--------------------+------------------+-------------------+-----------------+------------------------+\n" ] } ], "source": [ "# poisson_test_ga.probabs_of_being_best(sim_count = 20000, seed=52, min_is_best=True)\n", "# poisson_test_ga.expected_loss(sim_count = 20000, seed=52, min_is_best=True)\n", - "results_ga = poisson_test_ga.evaluate(min_is_best=True)\n", + "results_ga = poisson_test_ga.evaluate(min_is_best=True, interval_alpha=0.99)\n", "print(pd.DataFrame(results_ga).to_markdown(tablefmt=\"grid\", index=False))" ] }, @@ -161,7 +173,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.5" + "version": "3.8.17" } }, "nbformat": 4, diff --git a/examples/session_data_ab_testing.ipynb b/examples/session_data_ab_testing.ipynb index 6a57ddb..2302d24 100644 --- a/examples/session_data_ab_testing.ipynb +++ b/examples/session_data_ab_testing.ipynb @@ -4,7 +4,9 @@ "cell_type": "code", "execution_count": 1, "id": "509246a9-8b41-477a-ac9c-2d77b7836446", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "from IPython.core.interactiveshell import InteractiveShell\n", @@ -19,7 +21,9 @@ "cell_type": "code", "execution_count": 2, "id": "d8f566ec-e3e4-4a14-9146-e99710574cf1", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "df = pd.read_csv(\"data/session_data.csv\")" @@ -29,7 +33,9 @@ "cell_type": "code", "execution_count": 3, "id": "4d55cc08-62af-42be-a27c-991bea1bbcfd", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [ { "data": { @@ -138,7 +144,9 @@ "cell_type": "code", "execution_count": 4, "id": "3b465a33-b51d-49bb-9826-9d7e4e60a997", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [ { "data": { @@ -255,7 +263,9 @@ "cell_type": "code", "execution_count": 5, "id": "62a41343-2708-4956-836b-5cb3d726c784", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "conv_test = BinaryDataTest()" @@ -265,7 +275,9 @@ "cell_type": "code", "execution_count": 6, "id": "30174edb-8abb-427f-be28-9d7f3e0d19d2", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [ { "data": { @@ -301,7 +313,9 @@ "cell_type": "code", "execution_count": 7, "id": "c8360b99-e469-4e4b-8a60-ced4e2bf6f53", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "conv_test.add_variant_data(\"A\", variant_A)" @@ -311,7 +325,9 @@ "cell_type": "code", "execution_count": 8, "id": "8ecf0c2f-6661-4027-afc0-73fb08d893f0", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "conv_test.add_variant_data(\"B\", variant_B)" @@ -321,7 +337,9 @@ "cell_type": "code", "execution_count": 9, "id": "9df665b5-0983-472b-83fb-3814fd6e95d8", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "conv_test.add_variant_data_agg(\"C\", len(variant_C), sum(variant_C))" @@ -331,7 +349,9 @@ "cell_type": "code", "execution_count": 10, "id": "b016a4ae-a48c-413a-99a9-a9e6832dee96", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [ { "data": { @@ -387,7 +407,9 @@ "cell_type": "code", "execution_count": 11, "id": "69af1bd6-5317-42af-bb78-1f9eec27f058", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [ { "data": { @@ -408,7 +430,9 @@ "cell_type": "code", "execution_count": 12, "id": "ec654e7d-990d-4cfa-8b1c-324ce054ea2f", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [ { "data": { @@ -429,21 +453,23 @@ "cell_type": "code", "execution_count": 13, "id": "a253d8a4-112b-42ee-b324-8ffa58d0d31e", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "+-----------+----------+-------------+-----------------+------------------+-------------------+-----------------+\n", - "| variant | totals | positives | positive_rate | posterior_mean | prob_being_best | expected_loss |\n", - "+===========+==========+=============+=================+==================+===================+=================+\n", - "| A | 31500 | 1580 | 0.05016 | 0.05017 | 0.04055 | 0.0030061 |\n", - "+-----------+----------+-------------+-----------------+------------------+-------------------+-----------------+\n", - "| B | 32000 | 1700 | 0.05312 | 0.05314 | 0.9262 | 5.46e-05 |\n", - "+-----------+----------+-------------+-----------------+------------------+-------------------+-----------------+\n", - "| C | 31000 | 1550 | 0.05 | 0.05001 | 0.03325 | 0.0031814 |\n", - "+-----------+----------+-------------+-----------------+------------------+-------------------+-----------------+\n" + "+-----------+----------+-------------+-----------------+------------------+-------------------+-----------------+------------------------+\n", + "| variant | totals | positives | positive_rate | posterior_mean | prob_being_best | expected_loss | credible_interval |\n", + "+===========+==========+=============+=================+==================+===================+=================+========================+\n", + "| A | 31500 | 1580 | 0.05016 | 0.05017 | 0.04215 | 0.0030266 | [0.0477713, 0.0526109] |\n", + "+-----------+----------+-------------+-----------------+------------------+-------------------+-----------------+------------------------+\n", + "| B | 32000 | 1700 | 0.05312 | 0.05314 | 0.9235 | 5.64e-05 | [0.0506872, 0.0556353] |\n", + "+-----------+----------+-------------+-----------------+------------------+-------------------+-----------------+------------------------+\n", + "| C | 31000 | 1550 | 0.05 | 0.05001 | 0.03435 | 0.0031789 | [0.0476514, 0.0525011] |\n", + "+-----------+----------+-------------+-----------------+------------------+-------------------+-----------------+------------------------+\n" ] } ], @@ -456,7 +482,9 @@ "cell_type": "code", "execution_count": 14, "id": "e953f4af-9205-4290-9e62-f0f4fa063935", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [ { "data": { @@ -477,12 +505,25 @@ "execution_count": 14, "metadata": {}, "output_type": "execute_result" + }, + { + "data": { + "text/plain": [ + "{'A': [0.0477826, 0.0526302],\n", + " 'B': [0.0506933, 0.0555936],\n", + " 'C': [0.0476604, 0.0524757]}" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ "#reversed_probabilities (for case when \"best = min\")\n", "conv_test.probabs_of_being_best(sim_count = 20000, seed=52, min_is_best=True)\n", - "conv_test.expected_loss(sim_count = 20000, seed=52, min_is_best=True)" + "conv_test.expected_loss(sim_count = 20000, seed=52, min_is_best=True)\n", + "conv_test.credible_intervals(sim_count = 20000, seed=52, min_is_best=True)" ] }, { @@ -497,7 +538,9 @@ "cell_type": "code", "execution_count": 15, "id": "dd8f2d91-504a-47bd-8eee-5daedd0432d9", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "rev_test = DeltaLognormalDataTest()" @@ -507,7 +550,9 @@ "cell_type": "code", "execution_count": 16, "id": "6be09a5b-fc28-4e66-88c0-4bfec41997db", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [ { "data": { @@ -543,7 +588,9 @@ "cell_type": "code", "execution_count": 17, "id": "ea77ff1e-da7a-419d-85a8-bd5dc286a02e", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "rev_test.add_variant_data(\"A\", variant_A)\n", @@ -554,7 +601,9 @@ "cell_type": "code", "execution_count": 18, "id": "0d2ea211-47eb-4545-99c8-8496870ed07d", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "# adding variant using aggregated data\n", @@ -572,7 +621,9 @@ "cell_type": "code", "execution_count": 19, "id": "0260cbf3-d7d4-4cca-a1e9-4b67e228d530", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [ { "data": { @@ -649,12 +700,14 @@ "cell_type": "code", "execution_count": 20, "id": "d5b4a760-0750-4f27-9316-cb7ab8ff6a1d", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [ { "data": { "text/plain": [ - "{'A': 0.0002, 'B': 0.03435, 'C': 0.96545}" + "{'A': 0.0004, 'B': 0.0323, 'C': 0.9673}" ] }, "execution_count": 20, @@ -670,12 +723,14 @@ "cell_type": "code", "execution_count": 21, "id": "40dbcf56-05fa-4a58-bc15-853b08a7125e", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [ { "data": { "text/plain": [ - "{'A': 0.2202791, 'B': 0.1206026, 'C': 0.0008934}" + "{'A': 0.2212141, 'B': 0.1209395, 'C': 0.0007548}" ] }, "execution_count": 21, @@ -690,22 +745,49 @@ { "cell_type": "code", "execution_count": 22, + "id": "c6d7cc7d-fb1d-4645-8582-d3ff7e339752", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'A': [0.9077767, 1.0674761],\n", + " 'B': [1.0035792, 1.1694117],\n", + " 'C': [1.1079753, 1.3061034]}" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "rev_test.credible_intervals()" + ] + }, + { + "cell_type": "code", + "execution_count": 23, "id": "14380bf8-d85b-41b5-a854-98eea12af845", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "+-----------+----------+-------------+--------------+--------------+-----------------------+-------------------+-----------------+\n", - "| variant | totals | positives | sum_values | avg_values | avg_positive_values | prob_being_best | expected_loss |\n", - "+===========+==========+=============+==============+==============+=======================+===================+=================+\n", - "| A | 31500 | 1580 | 30830 | 0.97873 | 19.5127 | 0.0001 | 0.220317 |\n", - "+-----------+----------+-------------+--------------+--------------+-----------------------+-------------------+-----------------+\n", - "| B | 32000 | 1700 | 35203.2 | 1.1001 | 20.7078 | 0.03465 | 0.120436 |\n", - "+-----------+----------+-------------+--------------+--------------+-----------------------+-------------------+-----------------+\n", - "| C | 31000 | 1550 | 37259.6 | 1.20192 | 24.0384 | 0.96525 | 0.0009299 |\n", - "+-----------+----------+-------------+--------------+--------------+-----------------------+-------------------+-----------------+\n" + "+-----------+----------+-------------+--------------+--------------+-----------------------+------------------+-------------------+-----------------+------------------------+\n", + "| variant | totals | positives | sum_values | avg_values | avg_positive_values | posterior_mean | prob_being_best | expected_loss | credible_interval |\n", + "+===========+==========+=============+==============+==============+=======================+==================+===================+=================+========================+\n", + "| A | 31500 | 1580 | 30830 | 0.97873 | 19.5127 | 0.98278 | 0.00015 | 0.220622 | [0.9073023, 1.0655237] |\n", + "+-----------+----------+-------------+--------------+--------------+-----------------------+------------------+-------------------+-----------------+------------------------+\n", + "| B | 32000 | 1700 | 35203.2 | 1.1001 | 20.7078 | 1.08266 | 0.0343 | 0.120709 | [1.0036959, 1.1703548] |\n", + "+-----------+----------+-------------+--------------+--------------+-----------------------+------------------+-------------------+-----------------+------------------------+\n", + "| C | 31000 | 1550 | 37259.6 | 1.20192 | 24.0384 | 1.20208 | 0.96555 | 0.0008776 | [1.1098911, 1.3051817] |\n", + "+-----------+----------+-------------+--------------+--------------+-----------------------+------------------+-------------------+-----------------+------------------------+\n" ] } ], @@ -724,9 +806,11 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 24, "id": "df26b082-0e3f-469d-882f-c391e4aeabad", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "# using time data we can generate probabilities of being best over time (cumulatively)\n", @@ -759,23 +843,25 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 25, "id": "a8c922d6-1283-4034-b574-ea9f5cb80cc5", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [ { "data": { "text/plain": [ - "[{'2021-08-01': {'A': 0.69915, 'B': 0.18295, 'C': 0.1179}},\n", - " {'2021-08-02': {'A': 0.3723, 'B': 0.2251, 'C': 0.4026}},\n", - " {'2021-08-03': {'A': 0.19765, 'B': 0.6473, 'C': 0.15505}},\n", - " {'2021-08-04': {'A': 0.16155, 'B': 0.74055, 'C': 0.0979}},\n", - " {'2021-08-05': {'A': 0.03095, 'B': 0.87455, 'C': 0.0945}},\n", - " {'2021-08-06': {'A': 0.0154, 'B': 0.9709, 'C': 0.0137}},\n", - " {'2021-08-07': {'A': 0.04175, 'B': 0.92305, 'C': 0.0352}}]" + "[{'2021-08-01': {'A': 0.69255, 'B': 0.18095, 'C': 0.1265}},\n", + " {'2021-08-02': {'A': 0.3707, 'B': 0.22545, 'C': 0.40385}},\n", + " {'2021-08-03': {'A': 0.19215, 'B': 0.64585, 'C': 0.162}},\n", + " {'2021-08-04': {'A': 0.16005, 'B': 0.7432, 'C': 0.09675}},\n", + " {'2021-08-05': {'A': 0.0311, 'B': 0.87515, 'C': 0.09375}},\n", + " {'2021-08-06': {'A': 0.014, 'B': 0.9724, 'C': 0.0136}},\n", + " {'2021-08-07': {'A': 0.03985, 'B': 0.92765, 'C': 0.0325}}]" ] }, - "execution_count": 24, + "execution_count": 25, "metadata": {}, "output_type": "execute_result" } @@ -786,23 +872,25 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 26, "id": "50e20fcd-c2b7-49af-be61-c27775706090", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [ { "data": { "text/plain": [ - "[{'2021-08-01': {'A': 0.0229, 'B': 0.2647, 'C': 0.7124}},\n", - " {'2021-08-02': {'A': 0.0566, 'B': 0.20685, 'C': 0.73655}},\n", - " {'2021-08-03': {'A': 0.0647, 'B': 0.20735, 'C': 0.72795}},\n", - " {'2021-08-04': {'A': 0.0214, 'B': 0.17795, 'C': 0.80065}},\n", - " {'2021-08-05': {'A': 0.00045, 'B': 0.04405, 'C': 0.9555}},\n", - " {'2021-08-06': {'A': 0.0003, 'B': 0.04835, 'C': 0.95135}},\n", - " {'2021-08-07': {'A': 0.00015, 'B': 0.0354, 'C': 0.96445}}]" + "[{'2021-08-01': {'A': 0.0249, 'B': 0.2654, 'C': 0.7097}},\n", + " {'2021-08-02': {'A': 0.05285, 'B': 0.20995, 'C': 0.7372}},\n", + " {'2021-08-03': {'A': 0.06495, 'B': 0.21025, 'C': 0.7248}},\n", + " {'2021-08-04': {'A': 0.02065, 'B': 0.1754, 'C': 0.80395}},\n", + " {'2021-08-05': {'A': 0.0006, 'B': 0.04585, 'C': 0.95355}},\n", + " {'2021-08-06': {'A': 0.0001, 'B': 0.0478, 'C': 0.9521}},\n", + " {'2021-08-07': {'A': 0.0002, 'B': 0.03195, 'C': 0.96785}}]" ] }, - "execution_count": 25, + "execution_count": 26, "metadata": {}, "output_type": "execute_result" } @@ -836,7 +924,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.5" + "version": "3.8.17" } }, "nbformat": 4, diff --git a/examples/session_data_manual_pbbs.ipynb b/examples/session_data_manual_pbbs.ipynb index f3fb79c..9aec57d 100644 --- a/examples/session_data_manual_pbbs.ipynb +++ b/examples/session_data_manual_pbbs.ipynb @@ -4,7 +4,9 @@ "cell_type": "code", "execution_count": 1, "id": "8b11e1e0-ccc4-4fc9-9cdd-9f906e64b1c7", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "from IPython.core.interactiveshell import InteractiveShell\n", @@ -19,7 +21,9 @@ "cell_type": "code", "execution_count": 2, "id": "b31da712-cdb9-4671-b3ed-63e351896915", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "df = pd.read_csv(\"data/session_data.csv\")" @@ -29,7 +33,9 @@ "cell_type": "code", "execution_count": 3, "id": "6032fc6e-3a4c-47f3-830f-1a85d49c253c", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [ { "data": { @@ -138,7 +144,9 @@ "cell_type": "code", "execution_count": 4, "id": "744e5833-cbc3-45d3-963d-11c2a92acff2", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [ { "data": { @@ -247,7 +255,9 @@ "cell_type": "code", "execution_count": 5, "id": "be57dc82-8958-4118-aab0-71122490d17a", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [ { "data": { @@ -337,44 +347,52 @@ "cell_type": "code", "execution_count": 6, "id": "3ade0625-fb50-434f-93f6-e70c3c543713", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Probabilities of being best: [0.0417, 0.92595, 0.03235]\n", - "Expected loss: [0.0030295, 5.28e-05, 0.0031848]\n" + "Probabilities of being best: [0.04215, 0.92285, 0.035]\n", + "Expected loss: [0.0030094, 5.86e-05, 0.0032051]\n", + "95% credible intervals: [[0.0478136, 0.0526138], [0.0506898, 0.0555755], [0.0475854, 0.0524707]]\n" ] } ], "source": [ - "# conversion rate probabilities of being best and expected loss for each variant\n", - "pbbs, loss = eval_bernoulli_agg(sessions, conversions)\n", + "# conversion rate probabilities of being best, expected loss and credible intervals for each variant\n", + "pbbs, loss, intervals = eval_bernoulli_agg(sessions, conversions)\n", "print(f\"Probabilities of being best: {pbbs}\")\n", - "print(f\"Expected loss: {loss}\")" + "print(f\"Expected loss: {loss}\")\n", + "print(f\"95% credible intervals: {intervals}\")" ] }, { "cell_type": "code", "execution_count": 7, "id": "21c3ae3c-46b3-4bc7-bd33-5306d3e20506", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Probabilities of being best: [0.0001, 0.03275, 0.96715]\n", - "Expected loss: [0.2209108, 0.1210127, 0.0008623]\n" + "Probabilities of being best: [0.00025, 0.0345, 0.96525]\n", + "Expected loss: [0.2205845, 0.1205336, 0.0008507]\n", + "95% credible intervals: [[0.9083484, 1.0669433], [1.0039529, 1.169006], [1.1092165, 1.3053917]]\n" ] } ], "source": [ - "# revenue per session probabilities of being best and expected loss for each variant\n", - "pbbs, loss = eval_delta_lognormal_agg(sessions, conversions, sum_log_revenue, sum_log_2_revenue)\n", + "# revenue per session probabilities of being best, expected loss and credible intervals for each variant\n", + "pbbs, loss, intervals = eval_delta_lognormal_agg(sessions, conversions, sum_log_revenue, sum_log_2_revenue)\n", "print(f\"Probabilities of being best: {pbbs}\")\n", - "print(f\"Expected loss: {loss}\")" + "print(f\"Expected loss: {loss}\")\n", + "print(f\"95% credible intervals: {intervals}\")" ] }, { @@ -389,44 +407,52 @@ "cell_type": "code", "execution_count": 8, "id": "b651c9d9-6d51-4ad1-aabb-475296963a88", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Probabilities of being best: [0.45965, 0.0096, 0.53075]\n", - "Expected loss: [0.0007803, 0.0037304, 0.0006225]\n" + "Probabilities of being best: [0.45225, 0.01065, 0.5371]\n", + "Expected loss: [0.0007916, 0.0037242, 0.000628]\n", + "95% credible intervals: [[0.0477928, 0.0526182], [0.0507342, 0.0555998], [0.0475664, 0.0524791]]\n" ] } ], "source": [ - "# conversion rate probabilities of being best and expected loss for each variant\n", - "pbbs, loss = eval_bernoulli_agg(sessions, conversions, min_is_best=True)\n", + "# conversion rate probabilities of being best, expected loss and credible intervals for each variant\n", + "pbbs, loss, intervals = eval_bernoulli_agg(sessions, conversions, min_is_best=True)\n", "print(f\"Probabilities of being best: {pbbs}\")\n", - "print(f\"Expected loss: {loss}\")" + "print(f\"Expected loss: {loss}\")\n", + "print(f\"95% credible intervals: {intervals}\")" ] }, { "cell_type": "code", "execution_count": 9, "id": "f6573b6b-314e-49de-ac63-c75201eac707", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Probabilities of being best: [0.95765, 0.04225, 0.0001]\n", - "Expected loss: [0.0009819, 0.1012578, 0.2206058]\n" + "Probabilities of being best: [0.9552, 0.04445, 0.00035]\n", + "Expected loss: [0.0010368, 0.1007368, 0.2202043]\n", + "95% credible intervals: [[0.9074834, 1.0659393], [1.0042571, 1.1693355], [1.1088342, 1.3046563]]\n" ] } ], "source": [ - "# revenue per session probabilities of being best and expected loss for each variant\n", - "pbbs, loss = eval_delta_lognormal_agg(sessions, conversions, sum_log_revenue, sum_log_2_revenue, min_is_best=True)\n", + "# revenue per session probabilities of being best, expected loss and credible intervals for each variant\n", + "pbbs, loss, intervals = eval_delta_lognormal_agg(sessions, conversions, sum_log_revenue, sum_log_2_revenue, min_is_best=True)\n", "print(f\"Probabilities of being best: {pbbs}\")\n", - "print(f\"Expected loss: {loss}\")" + "print(f\"Expected loss: {loss}\")\n", + "print(f\"95% credible intervals: {intervals}\")" ] }, { @@ -454,7 +480,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.5" + "version": "3.8.17" } }, "nbformat": 4, diff --git a/examples/waiting_time_ab_testing.ipynb b/examples/waiting_time_ab_testing.ipynb index b54839a..ced344e 100644 --- a/examples/waiting_time_ab_testing.ipynb +++ b/examples/waiting_time_ab_testing.ipynb @@ -28,7 +28,7 @@ { "data": { "text/plain": [ - "Generator(PCG64) at 0x1320F62E0" + "Generator(PCG64) at 0x12F3ABAC0" ] }, "execution_count": 2, @@ -68,7 +68,7 @@ { "data": { "text/plain": [ - "10.249875855930167" + "10.060038908988217" ] }, "execution_count": 4, @@ -78,7 +78,7 @@ { "data": { "text/plain": [ - "10.922064860918766" + "11.718233907638504" ] }, "execution_count": 4, @@ -88,7 +88,7 @@ { "data": { "text/plain": [ - "11.670477447408727" + "10.904717435173904" ] }, "execution_count": 4, @@ -130,15 +130,15 @@ "name": "stdout", "output_type": "stream", "text": [ - "+-----------+----------+--------------+--------------------+------------------+-------------------+-----------------+\n", - "| variant | totals | sum_values | observed_average | posterior_mean | prob_being_best | expected_loss |\n", - "+===========+==========+==============+====================+==================+===================+=================+\n", - "| A | 200 | 2049.98 | 10.2499 | 10.2453 | 0.7083 | 0.167217 |\n", - "+-----------+----------+--------------+--------------------+------------------+-------------------+-----------------+\n", - "| B | 210 | 2293.63 | 10.9221 | 10.9173 | 0.2385 | 0.84777 |\n", - "+-----------+----------+--------------+--------------------+------------------+-------------------+-----------------+\n", - "| C | 220 | 2567.51 | 11.6705 | 11.6656 | 0.0532 | 1.59189 |\n", - "+-----------+----------+--------------+--------------------+------------------+-------------------+-----------------+\n" + "+-----------+----------+--------------+--------------------+------------------+-------------------+-----------------+--------------------------+\n", + "| variant | totals | sum_values | observed_average | posterior_mean | prob_being_best | expected_loss | credible_interval |\n", + "+===========+==========+==============+====================+==================+===================+=================+==========================+\n", + "| A | 200 | 2012.01 | 10.06 | 10.0555 | 0.77115 | 0.131056 | [8.792003, 11.5889343] |\n", + "+-----------+----------+--------------+--------------------+------------------+-------------------+-----------------+--------------------------+\n", + "| B | 210 | 2460.83 | 11.7182 | 11.7131 | 0.03455 | 1.8058 | [10.2801004, 13.4695842] |\n", + "+-----------+----------+--------------+--------------------+------------------+-------------------+-----------------+--------------------------+\n", + "| C | 220 | 2399.04 | 10.9047 | 10.9002 | 0.1943 | 0.977569 | [9.5901115, 12.4876526] |\n", + "+-----------+----------+--------------+--------------------+------------------+-------------------+-----------------+--------------------------+\n" ] } ],