Skip to content

Commit

Permalink
update example notebooks
Browse files Browse the repository at this point in the history
  • Loading branch information
@Matt52
Matt52 committed Sep 24, 2024
1 parent 9b7081c commit c3dccb9
Show file tree
Hide file tree
Showing 5 changed files with 319 additions and 173 deletions.
94 changes: 57 additions & 37 deletions examples/dice_rolls_ab_testing.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,9 @@
"cell_type": "code",
"execution_count": 1,
"id": "45ce22be-8ae0-4b0e-bce5-9e9aab6f105f",
"metadata": {},
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"from IPython.core.interactiveshell import InteractiveShell\n",
Expand All @@ -19,12 +21,14 @@
"cell_type": "code",
"execution_count": 2,
"id": "f888f299-69a0-4f3c-bd57-af3a59bedba0",
"metadata": {},
"metadata": {
"tags": []
},
"outputs": [
{
"data": {
"text/plain": [
"Generator(PCG64) at 0x12A84CBA0"
"Generator(PCG64) at 0x132F469E0"
]
},
"execution_count": 2,
Expand All @@ -42,7 +46,9 @@
"cell_type": "code",
"execution_count": 3,
"id": "24a15d66-d928-432c-beb3-e25e3be10cc0",
"metadata": {},
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"die_A_rolls = list(np.random.choice(values, 1000, p=[1/6, 1/6, 1/6, 1/6, 1/6, 1/6]))\n",
Expand All @@ -54,7 +60,9 @@
"cell_type": "code",
"execution_count": 4,
"id": "35989040-af25-4129-9678-de04c0397c32",
"metadata": {},
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"discrete_test = DiscreteDataTest(values)"
Expand All @@ -64,7 +72,9 @@
"cell_type": "code",
"execution_count": 5,
"id": "e902885d-7382-42c8-af7f-1d82fba06bb4",
"metadata": {},
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"discrete_test.add_variant_data('A', die_A_rolls)\n",
Expand All @@ -76,16 +86,18 @@
"cell_type": "code",
"execution_count": 6,
"id": "a3ee97a1-d48c-407b-b13c-5cfb11e6591f",
"metadata": {},
"metadata": {
"tags": []
},
"outputs": [
{
"data": {
"text/plain": [
"{'A': {'concentration': [179.0, 158.0, 174.0, 180.0, 154.0, 155.0],\n",
"{'A': {'concentration': [170.0, 188.0, 159.0, 161.0, 168.0, 154.0],\n",
" 'prior': [1, 1, 1, 1, 1, 1]},\n",
" 'B': {'concentration': [230.0, 263.0, 119.0, 119.0, 245.0, 224.0],\n",
" 'B': {'concentration': [243.0, 240.0, 129.0, 124.0, 254.0, 210.0],\n",
" 'prior': [1, 1, 1, 1, 1, 1]},\n",
" 'C': {'concentration': [124.0, 43.0, 48.0, 102.0, 74.0, 109.0],\n",
" 'C': {'concentration': [93.0, 53.0, 50.0, 100.0, 105.0, 99.0],\n",
" 'prior': [1, 1, 1, 1, 1, 1]}}"
]
},
Expand All @@ -102,12 +114,14 @@
"cell_type": "code",
"execution_count": 7,
"id": "23484578-dc84-4325-9aa0-7a1498ee161b",
"metadata": {},
"metadata": {
"tags": []
},
"outputs": [
{
"data": {
"text/plain": [
"{'A': 0.05845, 'B': 0.1185, 'C': 0.82305}"
"{'A': 0.0006, 'B': 0.00115, 'C': 0.99825}"
]
},
"execution_count": 7,
Expand All @@ -123,12 +137,14 @@
"cell_type": "code",
"execution_count": 8,
"id": "7001814b-7705-420a-813d-b65393e68288",
"metadata": {},
"metadata": {
"tags": []
},
"outputs": [
{
"data": {
"text/plain": [
"{'A': 0.1433081, 'B': 0.1150933, 'C': 0.009188}"
"{'A': 0.3012179, 'B': 0.2862335, 'C': 6.09e-05}"
]
},
"execution_count": 8,
Expand All @@ -144,21 +160,23 @@
"cell_type": "code",
"execution_count": 9,
"id": "86cb2b3b-cc93-489f-ae1d-7becac229c33",
"metadata": {},
"metadata": {
"tags": []
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"+-----------+--------------------------------------------------------------+-----------------+-------------------+-----------------+\n",
"| variant | concentration | average_value | prob_being_best | expected_loss |\n",
"+===========+==============================================================+=================+===================+=================+\n",
"| A | {1: 179.0, 2: 158.0, 3: 174.0, 4: 180.0, 5: 154.0, 6: 155.0} | 3.437 | 0.05865 | 0.141573 |\n",
"+-----------+--------------------------------------------------------------+-----------------+-------------------+-----------------+\n",
"| B | {1: 230.0, 2: 263.0, 3: 119.0, 4: 119.0, 5: 245.0, 6: 224.0} | 3.465 | 0.12065 | 0.114587 |\n",
"+-----------+--------------------------------------------------------------+-----------------+-------------------+-----------------+\n",
"| C | {1: 124.0, 2: 43.0, 3: 48.0, 4: 102.0, 5: 74.0, 6: 109.0} | 3.572 | 0.8207 | 0.0092546 |\n",
"+-----------+--------------------------------------------------------------+-----------------+-------------------+-----------------+\n"
"+-----------+--------------------------------------------------------------+-----------------+------------------+-------------------+-----------------+------------------------+\n",
"| variant | concentration | average_value | posterior_mean | prob_being_best | expected_loss | credible_interval |\n",
"+===========+==============================================================+=================+==================+===================+=================+========================+\n",
"| A | {1: 170.0, 2: 188.0, 3: 159.0, 4: 161.0, 5: 168.0, 6: 154.0} | 3.431 | 3.43141 | 0.00065 | 0.301061 | [3.3254478, 3.5351591] |\n",
"+-----------+--------------------------------------------------------------+-----------------+------------------+-------------------+-----------------+------------------------+\n",
"| B | {1: 243.0, 2: 240.0, 3: 129.0, 4: 124.0, 5: 254.0, 6: 210.0} | 3.44667 | 3.44693 | 0.0011 | 0.285589 | [3.3445751, 3.5485946] |\n",
"+-----------+--------------------------------------------------------------+-----------------+------------------+-------------------+-----------------+------------------------+\n",
"| C | {1: 93.0, 2: 53.0, 3: 50.0, 4: 100.0, 5: 105.0, 6: 99.0} | 3.736 | 3.7332 | 0.99825 | 6.19e-05 | [3.575729, 3.8844363] |\n",
"+-----------+--------------------------------------------------------------+-----------------+------------------+-------------------+-----------------+------------------------+\n"
]
}
],
Expand All @@ -171,12 +189,14 @@
"cell_type": "code",
"execution_count": 10,
"id": "616e35ba-26d3-4d10-ad65-4dc37e5771a6",
"metadata": {},
"metadata": {
"tags": []
},
"outputs": [
{
"data": {
"text/plain": [
"{'A': 0.6135, 'B': 0.33145, 'C': 0.05505}"
"{'A': 0.5752, 'B': 0.42465, 'C': 0.00015}"
]
},
"execution_count": 10,
Expand All @@ -186,7 +206,7 @@
{
"data": {
"text/plain": [
"{'A': 0.0202774, 'B': 0.0484923, 'C': 0.1543976}"
"{'A': 0.0233034, 'B': 0.0382878, 'C': 0.3244604}"
]
},
"execution_count": 10,
Expand All @@ -197,15 +217,15 @@
"name": "stdout",
"output_type": "stream",
"text": [
"+-----------+--------------------------------------------------------------+-----------------+-------------------+-----------------+\n",
"| variant | concentration | average_value | prob_being_best | expected_loss |\n",
"+===========+==============================================================+=================+===================+=================+\n",
"| A | {1: 179.0, 2: 158.0, 3: 174.0, 4: 180.0, 5: 154.0, 6: 155.0} | 3.437 | 0.6119 | 0.0203846 |\n",
"+-----------+--------------------------------------------------------------+-----------------+-------------------+-----------------+\n",
"| B | {1: 230.0, 2: 263.0, 3: 119.0, 4: 119.0, 5: 245.0, 6: 224.0} | 3.465 | 0.3346 | 0.048674 |\n",
"+-----------+--------------------------------------------------------------+-----------------+-------------------+-----------------+\n",
"| C | {1: 124.0, 2: 43.0, 3: 48.0, 4: 102.0, 5: 74.0, 6: 109.0} | 3.572 | 0.0535 | 0.154126 |\n",
"+-----------+--------------------------------------------------------------+-----------------+-------------------+-----------------+\n"
"+-----------+--------------------------------------------------------------+-----------------+------------------+-------------------+-----------------+------------------------+\n",
"| variant | concentration | average_value | posterior_mean | prob_being_best | expected_loss | credible_interval |\n",
"+===========+==============================================================+=================+==================+===================+=================+========================+\n",
"| A | {1: 170.0, 2: 188.0, 3: 159.0, 4: 161.0, 5: 168.0, 6: 154.0} | 3.431 | 3.43141 | 0.5783 | 0.0233298 | [3.3252788, 3.5381138] |\n",
"+-----------+--------------------------------------------------------------+-----------------+------------------+-------------------+-----------------+------------------------+\n",
"| B | {1: 243.0, 2: 240.0, 3: 129.0, 4: 124.0, 5: 254.0, 6: 210.0} | 3.44667 | 3.44693 | 0.42165 | 0.0381169 | [3.3426174, 3.5499728] |\n",
"+-----------+--------------------------------------------------------------+-----------------+------------------+-------------------+-----------------+------------------------+\n",
"| C | {1: 93.0, 2: 53.0, 3: 50.0, 4: 100.0, 5: 105.0, 6: 99.0} | 3.736 | 3.7332 | 5e-05 | 0.325504 | [3.5781064, 3.8875444] |\n",
"+-----------+--------------------------------------------------------------+-----------------+------------------+-------------------+-----------------+------------------------+\n"
]
}
],
Expand Down Expand Up @@ -242,7 +262,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.5"
"version": "3.8.17"
}
},
"nbformat": 4,
Expand Down
66 changes: 39 additions & 27 deletions examples/goals_scored_ab_testing.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,9 @@
"cell_type": "code",
"execution_count": 1,
"id": "c5f8cedc-94d6-4805-90d4-466d4de6b293",
"metadata": {},
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"from IPython.core.interactiveshell import InteractiveShell\n",
Expand All @@ -19,12 +21,14 @@
"cell_type": "code",
"execution_count": 2,
"id": "8e57546e-4b90-4c89-8668-aafe4aff6485",
"metadata": {},
"metadata": {
"tags": []
},
"outputs": [
{
"data": {
"text/plain": [
"Generator(PCG64) at 0x1283A4BA0"
"Generator(PCG64) at 0x132C389E0"
]
},
"execution_count": 2,
Expand All @@ -40,7 +44,9 @@
"cell_type": "code",
"execution_count": 3,
"id": "1cc3f939-43a0-4d19-af63-9ae632861dee",
"metadata": {},
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# goals scored - more is better (duh...)\n",
Expand All @@ -59,7 +65,9 @@
"cell_type": "code",
"execution_count": 4,
"id": "fe532f01-6c91-4462-9213-e33379be1f9e",
"metadata": {},
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# Poisson test for \"goals for\"\n",
Expand All @@ -82,21 +90,23 @@
"cell_type": "code",
"execution_count": 5,
"id": "93cd6353-01c6-4873-a62e-9816932679fe",
"metadata": {},
"metadata": {
"tags": []
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"+-----------+----------+--------------+--------------------+------------------+-------------------+-----------------+\n",
"| variant | totals | sum_values | observed_average | posterior_mean | prob_being_best | expected_loss |\n",
"+===========+==========+==============+====================+==================+===================+=================+\n",
"| psg | 15 | 43 | 2.86667 | 2.8543 | 0.24375 | 0.509445 |\n",
"+-----------+----------+--------------+--------------------+------------------+-------------------+-----------------+\n",
"| city | 14 | 40 | 2.85714 | 2.5 | 0.05095 | 0.852689 |\n",
"+-----------+----------+--------------+--------------------+------------------+-------------------+-----------------+\n",
"| bayern | 15 | 49 | 3.26667 | 3.25166 | 0.7053 | 0.111539 |\n",
"+-----------+----------+--------------+--------------------+------------------+-------------------+-----------------+\n"
"+-----------+----------+--------------+--------------------+------------------+-------------------+-----------------+------------------------+\n",
"| variant | totals | sum_values | observed_average | posterior_mean | prob_being_best | expected_loss | credible_interval |\n",
"+===========+==========+==============+====================+==================+===================+=================+========================+\n",
"| psg | 15 | 43 | 2.86667 | 2.8543 | 0.2484 | 0.507051 | [2.071789, 3.7911295] |\n",
"+-----------+----------+--------------+--------------------+------------------+-------------------+-----------------+------------------------+\n",
"| city | 14 | 40 | 2.85714 | 2.5 | 0.04775 | 0.866365 | [1.9051119, 3.1609537] |\n",
"+-----------+----------+--------------+--------------------+------------------+-------------------+-----------------+------------------------+\n",
"| bayern | 15 | 49 | 3.26667 | 3.25166 | 0.70385 | 0.112843 | [2.4082776, 4.2095072] |\n",
"+-----------+----------+--------------+--------------------+------------------+-------------------+-----------------+------------------------+\n"
]
}
],
Expand All @@ -111,28 +121,30 @@
"cell_type": "code",
"execution_count": 6,
"id": "39220217-6553-4f88-b537-064ade561996",
"metadata": {},
"metadata": {
"tags": []
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"+-----------+----------+--------------+--------------------+------------------+-------------------+-----------------+\n",
"| variant | totals | sum_values | observed_average | posterior_mean | prob_being_best | expected_loss |\n",
"+===========+==========+==============+====================+==================+===================+=================+\n",
"| psg | 15 | 9 | 0.6 | 0.60265 | 0.7599 | 0.0411113 |\n",
"+-----------+----------+--------------+--------------------+------------------+-------------------+-----------------+\n",
"| city | 14 | 14 | 1 | 1 | 0.07465 | 0.439372 |\n",
"+-----------+----------+--------------+--------------------+------------------+-------------------+-----------------+\n",
"| bayern | 15 | 13 | 0.86667 | 0.86755 | 0.16545 | 0.30863 |\n",
"+-----------+----------+--------------+--------------------+------------------+-------------------+-----------------+\n"
"+-----------+----------+--------------+--------------------+------------------+-------------------+-----------------+------------------------+\n",
"| variant | totals | sum_values | observed_average | posterior_mean | prob_being_best | expected_loss | credible_interval |\n",
"+===========+==========+==============+====================+==================+===================+=================+========================+\n",
"| psg | 15 | 9 | 0.6 | 0.60265 | 0.7468 | 0.0444494 | [0.2171908, 1.2237673] |\n",
"+-----------+----------+--------------+--------------------+------------------+-------------------+-----------------+------------------------+\n",
"| city | 14 | 14 | 1 | 1 | 0.0778 | 0.44033 | [0.4371844, 1.8322232] |\n",
"+-----------+----------+--------------+--------------------+------------------+-------------------+-----------------+------------------------+\n",
"| bayern | 15 | 13 | 0.86667 | 0.86755 | 0.1754 | 0.304051 | [0.3759632, 1.6091403] |\n",
"+-----------+----------+--------------+--------------------+------------------+-------------------+-----------------+------------------------+\n"
]
}
],
"source": [
"# poisson_test_ga.probabs_of_being_best(sim_count = 20000, seed=52, min_is_best=True)\n",
"# poisson_test_ga.expected_loss(sim_count = 20000, seed=52, min_is_best=True)\n",
"results_ga = poisson_test_ga.evaluate(min_is_best=True)\n",
"results_ga = poisson_test_ga.evaluate(min_is_best=True, interval_alpha=0.99)\n",
"print(pd.DataFrame(results_ga).to_markdown(tablefmt=\"grid\", index=False))"
]
},
Expand Down Expand Up @@ -161,7 +173,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.5"
"version": "3.8.17"
}
},
"nbformat": 4,
Expand Down
Loading

0 comments on commit c3dccb9

Please sign in to comment.