Synthesizing statistical and visual information is a crucial skill for interpreting A/B test results and making actionable recommendations. When reviewing a grid of A/B test outcomes, you should combine your understanding of **p-values**, **confidence intervals**, and visual representations such as **bar plots** or **error bars**. Start by examining the `p-value`: a value below your predetermined threshold (commonly `0.05`) suggests statistical significance, but this alone is not enough. Next, review the **confidence interval** for the estimated effect - if it does not cross zero, this supports the finding of a statistically significant difference, and the range tells you about the possible size of the effect. **Visualizations** help you quickly spot patterns, outliers, or overlaps between groups, reinforcing or challenging what the numbers suggest. By integrating these elements, you can confidently interpret whether the observed difference is meaningful and what practical action should follow - such as rolling out a new feature, running further tests, or maintaining the status quo.


import unittest
import user_code
import ast
import re   
import importlib
import csv
import unittest
import pandas as pd
import importlib

class TestTask(unittest.TestCase):
    def setUp(self):
        self.user_code = importlib.import_module("user_code")

    def test_recommendation_column_added(self):
        results_data = {
            "test_scenario": ["A", "B"],
            "p_value": [0.01, 0.2],
            "confidence_interval": [(0.05, 0.10), (-0.01, 0.03)],
            "mean_difference": [0.08, 0.01]
        }
        df = pd.DataFrame(results_data)
        df_out = self.user_code.interpret_ab_results(df)
        has_column = "recommendation" in df_out.columns
        correct_length = len(df_out["recommendation"]) == len(df_out)
        _dynamic_test(self,
            has_column and correct_length,
            "Adds 'recommendation' column with one entry per scenario.",
            f"Recommendation column missing or wrong length. Columns: {df_out.columns}, length: {len(df_out['recommendation']) if has_column else 'N/A'}"
        )

    def test_recommend_rollout(self):
        results_data = {
            "test_scenario": ["C"],
            "p_value": [0.01],
            "confidence_interval": [(0.01, 0.05)],
            "mean_difference": [0.03]
        }
        df = pd.DataFrame(results_data)
        df_out = self.user_code.interpret_ab_results(df)
        rec = df_out["recommendation"][0]
        _dynamic_test(self,
            "rollout" in rec.lower() and "improvement" in rec.lower(),
            "Recommends rollout when p < 0.05 and CI above zero.",
            f"Expected rollout recommendation, got: {rec}"
        )

    def test_recommend_do_not_rollout(self):
        results_data = {
            "test_scenario": ["D"],
            "p_value": [0.01],
            "confidence_interval": [(-0.08, -0.01)],
            "mean_difference": [-0.05]
        }
        df = pd.DataFrame(results_data)
        df_out = self.user_code.interpret_ab_results(df)
        rec = df_out["recommendation"][0]
        _dynamic_test(self,
            "do not rollout" in rec.lower() or ("not rollout" in rec.lower()),
            "Recommends not rolling out when p < 0.05 and CI below zero.",
            f"Expected do not rollout recommendation, got: {rec}"
        )

    def test_recommend_investigate_further(self):
        results_data = {
            "test_scenario": ["E"],
            "p_value": [0.01],
            "confidence_interval": [(-0.02, 0.03)],
            "mean_difference": [0.00]
        }
        df = pd.DataFrame(results_data)
        df_out = self.user_code.interpret_ab_results(df)
        rec = df_out["recommendation"][0]
        _dynamic_test(self,
            "investigate" in rec.lower(),
            "Recommends to investigate further when p < 0.05 and CI crosses zero.",
            f"Expected investigate further recommendation, got: {rec}"
        )

    def test_recommend_maintain(self):
        results_data = {
            "test_scenario": ["F"],
            "p_value": [0.2],
            "confidence_interval": [(-0.01, 0.03)],
            "mean_difference": [0.01]
        }
        df = pd.DataFrame(results_data)
        df_out = self.user_code.interpret_ab_results(df)
        rec = df_out["recommendation"][0]
        _dynamic_test(self,
            "maintain" in rec.lower() or "no statistically significant difference" in rec.lower(),
            "Recommends maintaining current version when p >= 0.05.",
            f"Expected maintain recommendation, got: {rec}"
        )

def _dynamic_test(test_case, condition, success_message, failure_message):
    if condition:
        test_case._testMethodName = success_message
        test_case.assertTrue(True, success_message)
    else:
        test_case._testMethodName = failure_message
        test_case.fail(failure_message)

def normalize_text(text):
    text = text.lower()
    text = re.sub(r"\\s{2,}", " ", text)
    text = re.sub(r"\\s*([,:?])\\s*", r"\\1 ", text)
    return text.strip()

def change_var(code: str, var_name: str, value: str) -> str:
    tree = ast.parse(code)
    lines = code.splitlines()
    changed = False
    # Collect all assignment nodes to modify
    assign_nodes = [
        (i, node)
        for i, node in enumerate(tree.body)
        if isinstance(node, ast.Assign)
        and any(isinstance(target, ast.Name) and target.id == var_name for target in node.targets)
    ]

    # If nothing to change, return unmodified code
    if not assign_nodes:
        return code

    # Perform replacements for all matching assignments (from last to first to not break line offsets)
    for i, node in reversed(assign_nodes):
        start_line = node.lineno - 1
        line = lines[start_line]
        indent = ' ' * (len(line) - len(line.lstrip()))
        lines[start_line] = f"{indent}{var_name} = {value}"
        next_line = len(lines)
        for next_node in tree.body[i+1:]:
            if hasattr(next_node, 'lineno'):
                next_line = next_node.lineno - 1
                break
        if next_line > start_line + 1:
            lines[start_line+1:next_line] = []
        changed = True

    return '\\n'.join(lines) if changed else code

if __name__ == "__main__":
    unittest.main()


Challenge: Interpreting A/B Test Results

Solution