@inproceedings{31d2806f2bf343c39e713985fd7368a6,
title = "Automated Sample Ratio Mismatch (SRM) Detection and Analysis",
abstract = "Background: Sample Ratio Mismatch (SRM) checks can help detect data quality issues in online experimentation [3]. Not all experimentation platforms provide these checks as part of their solution. Users of these platforms must therefore manually check for SRM, or rely on additional processes—such as checklists [2]—or automation. Objective: To ensure reliable and early detection of SRM, we wanted to automate the detection and analysis of SRM in experiments running on third-party experimentation platforms. Method: A set of Looker dashboards were built to facilitate self-serve SRM detection and root cause analysis. In addition, we added email and chat based alerting to pro-actively inform experimenters of SRM and guide them towards these dashboards when needed. Results: Several cases of SRM have been detected and experimenters have been warned. Bad decisions based on flawed data were avoided. We provide one such example as an illustration. Conclusions: SRM checks are relatively straightforward to automate and can be useful for data quality monitoring even for companies who rely on third-party experimentation platforms. Pro-active alerting—rather than passive reporting—can reduce time to detection and help non-experts avoid making decisions based on biased data.",
keywords = "Infrastructure, Data Quality, Trustworthiness, A/B Testing, Sample Ratio Mismatch, SRM, Online Controlled Experimentation",
author = "Lukas Vermeer and Kevin Anderson and Mauricio Acebal",
year = "2022",
doi = "10.1145/3530019.3534982",
language = "English",
isbn = "978-1-4503-9613-4",
series = "ACM International Conference Proceeding Series",
publisher = "Association for Computing Machinery (ACM)",
pages = "268–269",
editor = "M. Staron and C. Berger and J. Simmonds and R. Prikladnicki",
booktitle = "Proceedings of the ACM International Conference on Evaluation and Assessment in Software Engineering, EASE 2022",
address = "United States",
note = "EASE 2022 : The International Conference on Evaluation and Assessment in Software Engineering 2022 ; Conference date: 13-06-2022 Through 15-06-2022",
}