@inproceedings{mahon2023no,
author = {Mahon, Joyce and Mac Namee, Brian and Becker, Brett A.},
title = {No More Pencils No More Books: Capabilities of Generative AI on Irish and UK Computer Science School Leaving Examinations},
year = {2023},
isbn = {9798400708763},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3610969.3610982},
doi = {10.1145/3610969.3610982},
abstract = {We investigate the capabilities of ChatGPT (GPT-4) on second-level (high-school) computer science examinations: the UK A-Level and Irish Leaving Certificate. Both are national, government-set / approved, and centrally assessed examinations. We also evaluate performance differences in exams made publicly available before and after the ChatGPT knowledge cutoff date, and investigate what types of question ChatGPT struggles with. We find that ChatGPT is capable of achieving very high marks on both exams and that the performance difference before and after the knowledge cutoff date are minimal. We also observe that ChatGPT struggles with questions involving symbols or images, which can be mitigated when in-text information ‘fills in the gaps’. Additionally, GPT-4 performance can be negatively impacted when an initial inaccurate answer leads to further inaccuracies in subsequent parts of the same question. Finally, the element of choice on the Leaving Certificate is a significant advantage in achieving a high grade. Notably, there are minimal occurrences of hallucinations in answers and few errors in solutions not involving images. These results reveal several strengths and weaknesses of these exams in terms of how generative AI performs on them and have implications for exam design, the construction of marking schemes, and could also shift the focus of what is examined and how.},
booktitle = {Proceedings of the 2023 Conference on United Kingdom \& Ireland Computing Education Research},
articleno = {2},
numpages = {7},
keywords = {Ireland, Generative AI, K-12, Artificial Intelligence, high school, examinations, GPT-4, Leaving Certificate, second-level, A-Level, UK, ChatGPT, LCCS, school},
location = {Swansea, Wales Uk},
series = {UKICER '23}
}

EndNote:

%0 Conference Paper
%T No More Pencils No More Books: Capabilities of Generative AI on Irish and UK Computer Science School Leaving Examinations
%@ 9798400708763
%U https://doi.org/10.1145/3610969.3610982
%R 10.1145/3610969.3610982
%B Proceedings of the 2023 Conference on United Kingdom & Ireland Computing Education Research
%I Association for Computing Machinery
%A Joyce Mahon
%A Brian Mac Namee
%A Brett A. Becker
%D 2023
%P Article 2
%K A-Level, K-12, high school, second-level, Ireland, ChatGPT, Leaving Certificate, school, examinations, GPT-4, Generative AI, Artificial Intelligence, LCCS, UK
%C Swansea, Wales Uk

ACM:

Joyce Mahon, Brian Mac Namee, and Brett A. Becker. 2023. No More Pencils No More Books: Capabilities of Generative AI on Irish and UK Computer Science School Leaving Examinations. In Proceedings of the 2023 Conference on United Kingdom & Ireland Computing Education Research (UKICER '23). Association for Computing Machinery, New York, NY, USA, Article 2, 1–7. https://doi.org/10.1145/3610969.3610982