@conference{500, keywords = {Tabular data · Transformer architectures · Gradient Boost- ing Decision Trees · Hyperparameter tuning · Rossmann Store Sales}, author = {Coenraad Middel and Marelie Davel}, title = {Comparing Transformer-based and GBDT models on tabular data: A Rossmann Store Sales case study}, abstract = {Heterogeneous tabular data is a common and important data format. This empirical study investigates how the performance of deep transformer models compares against benchmark gradient boosting decision tree (GBDT) methods, the more typical modelling approach. All models are optimised using a Bayesian hyperparameter optimisation protocol, which provides a stronger comparison than the random grid search hyperparameter optimisation utilized in earlier work. Since feature skewness is typically handled differently for GBDT and transformer-based models, we investigate the effect of a pre-processing step that normalises feature distribution on the model comparison process. Our analysis is based on the Rossmann Store Sales dataset, a widely recognized benchmark for regression tasks.}, year = {2023}, journal = {Southern African Conference for Artificial Intelligence Research (SACAIR)}, chapter = {115 - 129}, month = {December 2023}, }