Editing Tabular Dl (section)

== <span style="color: #FFFFFF;">Applying</span> ==
'''FT-Transformer on tabular benchmark:'''
<syntaxhighlight lang="python">
import torch
import torch.nn as nn
import numpy as np
from rtdl import FTTransformer  # Real-world Tabular Deep Learning library

# FT-Transformer via rtdl library
# pip install rtdl

# Separate numerical and categorical features
n_num_features = 8     # Number of continuous features
cat_cardinalities = [5, 100, 10]  # Cardinality of each categorical feature

model = FTTransformer.make_default(
    n_num_features=n_num_features,
    cat_cardinalities=cat_cardinalities,
    d_out=1,  # 1 for binary classification or regression; n_classes for multiclass
)

optimizer = model.make_default_optimizer()  # AdamW with standard tabular LR schedule

# Training loop
def train_epoch(model, loader, optimizer, task='classification'):
    model.train()
    total_loss = 0
    for X_num, X_cat, y in loader:
        logits = model(X_num, X_cat).squeeze(1)
        if task == 'classification':
            loss = nn.BCEWithLogitsLoss()(logits, y.float())
        else:
            loss = nn.MSELoss()(logits, y.float())
        optimizer.zero_grad(); loss.backward(); optimizer.step()
        total_loss += loss.item()
    return total_loss / len(loader)

# Quick comparison: TabPFN for small datasets (no training!)
from tabpfn import TabPFNClassifier
from sklearn.metrics import roc_auc_score

clf = TabPFNClassifier(device='cpu', N_ensemble_configurations=32)
clf.fit(X_train_small, y_train_small)  # Instant — no gradient descent
preds = clf.predict_proba(X_test_small)
print(f"TabPFN AUC: {roc_auc_score(y_test_small, preds[:,1]):.3f}")
# Often matches XGBoost on small datasets without any hyperparameter tuning!
</syntaxhighlight>

; Tabular DL framework selection guide
: '''Small data (<1K samples)''' → TabPFN (no training needed), XGBoost with Bayesian HPO
: '''Medium data (1K–100K)''' → XGBoost/LightGBM baseline; try FT-Transformer
: '''Large data (>100K)''' → FT-Transformer, SAINT; potentially beats GBDT
: '''High-cardinality categoricals''' → Entity embeddings + any DL model; CatBoost also strong
: '''Multi-modal (tabular + text)''' → TabTransformer/FT-Transformer + BERT fusion
: '''AutoML''' → AutoGluon-Tabular (tests multiple models); strong default
</div>

<div style="background-color: #8B4500; color: #FFFFFF; padding: 20px; border-radius: 8px; margin-bottom: 15px;">