simocorbo/toxicthesis-gemini-3.5-flash-linear-classification-3

Linear - GEMINI-3.5-FLASH - Classification (3 classes)

Toxicity prediction model trained on the GEMINI-3.5-FLASH dataset.

Property	Value
Model	Linear
Task	Classification (3 classes)
Dataset	gemini-3.5-flash
Framework	PyTorch / PyTorch Lightning

Class: LinearModel

from src.models.linear import LinearModel

model = LinearModel(
    input_dim: int,              # Dimension of input embeddings (default: 300 for FastText)
    num_classes: int = 2,        # 1=regression, 2=binary, 3+=multi-class
    loss_fn: str = 'auto',       # 'auto', 'mse', 'bce', 'cross_entropy'
    dropout: float = 0.0,        # Dropout rate
    l2_lambda: float = 0.01,     # L2 regularization strength
    learning_rate: float = 0.01,
    max_epochs: int = 100,
    batch_size: int = 512,
    device: str = None           # 'cuda', 'mps', 'cpu', or None for auto
)

Methods

Method	Description
forward(x)	Forward pass. Input: (batch, input_dim). Returns raw logits.
predict_proba(x)	Returns probabilities in [0,1]. Shape depends on num_classes.
compute_loss(logits, targets)	Computes loss. Handles binning for classification.
bin_targets(targets)	Converts continuous [0,1] targets to class indices.
fit(X, y)	Train the model on data.
predict(X)	Get predictions for input embeddings.

Usage with ToxicThesis (Recommended)

# 1. Clone ToxicThesis repository
# git clone https://github.com/simo-corbo/ToxicThesis
# cd ToxicThesis && pip install -r requirements.txt

from huggingface_hub import hf_hub_download
import torch

# 2. Download checkpoint
checkpoint_path = hf_hub_download(
    repo_id="simocorbo/toxicthesis-gemini-3.5-flash-linear-classification-3",
    filename="checkpoints/best.pt"
)

# 3. Import and load model from ToxicThesis
from src.models.linear import LinearModel

# Load checkpoint to get hyperparameters
checkpoint = torch.load(checkpoint_path, map_location='cpu', weights_only=False)
hparams = checkpoint.get('hyper_parameters', {})

# Create model with same config
model = LinearModel(
    input_dim=hparams.get('input_dim', 300),
    num_classes=hparams.get('num_classes', 3),
    dropout=hparams.get('dropout', 0.0),
    l2_lambda=hparams.get('l2_lambda', 0.01)
)

# Load trained weights
state_dict = checkpoint.get('state_dict', checkpoint.get('model_state_dict', checkpoint))
model.model.load_state_dict(state_dict, strict=False)
model.model.eval()

# 4. Get predictions using built-in methods
import numpy as np
from src.utils.fasttext_utils import load_fasttext_model

ft = load_fasttext_model('cc.en.300.bin')

def get_embedding(text: str) -> np.ndarray:
    tokens = text.lower().split()
    embeddings = [ft.get_word_vector(w) for w in tokens]
    return np.mean(embeddings, axis=0) if embeddings else np.zeros(300)

# Single prediction
text = "Your text here"
emb = get_embedding(text)
X = torch.tensor(emb, dtype=torch.float32).unsqueeze(0).to(model.device)

with torch.no_grad():
    # Use predict_proba for probabilities
    probs = model.model.predict_proba(X)
    print(f"Probabilities: {probs}")
    
    # Or use forward for raw logits
    logits = model.model(X)
    print(f"Logits: {logits}")

Standalone Usage (without ToxicThesis)

from huggingface_hub import hf_hub_download
import torch
import torch.nn as nn
import fasttext
import numpy as np

# 1. Download checkpoint
checkpoint_path = hf_hub_download(
    repo_id="simocorbo/toxicthesis-gemini-3.5-flash-linear-classification-3",
    filename="checkpoints/best.pt"
)

# 2. Load FastText embeddings
ft = fasttext.load_model('cc.en.300.bin')

# 3. Define minimal model class
class LinearClassifier(nn.Module):
    def __init__(self, input_dim: int, num_classes: int):
        super().__init__()
        self.num_classes = num_classes
        output_dim = 1 if num_classes <= 2 else num_classes
        self.output = nn.Linear(input_dim, output_dim)
    
    def forward(self, x):
        return self.output(x)

# 4. Load checkpoint
checkpoint = torch.load(checkpoint_path, map_location='cpu', weights_only=False)
hparams = checkpoint.get('hyper_parameters', {})
num_classes = hparams.get('num_classes', 3)

model = LinearClassifier(input_dim=300, num_classes=num_classes)
state_dict = checkpoint.get('state_dict', checkpoint.get('model_state_dict', checkpoint))
state_dict = {k.replace('model.', '').replace('linear.', ''): v for k, v in state_dict.items()}
model.load_state_dict(state_dict, strict=False)
model.eval()

# 5. Inference function
def predict(text: str) -> dict:
    tokens = text.lower().split()
    emb = np.mean([ft.get_word_vector(w) for w in tokens], axis=0) if tokens else np.zeros(300)
    x = torch.tensor(emb, dtype=torch.float32).unsqueeze(0)
    
    with torch.no_grad():
        logits = model(x)
        if num_classes == 1:
            score = torch.sigmoid(logits).item()
            return {'score': score}
        elif num_classes == 2:
            prob = torch.sigmoid(logits).item()
            return {'probability': prob, 'class': int(prob >= 0.5)}
        else:
            probs = torch.softmax(logits, dim=-1).squeeze().tolist()
            return {'probabilities': probs, 'class': int(np.argmax(probs))}

result = predict("Your text here")
print(result)

Score Interpretation

Output	Range	Meaning
probabilities	List[float]	Probability distribution over 3 classes.
class	0 to 2	Predicted class (argmax of probabilities).

Classes: 3 toxicity levels, where higher class index = more toxic.

Files

File	Description
checkpoints/best.pt	Model checkpoint (best validation loss)
hparams.yaml	Hyperparameters used for training
train.csv	Training metrics per epoch
val.csv	Validation metrics per epoch
vocab_stanza_hybrid.pkl	Vocabulary (for tree-based models)

Installation

# Clone ToxicThesis for full model implementations
git clone https://github.com/simo-corbo/ToxicThesis
cd ToxicThesis
pip install -r requirements.txt

# Or install dependencies directly
pip install torch transformers huggingface_hub fasttext-wheel stanza

Citation

@software{toxicthesis2025,
  title={ToxicThesis},
  author={Corbo, Simone},
  year={2025},
  url={https://github.com/simo-corbo/ToxicThesis}
}