tokenization_shivik_m1_fast.py

from transformers import PreTrainedTokenizerFast

class ShivikM1TokenizerFast(PreTrainedTokenizerFast):
    """
    Custom fast tokenizer for Shivik-M1 models.
    Uses tokenizer.json + merges + vocab from HuggingFace repo.
    """
    model_input_names = ["input_ids", "attention_mask"]