megatron.data.dataset\_utils
============================

.. rubric:: Description

.. automodule:: megatron.data.dataset_utils

.. currentmodule:: megatron.data.dataset_utils


.. rubric:: Classes

.. autosummary::
    :toctree: .
    
    MaskedLmInstance
    




.. rubric:: Functions

.. autosummary::
    :toctree: .
    
    build_train_valid_test_datasets
    
    compile_helper
    
    create_masked_lm_predictions
    
    create_tokens_and_tokentypes
    
    get_a_and_b_segments
    
    get_datasets_weights_and_num_samples
    
    get_indexed_dataset_
    
    get_samples_mapping
    
    get_train_valid_test_split_
    
    is_start_piece
    
    pad_and_convert_to_numpy
    
    truncate_segments