import json
import copy
__all__=['AbsModelConfig', 'ModelConfig']
class AbsModelConfig(object):
def __init__(self):
pass
@classmethod
def from_dict(cls, json_object):
"""Constructs a `ModelConfig` from a Python dictionary of parameters."""
config = cls()
for key, value in json_object.items():
if isinstance(value, dict):
value = AbsModelConfig.from_dict(value)
config.__dict__[key] = value
return config
@classmethod
def from_json_file(cls, json_file):
"""Constructs a `ModelConfig` from a json file of parameters."""
with open(json_file, "r", encoding='utf-8') as reader:
text = reader.read()
return cls.from_dict(json.loads(text))
def __repr__(self):
return str(self.to_json_string())
def to_dict(self):
"""Serializes this instance to a Python dictionary."""
output = copy.deepcopy(self.__dict__)
return output
def to_json_string(self):
"""Serializes this instance to a JSON string."""
def _json_default(obj):
if isinstance(obj, AbsModelConfig):
return obj.__dict__
return json.dumps(self.__dict__, indent=2, sort_keys=True, default=_json_default) + "\n"
[docs]class ModelConfig(AbsModelConfig):
"""Configuration class to store the configuration of a :class:`~DeBERTa.deberta.DeBERTa` model.
Attributes:
hidden_size (int): Size of the encoder layers and the pooler layer, default: `768`.
num_hidden_layers (int): Number of hidden layers in the Transformer encoder, default: `12`.
num_attention_heads (int): Number of attention heads for each attention layer in
the Transformer encoder, default: `12`.
intermediate_size (int): The size of the "intermediate" (i.e., feed-forward)
layer in the Transformer encoder, default: `3072`.
hidden_act (str): The non-linear activation function (function or string) in the
encoder and pooler. If string, "gelu", "relu" and "swish" are supported, default: `gelu`.
hidden_dropout_prob (float): The dropout probabilitiy for all fully connected
layers in the embeddings, encoder, and pooler, default: `0.1`.
attention_probs_dropout_prob (float): The dropout ratio for the attention
probabilities, default: `0.1`.
max_position_embeddings (int): The maximum sequence length that this model might
ever be used with. Typically set this to something large just in case
(e.g., 512 or 1024 or 2048), default: `512`.
type_vocab_size (int): The vocabulary size of the `token_type_ids` passed into
`DeBERTa` model, default: `-1`.
initializer_range (int): The sttdev of the _normal_initializer for
initializing all weight matrices, default: `0.02`.
relative_attention (:obj:`bool`): Whether use relative position encoding, default: `False`.
max_relative_positions (int): The range of relative positions [`-max_position_embeddings`, `max_position_embeddings`], default: -1, use the same value as `max_position_embeddings`.
padding_idx (int): The value used to pad input_ids, default: `0`.
position_biased_input (:obj:`bool`): Whether add absolute position embedding to content embedding, default: `True`.
pos_att_type (:obj:`str`): The type of relative position attention, it can be a combination of [`p2c`, `c2p`, `p2p`], e.g. "p2c", "p2c|c2p", "p2c|c2p|p2p"., default: "None".
"""
def __init__(self):
"""Constructs ModelConfig.
"""
self.hidden_size = 768
self.num_hidden_layers = 12
self.num_attention_heads = 12
self.hidden_act = "gelu"
self.intermediate_size = 3072
self.hidden_dropout_prob = 0.1
self.attention_probs_dropout_prob = 0.1
self.max_position_embeddings = 512
self.type_vocab_size = 0
self.initializer_range = 0.02
self.layer_norm_eps = 1e-7
self.padding_idx = 0
self.vocab_size = -1