config.json

{
    "_name_or_path": "rhubarbwu/TinyStories-12x1024_10L",
    "activation_function": "gelu_new",
    "architectures": [
      "GPTNeoForCausalLM"
    ],
    "attention_dropout": 0,
    "attention_layers": [
      "global",
      "local",
      "global",
      "local",
      "global",
      "local",
      "global",
      "local",
      "global",
      "local",
      "global",
      "local"
    ],
    "attention_types": [
      [
        [
          "global",
          "local"
        ],
        6
      ]
    ],
    "bos_token_id": 50256,
    "classifier_dropout": 0.1,
    "embed_dropout": 0,
    "eos_token_id": 50256,
    "gradient_checkpointing": false,
    "hidden_size": 1024,
    "initializer_range": 0.02,
    "intermediate_size": null,
    "layer_norm_epsilon": 1e-05,
    "max_position_embeddings": 2048,
    "model_type": "gpt_neo",
    "num_heads": 16,
    "num_layers": 12,
    "resid_dropout": 0,
    "summary_activation": null,
    "summary_first_dropout": 0.1,
    "summary_proj_to_labels": true,
    "summary_type": "cls_index",
    "summary_use_proj": true,
    "torch_dtype": "float32",
    "transformers_version": "4.37.0.dev0",
    "use_cache": true,
    "vocab_size": 50257,
    "window_size": 256
  }