bhavanikambhampati commited on 2 days ago

Commit

21394e2

verified ·

1 Parent(s): 29afefb

Publish text2sql adapter (v3)

Browse files

Files changed (17) hide show

README.md +8 -10
adapter_config.json +45 -43
adapter_model.safetensors +2 -2
checkpoint-1006/README.md +209 -0
checkpoint-1006/adapter_config.json +45 -0
checkpoint-1006/adapter_model.safetensors +3 -0
checkpoint-1006/optimizer.pt +3 -0
checkpoint-1006/rng_state.pth +3 -0
checkpoint-1006/scheduler.pt +3 -0
checkpoint-1006/tokenizer.json +0 -0
checkpoint-1006/tokenizer_config.json +11 -0
checkpoint-1006/trainer_state.json +1065 -0
checkpoint-1006/training_args.bin +3 -0
run_metadata.json +2616 -0
tokenizer.json +0 -0
tokenizer_config.json +11 -0
training_args.bin +3 -0

README.md CHANGED Viewed

@@ -1,20 +1,18 @@
 ---
-library_name: peft
 base_model: Salesforce/codegen-350M-multi
-tags:
-- lora
-- peft
-- code
-- text2sql
-- codegen
 pipeline_tag: text-generation
 ---
 # codegenstudio/codegen-350M-text2sql-lora
-LoRA adapter for **text2sql** on [Salesforce/codegen-350M-multi](https://huggingface.co/Salesforce/codegen-350M-multi).
-- Checkpoint version: `v2`
 - Task: `text2sql`
 ## Usage
@@ -31,4 +29,4 @@ model = AutoModelForCausalLM.from_pretrained(base)
 model = PeftModel.from_pretrained(model, adapter)
 ```
-Or use the multi-adapter API in this project's `hf-deploy/` package.

 ---
 base_model: Salesforce/codegen-350M-multi
+library_name: peft
 pipeline_tag: text-generation
+tags:
+  - lora
+  - peft
+  - text2sql
 ---
 # codegenstudio/codegen-350M-text2sql-lora
+LoRA adapter for text2sql on Salesforce/codegen-350M-multi.
+- Checkpoint version: `v3`
 - Task: `text2sql`
 ## Usage
 model = PeftModel.from_pretrained(model, adapter)
 ```
+Or use the multi-adapter API in this project's `fastapi-deploy/` package.

adapter_config.json CHANGED Viewed

@@ -1,43 +1,45 @@
-{
-  "alora_invocation_tokens": null,
-  "alpha_pattern": {},
-  "arrow_config": null,
-  "auto_mapping": null,
-  "base_model_name_or_path": "Salesforce/codegen-350M-multi",
-  "bias": "none",
-  "corda_config": null,
-  "ensure_weight_tying": false,
-  "eva_config": null,
-  "exclude_modules": null,
-  "fan_in_fan_out": false,
-  "inference_mode": true,
-  "init_lora_weights": true,
-  "layer_replication": null,
-  "layers_pattern": null,
-  "layers_to_transform": null,
-  "loftq_config": {},
-  "lora_alpha": 32,
-  "lora_bias": false,
-  "lora_dropout": 0.05,
-  "lora_ga_config": null,
-  "megatron_config": null,
-  "megatron_core": "megatron.core",
-  "modules_to_save": null,
-  "peft_type": "LORA",
-  "peft_version": "0.19.1",
-  "qalora_group_size": 16,
-  "r": 16,
-  "rank_pattern": {},
-  "revision": null,
-  "target_modules": [
-    "qkv_proj",
-    "out_proj"
-  ],
-  "target_parameters": null,
-  "task_type": "CAUSAL_LM",
-  "trainable_token_indices": null,
-  "use_bdlora": null,
-  "use_dora": false,
-  "use_qalora": false,
-  "use_rslora": false
-}

+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "/kaggle/working/models/base/Salesforce__codegen-350M-multi",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 64,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "lora_ga_config": null,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.19.1",
+  "qalora_group_size": 16,
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "qkv_proj",
+    "out_proj",
+    "fc_in",
+    "fc_out"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_bdlora": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

adapter_model.safetensors CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3123afdfba0bd36d6e30834e76d296cbc4f60f1e1ca77b532470d0e881037c9a
-size 7874776

 version https://git-lfs.github.com/spec/v1
+oid sha256:bd79cf6b9b17bd9e9a5fdbd0fb0bf0d349afdb5f32323796739e15c6c17d8831
+size 41963912

checkpoint-1006/README.md ADDED Viewed

	@@ -0,0 +1,209 @@

+---
+base_model: /kaggle/working/models/base/Salesforce__codegen-350M-multi
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:/kaggle/working/models/base/Salesforce__codegen-350M-multi
+- lora
+- sft
+- transformers
+- trl
+---
+# Model Card for Model ID
+<!-- Provide a quick summary of what the model is/does. -->
+## Model Details
+### Model Description
+<!-- Provide a longer summary of what this model is. -->
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+### Model Sources [optional]
+<!-- Provide the basic links for the model. -->
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+## Uses
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+### Direct Use
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+[More Information Needed]
+### Downstream Use [optional]
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+[More Information Needed]
+### Out-of-Scope Use
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+[More Information Needed]
+## Bias, Risks, and Limitations
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+[More Information Needed]
+### Recommendations
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+## How to Get Started with the Model
+Use the code below to get started with the model.
+[More Information Needed]
+## Training Details
+### Training Data
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+[More Information Needed]
+### Training Procedure
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+#### Preprocessing [optional]
+[More Information Needed]
+#### Training Hyperparameters
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+#### Speeds, Sizes, Times [optional]
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+[More Information Needed]
+## Evaluation
+<!-- This section describes the evaluation protocols and provides the results. -->
+### Testing Data, Factors & Metrics
+#### Testing Data
+<!-- This should link to a Dataset Card if possible. -->
+[More Information Needed]
+#### Factors
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+[More Information Needed]
+#### Metrics
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+[More Information Needed]
+### Results
+[More Information Needed]
+#### Summary
+## Model Examination [optional]
+<!-- Relevant interpretability work for the model goes here -->
+[More Information Needed]
+## Environmental Impact
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+## Technical Specifications [optional]
+### Model Architecture and Objective
+[More Information Needed]
+### Compute Infrastructure
+[More Information Needed]
+#### Hardware
+[More Information Needed]
+#### Software
+[More Information Needed]
+## Citation [optional]
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+**BibTeX:**
+[More Information Needed]
+**APA:**
+[More Information Needed]
+## Glossary [optional]
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+[More Information Needed]
+## More Information [optional]
+[More Information Needed]
+## Model Card Authors [optional]
+[More Information Needed]
+## Model Card Contact
+[More Information Needed]
+### Framework versions
+- PEFT 0.19.1

checkpoint-1006/adapter_config.json ADDED Viewed

	@@ -0,0 +1,45 @@

+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "/kaggle/working/models/base/Salesforce__codegen-350M-multi",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 64,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "lora_ga_config": null,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.19.1",
+  "qalora_group_size": 16,
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "qkv_proj",
+    "out_proj",
+    "fc_in",
+    "fc_out"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_bdlora": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

checkpoint-1006/adapter_model.safetensors ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:bd79cf6b9b17bd9e9a5fdbd0fb0bf0d349afdb5f32323796739e15c6c17d8831
+size 41963912

checkpoint-1006/optimizer.pt ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:ecfc8a4189800760f4908770a0fe06c314be80407500263c8605aef1ab433402
+size 84023435

checkpoint-1006/rng_state.pth ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:551ce85c4ee7b14573962088b61a4edc00d45e62de2605ca353dd2b2da06b286
+size 14645

checkpoint-1006/scheduler.pt ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:d5734fc5864fa0cf60d0b4b406a8aa9ae252f5431d8cb9b26e34057c079f216b
+size 1465

checkpoint-1006/tokenizer.json ADDED Viewed

The diff for this file is too large to render. See raw diff

checkpoint-1006/tokenizer_config.json ADDED Viewed

	@@ -0,0 +1,11 @@

+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "is_local": true,
+  "model_max_length": 2048,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "TokenizersBackend",
+  "unk_token": "<|endoftext|>"
+}

checkpoint-1006/trainer_state.json ADDED Viewed

	@@ -0,0 +1,1065 @@

+{
+  "best_global_step": 1006,
+  "best_metric": 0.24770455062389374,
+  "best_model_checkpoint": "/kaggle/working/models/checkpoints/v4/text2sql/checkpoint-1006",
+  "epoch": 2.0,
+  "eval_steps": 500,
+  "global_step": 1006,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 1.1253886640071868,
+      "epoch": 0.01990049751243781,
+      "grad_norm": 2.338360548019409,
+      "learning_rate": 7.142857142857143e-06,
+      "loss": 1.4502812385559083,
+      "mean_token_accuracy": 0.6671988122165203,
+      "num_tokens": 68027.0,
+      "step": 10
+    },
+    {
+      "entropy": 1.0893175967037678,
+      "epoch": 0.03980099502487562,
+      "grad_norm": 1.6519114971160889,
+      "learning_rate": 1.5079365079365079e-05,
+      "loss": 1.1673696517944336,
+      "mean_token_accuracy": 0.698525931686163,
+      "num_tokens": 135033.0,
+      "step": 20
+    },
+    {
+      "entropy": 1.0583932913839817,
+      "epoch": 0.05970149253731343,
+      "grad_norm": 1.202163815498352,
+      "learning_rate": 2.3015873015873015e-05,
+      "loss": 1.0070637702941894,
+      "mean_token_accuracy": 0.7307960025966167,
+      "num_tokens": 201643.0,
+      "step": 30
+    },
+    {
+      "entropy": 0.8722786333411932,
+      "epoch": 0.07960199004975124,
+      "grad_norm": 1.3554679155349731,
+      "learning_rate": 3.095238095238095e-05,
+      "loss": 0.749023151397705,
+      "mean_token_accuracy": 0.8074305906891823,
+      "num_tokens": 267572.0,
+      "step": 40
+    },
+    {
+      "entropy": 0.6009022377431392,
+      "epoch": 0.09950248756218906,
+      "grad_norm": 1.4063202142715454,
+      "learning_rate": 3.888888888888889e-05,
+      "loss": 0.5123498916625977,
+      "mean_token_accuracy": 0.8385950662195683,
+      "num_tokens": 333673.0,
+      "step": 50
+    },
+    {
+      "entropy": 0.4164533382281661,
+      "epoch": 0.11940298507462686,
+      "grad_norm": 1.8205658197402954,
+      "learning_rate": 4.682539682539683e-05,
+      "loss": 0.4625075340270996,
+      "mean_token_accuracy": 0.8584991104900837,
+      "num_tokens": 399397.0,
+      "step": 60
+    },
+    {
+      "entropy": 0.4577839931473136,
+      "epoch": 0.13930348258706468,
+      "grad_norm": 1.393362283706665,
+      "learning_rate": 5.4761904761904766e-05,
+      "loss": 0.4174641609191895,
+      "mean_token_accuracy": 0.8803921975195408,
+      "num_tokens": 465864.0,
+      "step": 70
+    },
+    {
+      "entropy": 0.41640141475945713,
+      "epoch": 0.15920398009950248,
+      "grad_norm": 2.2472920417785645,
+      "learning_rate": 6.26984126984127e-05,
+      "loss": 0.417645263671875,
+      "mean_token_accuracy": 0.8779219165444374,
+      "num_tokens": 534817.0,
+      "step": 80
+    },
+    {
+      "entropy": 0.3839787464588881,
+      "epoch": 0.1791044776119403,
+      "grad_norm": 1.300377368927002,
+      "learning_rate": 7.063492063492065e-05,
+      "loss": 0.38586442470550536,
+      "mean_token_accuracy": 0.8903462648391723,
+      "num_tokens": 600614.0,
+      "step": 90
+    },
+    {
+      "entropy": 0.3429916022345424,
+      "epoch": 0.19900497512437812,
+      "grad_norm": 1.5014147758483887,
+      "learning_rate": 7.857142857142858e-05,
+      "loss": 0.33204805850982666,
+      "mean_token_accuracy": 0.9070690087974072,
+      "num_tokens": 665888.0,
+      "step": 100
+    },
+    {
+      "entropy": 0.36409866753965614,
+      "epoch": 0.21890547263681592,
+      "grad_norm": 1.417038083076477,
+      "learning_rate": 8.650793650793651e-05,
+      "loss": 0.3376448631286621,
+      "mean_token_accuracy": 0.8962906174361706,
+      "num_tokens": 732400.0,
+      "step": 110
+    },
+    {
+      "entropy": 0.263674839399755,
+      "epoch": 0.23880597014925373,
+      "grad_norm": 1.2735201120376587,
+      "learning_rate": 9.444444444444444e-05,
+      "loss": 0.270894718170166,
+      "mean_token_accuracy": 0.9201160937547683,
+      "num_tokens": 799887.0,
+      "step": 120
+    },
+    {
+      "entropy": 0.30584911033511164,
+      "epoch": 0.25870646766169153,
+      "grad_norm": 1.2719416618347168,
+      "learning_rate": 0.00010238095238095237,
+      "loss": 0.27406151294708253,
+      "mean_token_accuracy": 0.9204320795834064,
+      "num_tokens": 866147.0,
+      "step": 130
+    },
+    {
+      "entropy": 0.28959042597562074,
+      "epoch": 0.27860696517412936,
+      "grad_norm": 2.2021138668060303,
+      "learning_rate": 0.00011031746031746033,
+      "loss": 0.32373068332672117,
+      "mean_token_accuracy": 0.8983257927000523,
+      "num_tokens": 933129.0,
+      "step": 140
+    },
+    {
+      "entropy": 0.29785235710442065,
+      "epoch": 0.29850746268656714,
+      "grad_norm": 0.9740720987319946,
+      "learning_rate": 0.00011825396825396826,
+      "loss": 0.2523745775222778,
+      "mean_token_accuracy": 0.9215191625058651,
+      "num_tokens": 997739.0,
+      "step": 150
+    },
+    {
+      "entropy": 0.28159743677824733,
+      "epoch": 0.31840796019900497,
+      "grad_norm": 1.025788426399231,
+      "learning_rate": 0.0001261904761904762,
+      "loss": 0.27993102073669435,
+      "mean_token_accuracy": 0.9143906190991402,
+      "num_tokens": 1064807.0,
+      "step": 160
+    },
+    {
+      "entropy": 0.282751829456538,
+      "epoch": 0.3383084577114428,
+      "grad_norm": 1.149941086769104,
+      "learning_rate": 0.00013412698412698412,
+      "loss": 0.289961838722229,
+      "mean_token_accuracy": 0.9160969875752926,
+      "num_tokens": 1130310.0,
+      "step": 170
+    },
+    {
+      "entropy": 0.30801400616765023,
+      "epoch": 0.3582089552238806,
+      "grad_norm": 1.1288280487060547,
+      "learning_rate": 0.00014206349206349208,
+      "loss": 0.2773977518081665,
+      "mean_token_accuracy": 0.9185432456433773,
+      "num_tokens": 1197192.0,
+      "step": 180
+    },
+    {
+      "entropy": 0.24713189490139484,
+      "epoch": 0.3781094527363184,
+      "grad_norm": 0.9636886119842529,
+      "learning_rate": 0.00015000000000000001,
+      "loss": 0.26672122478485105,
+      "mean_token_accuracy": 0.9264318533241749,
+      "num_tokens": 1263466.0,
+      "step": 190
+    },
+    {
+      "entropy": 0.287679692171514,
+      "epoch": 0.39800995024875624,
+      "grad_norm": 1.137581467628479,
+      "learning_rate": 0.00015793650793650795,
+      "loss": 0.2587902069091797,
+      "mean_token_accuracy": 0.9211024351418018,
+      "num_tokens": 1330385.0,
+      "step": 200
+    },
+    {
+      "entropy": 0.2620579367503524,
+      "epoch": 0.417910447761194,
+      "grad_norm": 1.3906782865524292,
+      "learning_rate": 0.0001658730158730159,
+      "loss": 0.25291283130645753,
+      "mean_token_accuracy": 0.922594179213047,
+      "num_tokens": 1399272.0,
+      "step": 210
+    },
+    {
+      "entropy": 0.24558336716145276,
+      "epoch": 0.43781094527363185,
+      "grad_norm": 1.1515614986419678,
+      "learning_rate": 0.00017380952380952383,
+      "loss": 0.23645257949829102,
+      "mean_token_accuracy": 0.9315326489508152,
+      "num_tokens": 1466197.0,
+      "step": 220
+    },
+    {
+      "entropy": 0.23100281171500683,
+      "epoch": 0.4577114427860697,
+      "grad_norm": 1.0060856342315674,
+      "learning_rate": 0.00018174603174603177,
+      "loss": 0.21004528999328614,
+      "mean_token_accuracy": 0.9294509522616863,
+      "num_tokens": 1533135.0,
+      "step": 230
+    },
+    {
+      "entropy": 0.2551280764862895,
+      "epoch": 0.47761194029850745,
+      "grad_norm": 0.8229029178619385,
+      "learning_rate": 0.0001896825396825397,
+      "loss": 0.23107924461364746,
+      "mean_token_accuracy": 0.9292825579643249,
+      "num_tokens": 1600905.0,
+      "step": 240
+    },
+    {
+      "entropy": 0.2424938028678298,
+      "epoch": 0.4975124378109453,
+      "grad_norm": 1.199968695640564,
+      "learning_rate": 0.00019761904761904763,
+      "loss": 0.25817849636077883,
+      "mean_token_accuracy": 0.9277816534042358,
+      "num_tokens": 1666956.0,
+      "step": 250
+    },
+    {
+      "entropy": 0.27997240191325545,
+      "epoch": 0.5174129353233831,
+      "grad_norm": 0.6602728962898254,
+      "learning_rate": 0.0001999989408126818,
+      "loss": 0.25299272537231443,
+      "mean_token_accuracy": 0.926287354528904,
+      "num_tokens": 1734782.0,
+      "step": 260
+    },
+    {
+      "entropy": 0.20782412360422314,
+      "epoch": 0.5373134328358209,
+      "grad_norm": 0.7651694416999817,
+      "learning_rate": 0.0001999937530104439,
+      "loss": 0.20403761863708497,
+      "mean_token_accuracy": 0.9310354895889759,
+      "num_tokens": 1800055.0,
+      "step": 270
+    },
+    {
+      "entropy": 0.2172251005657017,
+      "epoch": 0.5572139303482587,
+      "grad_norm": 1.2643989324569702,
+      "learning_rate": 0.00019998424227267588,
+      "loss": 0.20663719177246093,
+      "mean_token_accuracy": 0.9364220850169659,
+      "num_tokens": 1866900.0,
+      "step": 280
+    },
+    {
+      "entropy": 0.19974687648937106,
+      "epoch": 0.5771144278606966,
+      "grad_norm": 0.9468103051185608,
+      "learning_rate": 0.00019997040901054646,
+      "loss": 0.20071234703063964,
+      "mean_token_accuracy": 0.9412682101130485,
+      "num_tokens": 1934548.0,
+      "step": 290
+    },
+    {
+      "entropy": 0.1886322578880936,
+      "epoch": 0.5970149253731343,
+      "grad_norm": 0.8480322360992432,
+      "learning_rate": 0.000199952253822096,
+      "loss": 0.18384914398193358,
+      "mean_token_accuracy": 0.948433005809784,
+      "num_tokens": 2001379.0,
+      "step": 300
+    },
+    {
+      "entropy": 0.21811659364029765,
+      "epoch": 0.6169154228855721,
+      "grad_norm": 0.8029792308807373,
+      "learning_rate": 0.00019992977749221064,
+      "loss": 0.20460138320922852,
+      "mean_token_accuracy": 0.9401800245046615,
+      "num_tokens": 2070001.0,
+      "step": 310
+    },
+    {
+      "entropy": 0.18392337979748846,
+      "epoch": 0.6368159203980099,
+      "grad_norm": 0.8735001087188721,
+      "learning_rate": 0.0001999029809925883,
+      "loss": 0.17823137044906617,
+      "mean_token_accuracy": 0.9488276831805706,
+      "num_tokens": 2135350.0,
+      "step": 320
+    },
+    {
+      "entropy": 0.18473064368590714,
+      "epoch": 0.6567164179104478,
+      "grad_norm": 0.7449737787246704,
+      "learning_rate": 0.00019987186548169682,
+      "loss": 0.16862742900848388,
+      "mean_token_accuracy": 0.9462769009172917,
+      "num_tokens": 2203472.0,
+      "step": 330
+    },
+    {
+      "entropy": 0.16661408836953343,
+      "epoch": 0.6766169154228856,
+      "grad_norm": 0.8586801290512085,
+      "learning_rate": 0.0001998364323047236,
+      "loss": 0.16848835945129395,
+      "mean_token_accuracy": 0.9531193666160107,
+      "num_tokens": 2269601.0,
+      "step": 340
+    },
+    {
+      "entropy": 0.1743609025143087,
+      "epoch": 0.6965174129353234,
+      "grad_norm": 0.7884849309921265,
+      "learning_rate": 0.00019979668299351783,
+      "loss": 0.17167186737060547,
+      "mean_token_accuracy": 0.9482224509119987,
+      "num_tokens": 2335815.0,
+      "step": 350
+    },
+    {
+      "entropy": 0.1952204409521073,
+      "epoch": 0.7164179104477612,
+      "grad_norm": 0.941378116607666,
+      "learning_rate": 0.00019975261926652392,
+      "loss": 0.18604878187179566,
+      "mean_token_accuracy": 0.942904282361269,
+      "num_tokens": 2404130.0,
+      "step": 360
+    },
+    {
+      "entropy": 0.19471225845627488,
+      "epoch": 0.736318407960199,
+      "grad_norm": 0.7770227789878845,
+      "learning_rate": 0.00019970424302870739,
+      "loss": 0.1847616672515869,
+      "mean_token_accuracy": 0.9433550946414471,
+      "num_tokens": 2470744.0,
+      "step": 370
+    },
+    {
+      "entropy": 0.19339222041890025,
+      "epoch": 0.7562189054726368,
+      "grad_norm": 1.1318728923797607,
+      "learning_rate": 0.00019965155637147243,
+      "loss": 0.18451868295669555,
+      "mean_token_accuracy": 0.9457193531095982,
+      "num_tokens": 2537581.0,
+      "step": 380
+    },
+    {
+      "entropy": 0.17824228820391,
+      "epoch": 0.7761194029850746,
+      "grad_norm": 0.8256890773773193,
+      "learning_rate": 0.0001995945615725716,
+      "loss": 0.17818082571029664,
+      "mean_token_accuracy": 0.9509255833923816,
+      "num_tokens": 2604045.0,
+      "step": 390
+    },
+    {
+      "entropy": 0.17659657467156648,
+      "epoch": 0.7960199004975125,
+      "grad_norm": 0.7419007420539856,
+      "learning_rate": 0.00019953326109600728,
+      "loss": 0.15658079385757445,
+      "mean_token_accuracy": 0.9504644252359867,
+      "num_tokens": 2669475.0,
+      "step": 400
+    },
+    {
+      "entropy": 0.16667078505270183,
+      "epoch": 0.8159203980099502,
+      "grad_norm": 0.8381772041320801,
+      "learning_rate": 0.00019946765759192497,
+      "loss": 0.16461316347122193,
+      "mean_token_accuracy": 0.9532247520983219,
+      "num_tokens": 2735709.0,
+      "step": 410
+    },
+    {
+      "entropy": 0.1525796527042985,
+      "epoch": 0.835820895522388,
+      "grad_norm": 0.7996618747711182,
+      "learning_rate": 0.00019939775389649916,
+      "loss": 0.15512030124664306,
+      "mean_token_accuracy": 0.9544322639703751,
+      "num_tokens": 2804586.0,
+      "step": 420
+    },
+    {
+      "entropy": 0.16498080925084652,
+      "epoch": 0.8557213930348259,
+      "grad_norm": 0.6151495575904846,
+      "learning_rate": 0.00019932355303181026,
+      "loss": 0.1559414744377136,
+      "mean_token_accuracy": 0.956156824529171,
+      "num_tokens": 2871824.0,
+      "step": 430
+    },
+    {
+      "entropy": 0.20550905396230518,
+      "epoch": 0.8756218905472637,
+      "grad_norm": 0.7512227892875671,
+      "learning_rate": 0.00019924505820571425,
+      "loss": 0.17734644412994385,
+      "mean_token_accuracy": 0.9471527449786663,
+      "num_tokens": 2937826.0,
+      "step": 440
+    },
+    {
+      "entropy": 0.1697809119708836,
+      "epoch": 0.8955223880597015,
+      "grad_norm": 0.762320339679718,
+      "learning_rate": 0.0001991622728117038,
+      "loss": 0.15445693731307983,
+      "mean_token_accuracy": 0.9543316774070263,
+      "num_tokens": 3006135.0,
+      "step": 450
+    },
+    {
+      "entropy": 0.17613516801502554,
+      "epoch": 0.9154228855721394,
+      "grad_norm": 0.6466898918151855,
+      "learning_rate": 0.00019907520042876172,
+      "loss": 0.16953592300415038,
+      "mean_token_accuracy": 0.950883662700653,
+      "num_tokens": 3073150.0,
+      "step": 460
+    },
+    {
+      "entropy": 0.15535307771060616,
+      "epoch": 0.9353233830845771,
+      "grad_norm": 1.1007940769195557,
+      "learning_rate": 0.00019898384482120614,
+      "loss": 0.15585366487503052,
+      "mean_token_accuracy": 0.9595168434083462,
+      "num_tokens": 3138638.0,
+      "step": 470
+    },
+    {
+      "entropy": 0.1557972011389211,
+      "epoch": 0.9552238805970149,
+      "grad_norm": 0.6533820629119873,
+      "learning_rate": 0.0001988882099385278,
+      "loss": 0.15293551683425904,
+      "mean_token_accuracy": 0.9572585269808769,
+      "num_tokens": 3206372.0,
+      "step": 480
+    },
+    {
+      "entropy": 0.13091885023750366,
+      "epoch": 0.9751243781094527,
+      "grad_norm": 0.5975553393363953,
+      "learning_rate": 0.00019878829991521935,
+      "loss": 0.1253079891204834,
+      "mean_token_accuracy": 0.9635655723512173,
+      "num_tokens": 3274281.0,
+      "step": 490
+    },
+    {
+      "entropy": 0.1669132244773209,
+      "epoch": 0.9950248756218906,
+      "grad_norm": 0.8513726592063904,
+      "learning_rate": 0.00019868411907059645,
+      "loss": 0.17081425189971924,
+      "mean_token_accuracy": 0.9505244322121144,
+      "num_tokens": 3340159.0,
+      "step": 500
+    },
+    {
+      "epoch": 1.0,
+      "eval_entropy": 0.23721536022024842,
+      "eval_loss": 0.24836121499538422,
+      "eval_mean_token_accuracy": 0.9339753162676764,
+      "eval_num_tokens": 3356986.0,
+      "eval_runtime": 67.6123,
+      "eval_samples_per_second": 15.308,
+      "eval_steps_per_second": 7.661,
+      "step": 503
+    },
+    {
+      "entropy": 0.1568159531605871,
+      "epoch": 1.0139303482587065,
+      "grad_norm": 0.5372758507728577,
+      "learning_rate": 0.00019857567190861126,
+      "loss": 0.1242946743965149,
+      "mean_token_accuracy": 0.9635580613424903,
+      "num_tokens": 3404334.0,
+      "step": 510
+    },
+    {
+      "entropy": 0.1316974400077015,
+      "epoch": 1.0338308457711443,
+      "grad_norm": 0.7160713076591492,
+      "learning_rate": 0.00019846296311765754,
+      "loss": 0.1351562261581421,
+      "mean_token_accuracy": 0.9581282936036587,
+      "num_tokens": 3471525.0,
+      "step": 520
+    },
+    {
+      "entropy": 0.14656153018586338,
+      "epoch": 1.053731343283582,
+      "grad_norm": 0.5991392731666565,
+      "learning_rate": 0.00019834599757036803,
+      "loss": 0.12300963401794433,
+      "mean_token_accuracy": 0.9646149106323719,
+      "num_tokens": 3538338.0,
+      "step": 530
+    },
+    {
+      "entropy": 0.15197489713318646,
+      "epoch": 1.07363184079602,
+      "grad_norm": 0.9330605864524841,
+      "learning_rate": 0.00019822478032340387,
+      "loss": 0.12765015363693238,
+      "mean_token_accuracy": 0.9586149267852306,
+      "num_tokens": 3606642.0,
+      "step": 540
+    },
+    {
+      "entropy": 0.11335502485744656,
+      "epoch": 1.0935323383084576,
+      "grad_norm": 0.8762836456298828,
+      "learning_rate": 0.0001980993166172358,
+      "loss": 0.11172252893447876,
+      "mean_token_accuracy": 0.9656657867133618,
+      "num_tokens": 3674420.0,
+      "step": 550
+    },
+    {
+      "entropy": 0.12588824331760406,
+      "epoch": 1.1134328358208956,
+      "grad_norm": 0.8866952061653137,
+      "learning_rate": 0.00019796961187591781,
+      "loss": 0.10862302780151367,
+      "mean_token_accuracy": 0.9648959740996361,
+      "num_tokens": 3741381.0,
+      "step": 560
+    },
+    {
+      "entropy": 0.11430091026704758,
+      "epoch": 1.1333333333333333,
+      "grad_norm": 0.6481871604919434,
+      "learning_rate": 0.00019783567170685262,
+      "loss": 0.11582423448562622,
+      "mean_token_accuracy": 0.9628987669944763,
+      "num_tokens": 3808318.0,
+      "step": 570
+    },
+    {
+      "entropy": 0.11517859897576273,
+      "epoch": 1.153233830845771,
+      "grad_norm": 0.5650383234024048,
+      "learning_rate": 0.00019769750190054905,
+      "loss": 0.1037294864654541,
+      "mean_token_accuracy": 0.9702431283891201,
+      "num_tokens": 3874410.0,
+      "step": 580
+    },
+    {
+      "entropy": 0.10291576159652323,
+      "epoch": 1.173134328358209,
+      "grad_norm": 0.5433067083358765,
+      "learning_rate": 0.00019755510843037191,
+      "loss": 0.10045719146728516,
+      "mean_token_accuracy": 0.9618785113096238,
+      "num_tokens": 3941134.0,
+      "step": 590
+    },
+    {
+      "entropy": 0.11075262987287715,
+      "epoch": 1.1930348258706467,
+      "grad_norm": 0.7597805261611938,
+      "learning_rate": 0.00019740849745228367,
+      "loss": 0.12216674089431763,
+      "mean_token_accuracy": 0.9657749280333519,
+      "num_tokens": 4008097.0,
+      "step": 600
+    },
+    {
+      "entropy": 0.12675938094034792,
+      "epoch": 1.2129353233830846,
+      "grad_norm": 0.6820019483566284,
+      "learning_rate": 0.00019725767530457837,
+      "loss": 0.11509623527526855,
+      "mean_token_accuracy": 0.9657132118940354,
+      "num_tokens": 4073570.0,
+      "step": 610
+    },
+    {
+      "entropy": 0.13302950132638217,
+      "epoch": 1.2328358208955223,
+      "grad_norm": 0.5021085739135742,
+      "learning_rate": 0.00019710264850760756,
+      "loss": 0.10812582969665527,
+      "mean_token_accuracy": 0.9639534369111061,
+      "num_tokens": 4140207.0,
+      "step": 620
+    },
+    {
+      "entropy": 0.12266454068012536,
+      "epoch": 1.25273631840796,
+      "grad_norm": 0.7293747067451477,
+      "learning_rate": 0.00019694342376349835,
+      "loss": 0.12314709424972534,
+      "mean_token_accuracy": 0.9629024133086205,
+      "num_tokens": 4206997.0,
+      "step": 630
+    },
+    {
+      "entropy": 0.12240176484920084,
+      "epoch": 1.272636815920398,
+      "grad_norm": 0.9365243911743164,
+      "learning_rate": 0.00019678000795586386,
+      "loss": 0.117351496219635,
+      "mean_token_accuracy": 0.96337865665555,
+      "num_tokens": 4277152.0,
+      "step": 640
+    },
+    {
+      "entropy": 0.11147555778734386,
+      "epoch": 1.292537313432836,
+      "grad_norm": 0.7311879396438599,
+      "learning_rate": 0.00019661240814950536,
+      "loss": 0.11279709339141845,
+      "mean_token_accuracy": 0.9658049061894417,
+      "num_tokens": 4344652.0,
+      "step": 650
+    },
+    {
+      "entropy": 0.11891384413465858,
+      "epoch": 1.3124378109452737,
+      "grad_norm": 0.6283079981803894,
+      "learning_rate": 0.00019644063159010716,
+      "loss": 0.09745638966560363,
+      "mean_token_accuracy": 0.9698325954377651,
+      "num_tokens": 4411916.0,
+      "step": 660
+    },
+    {
+      "entropy": 0.12134865028783678,
+      "epoch": 1.3323383084577114,
+      "grad_norm": 0.6160532832145691,
+      "learning_rate": 0.00019626468570392298,
+      "loss": 0.1158707618713379,
+      "mean_token_accuracy": 0.9658548943698406,
+      "num_tokens": 4478621.0,
+      "step": 670
+    },
+    {
+      "entropy": 0.10654389052651822,
+      "epoch": 1.3522388059701491,
+      "grad_norm": 0.43357354402542114,
+      "learning_rate": 0.00019608457809745537,
+      "loss": 0.09168269634246826,
+      "mean_token_accuracy": 0.9723479963839055,
+      "num_tokens": 4544168.0,
+      "step": 680
+    },
+    {
+      "entropy": 0.109538364992477,
+      "epoch": 1.372139303482587,
+      "grad_norm": 0.7618773579597473,
+      "learning_rate": 0.00019590031655712631,
+      "loss": 0.11080507040023804,
+      "mean_token_accuracy": 0.9682544745504856,
+      "num_tokens": 4610518.0,
+      "step": 690
+    },
+    {
+      "entropy": 0.11043523394037039,
+      "epoch": 1.392039800995025,
+      "grad_norm": 0.8631065487861633,
+      "learning_rate": 0.00019571190904894115,
+      "loss": 0.09991470575332642,
+      "mean_token_accuracy": 0.9694355696439743,
+      "num_tokens": 4676248.0,
+      "step": 700
+    },
+    {
+      "entropy": 0.1043223840300925,
+      "epoch": 1.4119402985074627,
+      "grad_norm": 1.0989038944244385,
+      "learning_rate": 0.00019551936371814375,
+      "loss": 0.10194973945617676,
+      "mean_token_accuracy": 0.9693835198879241,
+      "num_tokens": 4742628.0,
+      "step": 710
+    },
+    {
+      "entropy": 0.10138569427654147,
+      "epoch": 1.4318407960199004,
+      "grad_norm": 0.9169466495513916,
+      "learning_rate": 0.0001953226888888647,
+      "loss": 0.10495258569717407,
+      "mean_token_accuracy": 0.970366296172142,
+      "num_tokens": 4809419.0,
+      "step": 720
+    },
+    {
+      "entropy": 0.11472290018573403,
+      "epoch": 1.4517412935323384,
+      "grad_norm": 0.49399080872535706,
+      "learning_rate": 0.00019512189306376118,
+      "loss": 0.11099306344985962,
+      "mean_token_accuracy": 0.9688441671431065,
+      "num_tokens": 4877119.0,
+      "step": 730
+    },
+    {
+      "entropy": 0.11205615981016308,
+      "epoch": 1.471641791044776,
+      "grad_norm": 0.61200350522995,
+      "learning_rate": 0.0001949169849236496,
+      "loss": 0.10629711151123047,
+      "mean_token_accuracy": 0.966337724775076,
+      "num_tokens": 4944433.0,
+      "step": 740
+    },
+    {
+      "entropy": 0.11642576553858816,
+      "epoch": 1.491542288557214,
+      "grad_norm": 0.6224406361579895,
+      "learning_rate": 0.00019470797332713012,
+      "loss": 0.11089148521423339,
+      "mean_token_accuracy": 0.9659203454852104,
+      "num_tokens": 5011634.0,
+      "step": 750
+    },
+    {
+      "entropy": 0.11514911148697138,
+      "epoch": 1.5114427860696518,
+      "grad_norm": 0.7650023698806763,
+      "learning_rate": 0.0001944948673102038,
+      "loss": 0.10163601636886596,
+      "mean_token_accuracy": 0.9675627797842026,
+      "num_tokens": 5077156.0,
+      "step": 760
+    },
+    {
+      "entropy": 0.10764023282099515,
+      "epoch": 1.5313432835820895,
+      "grad_norm": 0.49794143438339233,
+      "learning_rate": 0.00019427767608588183,
+      "loss": 0.098751300573349,
+      "mean_token_accuracy": 0.9677646860480309,
+      "num_tokens": 5143991.0,
+      "step": 770
+    },
+    {
+      "entropy": 0.1094623792450875,
+      "epoch": 1.5512437810945272,
+      "grad_norm": 0.6449922323226929,
+      "learning_rate": 0.0001940564090437875,
+      "loss": 0.11413514614105225,
+      "mean_token_accuracy": 0.9680206254124641,
+      "num_tokens": 5211604.0,
+      "step": 780
+    },
+    {
+      "entropy": 0.12053416313137859,
+      "epoch": 1.5711442786069652,
+      "grad_norm": 0.8340095281600952,
+      "learning_rate": 0.00019383107574974984,
+      "loss": 0.10809429883956909,
+      "mean_token_accuracy": 0.9674052610993386,
+      "num_tokens": 5277697.0,
+      "step": 790
+    },
+    {
+      "entropy": 0.10000467539066449,
+      "epoch": 1.591044776119403,
+      "grad_norm": 0.7644860744476318,
+      "learning_rate": 0.00019360168594539055,
+      "loss": 0.08709208965301514,
+      "mean_token_accuracy": 0.9731013409793376,
+      "num_tokens": 5342978.0,
+      "step": 800
+    },
+    {
+      "entropy": 0.10800170768052339,
+      "epoch": 1.6109452736318408,
+      "grad_norm": 0.6640422344207764,
+      "learning_rate": 0.0001933682495477024,
+      "loss": 0.09788179397583008,
+      "mean_token_accuracy": 0.9665102422237396,
+      "num_tokens": 5410289.0,
+      "step": 810
+    },
+    {
+      "entropy": 0.10117872587870806,
+      "epoch": 1.6308457711442785,
+      "grad_norm": 0.6755411624908447,
+      "learning_rate": 0.00019313077664862092,
+      "loss": 0.09605536460876465,
+      "mean_token_accuracy": 0.972070074826479,
+      "num_tokens": 5475880.0,
+      "step": 820
+    },
+    {
+      "entropy": 0.1069639899302274,
+      "epoch": 1.6507462686567163,
+      "grad_norm": 0.890627384185791,
+      "learning_rate": 0.00019288927751458766,
+      "loss": 0.10400503873825073,
+      "mean_token_accuracy": 0.9711074873805046,
+      "num_tokens": 5541606.0,
+      "step": 830
+    },
+    {
+      "entropy": 0.11311845399904996,
+      "epoch": 1.6706467661691542,
+      "grad_norm": 0.5509929656982422,
+      "learning_rate": 0.0001926437625861068,
+      "loss": 0.10754516124725341,
+      "mean_token_accuracy": 0.9695759303867817,
+      "num_tokens": 5608065.0,
+      "step": 840
+    },
+    {
+      "entropy": 0.12085098770912736,
+      "epoch": 1.6905472636815921,
+      "grad_norm": 0.6879218816757202,
+      "learning_rate": 0.00019239424247729345,
+      "loss": 0.12257307767868042,
+      "mean_token_accuracy": 0.9648775070905685,
+      "num_tokens": 5674308.0,
+      "step": 850
+    },
+    {
+      "entropy": 0.11081431191414595,
+      "epoch": 1.7104477611940299,
+      "grad_norm": 0.4920276999473572,
+      "learning_rate": 0.0001921407279754149,
+      "loss": 0.10849488973617553,
+      "mean_token_accuracy": 0.9698534436523915,
+      "num_tokens": 5742414.0,
+      "step": 860
+    },
+    {
+      "entropy": 0.0929627066012472,
+      "epoch": 1.7303482587064676,
+      "grad_norm": 0.7228904366493225,
+      "learning_rate": 0.00019188323004042435,
+      "loss": 0.0802489161491394,
+      "mean_token_accuracy": 0.9730034552514553,
+      "num_tokens": 5809827.0,
+      "step": 870
+    },
+    {
+      "entropy": 0.0936126358807087,
+      "epoch": 1.7502487562189055,
+      "grad_norm": 0.6558980345726013,
+      "learning_rate": 0.00019162175980448688,
+      "loss": 0.1109757661819458,
+      "mean_token_accuracy": 0.9671767763793468,
+      "num_tokens": 5875172.0,
+      "step": 880
+    },
+    {
+      "entropy": 0.11744439310859889,
+      "epoch": 1.7701492537313432,
+      "grad_norm": 0.5792869925498962,
+      "learning_rate": 0.0001913563285714984,
+      "loss": 0.09240159988403321,
+      "mean_token_accuracy": 0.9710902646183968,
+      "num_tokens": 5942033.0,
+      "step": 890
+    },
+    {
+      "entropy": 0.10555630044545979,
+      "epoch": 1.7900497512437812,
+      "grad_norm": 0.8990387320518494,
+      "learning_rate": 0.0001910869478165969,
+      "loss": 0.12259334325790405,
+      "mean_token_accuracy": 0.9644198954105377,
+      "num_tokens": 6007977.0,
+      "step": 900
+    },
+    {
+      "entropy": 0.10993905747309327,
+      "epoch": 1.809950248756219,
+      "grad_norm": 0.7768874168395996,
+      "learning_rate": 0.00019081362918566618,
+      "loss": 0.10547571182250977,
+      "mean_token_accuracy": 0.9728645481169224,
+      "num_tokens": 6074057.0,
+      "step": 910
+    },
+    {
+      "entropy": 0.10362131035653874,
+      "epoch": 1.8298507462686566,
+      "grad_norm": 0.4525396525859833,
+      "learning_rate": 0.00019053638449483259,
+      "loss": 0.10611696243286133,
+      "mean_token_accuracy": 0.9724654078483581,
+      "num_tokens": 6141345.0,
+      "step": 920
+    },
+    {
+      "entropy": 0.11784212745260447,
+      "epoch": 1.8497512437810946,
+      "grad_norm": 0.7492642998695374,
+      "learning_rate": 0.0001902552257299542,
+      "loss": 0.11176036596298218,
+      "mean_token_accuracy": 0.9628825642168521,
+      "num_tokens": 6209124.0,
+      "step": 930
+    },
+    {
+      "entropy": 0.11072989953681826,
+      "epoch": 1.8696517412935323,
+      "grad_norm": 0.5748555064201355,
+      "learning_rate": 0.00018997016504610246,
+      "loss": 0.10774084329605102,
+      "mean_token_accuracy": 0.9646185263991356,
+      "num_tokens": 6276485.0,
+      "step": 940
+    },
+    {
+      "entropy": 0.10556984411086887,
+      "epoch": 1.8895522388059702,
+      "grad_norm": 0.5937514305114746,
+      "learning_rate": 0.00018968121476703678,
+      "loss": 0.08848418593406678,
+      "mean_token_accuracy": 0.9734670996665955,
+      "num_tokens": 6342957.0,
+      "step": 950
+    },
+    {
+      "entropy": 0.09428299731807784,
+      "epoch": 1.909452736318408,
+      "grad_norm": 1.0715358257293701,
+      "learning_rate": 0.00018938838738467184,
+      "loss": 0.10269320011138916,
+      "mean_token_accuracy": 0.9720249362289906,
+      "num_tokens": 6408918.0,
+      "step": 960
+    },
+    {
+      "entropy": 0.10850229405332357,
+      "epoch": 1.9293532338308457,
+      "grad_norm": 0.5169550180435181,
+      "learning_rate": 0.00018909169555853743,
+      "loss": 0.0899561107158661,
+      "mean_token_accuracy": 0.9714486353099346,
+      "num_tokens": 6475062.0,
+      "step": 970
+    },
+    {
+      "entropy": 0.1026058561168611,
+      "epoch": 1.9492537313432836,
+      "grad_norm": 0.7642938494682312,
+      "learning_rate": 0.00018879115211523117,
+      "loss": 0.09649609923362731,
+      "mean_token_accuracy": 0.9703472301363945,
+      "num_tokens": 6542465.0,
+      "step": 980
+    },
+    {
+      "entropy": 0.09136548589449375,
+      "epoch": 1.9691542288557216,
+      "grad_norm": 0.3874703347682953,
+      "learning_rate": 0.0001884867700478641,
+      "loss": 0.09067035913467407,
+      "mean_token_accuracy": 0.9718083783984184,
+      "num_tokens": 6608985.0,
+      "step": 990
+    },
+    {
+      "entropy": 0.11761876428499818,
+      "epoch": 1.9890547263681593,
+      "grad_norm": 0.693695604801178,
+      "learning_rate": 0.00018817856251549867,
+      "loss": 0.09239903688430787,
+      "mean_token_accuracy": 0.9674227833747864,
+      "num_tokens": 6676885.0,
+      "step": 1000
+    },
+    {
+      "epoch": 2.0,
+      "eval_entropy": 0.13267684354209502,
+      "eval_loss": 0.24770455062389374,
+      "eval_mean_token_accuracy": 0.9410935246806347,
+      "eval_num_tokens": 6713972.0,
+      "eval_runtime": 67.3665,
+      "eval_samples_per_second": 15.364,
+      "eval_steps_per_second": 7.689,
+      "step": 1006
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 5030,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 10,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "EarlyStoppingCallback": {
+      "args": {
+        "early_stopping_patience": 3,
+        "early_stopping_threshold": 0.0
+      },
+      "attributes": {
+        "early_stopping_patience_counter": 0
+      }
+    },
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.3957721650888704e+16,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-1006/training_args.bin ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:2350592d6176e1137e9e5000ddaae8bcace4a32263a7e4abe5cae8ea3d5bbe86
+size 5841

run_metadata.json ADDED Viewed

	@@ -0,0 +1,2616 @@

+{
+  "task": "text2sql",
+  "model_name": "Salesforce/codegen-350M-multi",
+  "device": "cuda:0",
+  "checkpoint_run": "v4",
+  "adapter_path": "/kaggle/working/models/checkpoints/v4/text2sql",
+  "train_rows": 8040,
+  "eval_rows": 1035,
+  "train_loss": 0.10587940257242612,
+  "eval_loss": 0.24770455062389374,
+  "best_eval_loss": 0.24770455062389374,
+  "train_runtime_seconds": 8534.0451,
+  "mlflow_run_id": null,
+  "filter_stats": {
+    "input_rows": 8040,
+    "kept_rows": 8040,
+    "skipped_rows": 0,
+    "skip_reasons": {}
+  },
+  "token_stats": {
+    "rows": 8040,
+    "avg_prompt_tokens": 377.37,
+    "avg_target_tokens": 39.17,
+    "avg_total_tokens": 416.54,
+    "max_total_tokens": 1098,
+    "skipped_too_long_target": 0,
+    "skipped_too_long_prompt": 0
+  },
+  "log_history": [
+    {
+      "loss": 1.4502812385559083,
+      "grad_norm": 2.338360548019409,
+      "learning_rate": 7.142857142857143e-06,
+      "entropy": 1.1253886640071868,
+      "num_tokens": 68027.0,
+      "mean_token_accuracy": 0.6671988122165203,
+      "epoch": 0.01990049751243781,
+      "step": 10
+    },
+    {
+      "loss": 1.1673696517944336,
+      "grad_norm": 1.6519114971160889,
+      "learning_rate": 1.5079365079365079e-05,
+      "entropy": 1.0893175967037678,
+      "num_tokens": 135033.0,
+      "mean_token_accuracy": 0.698525931686163,
+      "epoch": 0.03980099502487562,
+      "step": 20
+    },
+    {
+      "loss": 1.0070637702941894,
+      "grad_norm": 1.202163815498352,
+      "learning_rate": 2.3015873015873015e-05,
+      "entropy": 1.0583932913839817,
+      "num_tokens": 201643.0,
+      "mean_token_accuracy": 0.7307960025966167,
+      "epoch": 0.05970149253731343,
+      "step": 30
+    },
+    {
+      "loss": 0.749023151397705,
+      "grad_norm": 1.3554679155349731,
+      "learning_rate": 3.095238095238095e-05,
+      "entropy": 0.8722786333411932,
+      "num_tokens": 267572.0,
+      "mean_token_accuracy": 0.8074305906891823,
+      "epoch": 0.07960199004975124,
+      "step": 40
+    },
+    {
+      "loss": 0.5123498916625977,
+      "grad_norm": 1.4063202142715454,
+      "learning_rate": 3.888888888888889e-05,
+      "entropy": 0.6009022377431392,
+      "num_tokens": 333673.0,
+      "mean_token_accuracy": 0.8385950662195683,
+      "epoch": 0.09950248756218906,
+      "step": 50
+    },
+    {
+      "loss": 0.4625075340270996,
+      "grad_norm": 1.8205658197402954,
+      "learning_rate": 4.682539682539683e-05,
+      "entropy": 0.4164533382281661,
+      "num_tokens": 399397.0,
+      "mean_token_accuracy": 0.8584991104900837,
+      "epoch": 0.11940298507462686,
+      "step": 60
+    },
+    {
+      "loss": 0.4174641609191895,
+      "grad_norm": 1.393362283706665,
+      "learning_rate": 5.4761904761904766e-05,
+      "entropy": 0.4577839931473136,
+      "num_tokens": 465864.0,
+      "mean_token_accuracy": 0.8803921975195408,
+      "epoch": 0.13930348258706468,
+      "step": 70
+    },
+    {
+      "loss": 0.417645263671875,
+      "grad_norm": 2.2472920417785645,
+      "learning_rate": 6.26984126984127e-05,
+      "entropy": 0.41640141475945713,
+      "num_tokens": 534817.0,
+      "mean_token_accuracy": 0.8779219165444374,
+      "epoch": 0.15920398009950248,
+      "step": 80
+    },
+    {
+      "loss": 0.38586442470550536,
+      "grad_norm": 1.300377368927002,
+      "learning_rate": 7.063492063492065e-05,
+      "entropy": 0.3839787464588881,
+      "num_tokens": 600614.0,
+      "mean_token_accuracy": 0.8903462648391723,
+      "epoch": 0.1791044776119403,
+      "step": 90
+    },
+    {
+      "loss": 0.33204805850982666,
+      "grad_norm": 1.5014147758483887,
+      "learning_rate": 7.857142857142858e-05,
+      "entropy": 0.3429916022345424,
+      "num_tokens": 665888.0,
+      "mean_token_accuracy": 0.9070690087974072,
+      "epoch": 0.19900497512437812,
+      "step": 100
+    },
+    {
+      "loss": 0.3376448631286621,
+      "grad_norm": 1.417038083076477,
+      "learning_rate": 8.650793650793651e-05,
+      "entropy": 0.36409866753965614,
+      "num_tokens": 732400.0,
+      "mean_token_accuracy": 0.8962906174361706,
+      "epoch": 0.21890547263681592,
+      "step": 110
+    },
+    {
+      "loss": 0.270894718170166,
+      "grad_norm": 1.2735201120376587,
+      "learning_rate": 9.444444444444444e-05,
+      "entropy": 0.263674839399755,
+      "num_tokens": 799887.0,
+      "mean_token_accuracy": 0.9201160937547683,
+      "epoch": 0.23880597014925373,
+      "step": 120
+    },
+    {
+      "loss": 0.27406151294708253,
+      "grad_norm": 1.2719416618347168,
+      "learning_rate": 0.00010238095238095237,
+      "entropy": 0.30584911033511164,
+      "num_tokens": 866147.0,
+      "mean_token_accuracy": 0.9204320795834064,
+      "epoch": 0.25870646766169153,
+      "step": 130
+    },
+    {
+      "loss": 0.32373068332672117,
+      "grad_norm": 2.2021138668060303,
+      "learning_rate": 0.00011031746031746033,
+      "entropy": 0.28959042597562074,
+      "num_tokens": 933129.0,
+      "mean_token_accuracy": 0.8983257927000523,
+      "epoch": 0.27860696517412936,
+      "step": 140
+    },
+    {
+      "loss": 0.2523745775222778,
+      "grad_norm": 0.9740720987319946,
+      "learning_rate": 0.00011825396825396826,
+      "entropy": 0.29785235710442065,
+      "num_tokens": 997739.0,
+      "mean_token_accuracy": 0.9215191625058651,
+      "epoch": 0.29850746268656714,
+      "step": 150
+    },
+    {
+      "loss": 0.27993102073669435,
+      "grad_norm": 1.025788426399231,
+      "learning_rate": 0.0001261904761904762,
+      "entropy": 0.28159743677824733,
+      "num_tokens": 1064807.0,
+      "mean_token_accuracy": 0.9143906190991402,
+      "epoch": 0.31840796019900497,
+      "step": 160
+    },
+    {
+      "loss": 0.289961838722229,
+      "grad_norm": 1.149941086769104,
+      "learning_rate": 0.00013412698412698412,
+      "entropy": 0.282751829456538,
+      "num_tokens": 1130310.0,
+      "mean_token_accuracy": 0.9160969875752926,
+      "epoch": 0.3383084577114428,
+      "step": 170
+    },
+    {
+      "loss": 0.2773977518081665,
+      "grad_norm": 1.1288280487060547,
+      "learning_rate": 0.00014206349206349208,
+      "entropy": 0.30801400616765023,
+      "num_tokens": 1197192.0,
+      "mean_token_accuracy": 0.9185432456433773,
+      "epoch": 0.3582089552238806,
+      "step": 180
+    },
+    {
+      "loss": 0.26672122478485105,
+      "grad_norm": 0.9636886119842529,
+      "learning_rate": 0.00015000000000000001,
+      "entropy": 0.24713189490139484,
+      "num_tokens": 1263466.0,
+      "mean_token_accuracy": 0.9264318533241749,
+      "epoch": 0.3781094527363184,
+      "step": 190
+    },
+    {
+      "loss": 0.2587902069091797,
+      "grad_norm": 1.137581467628479,
+      "learning_rate": 0.00015793650793650795,
+      "entropy": 0.287679692171514,
+      "num_tokens": 1330385.0,
+      "mean_token_accuracy": 0.9211024351418018,
+      "epoch": 0.39800995024875624,
+      "step": 200
+    },
+    {
+      "loss": 0.25291283130645753,
+      "grad_norm": 1.3906782865524292,
+      "learning_rate": 0.0001658730158730159,
+      "entropy": 0.2620579367503524,
+      "num_tokens": 1399272.0,
+      "mean_token_accuracy": 0.922594179213047,
+      "epoch": 0.417910447761194,
+      "step": 210
+    },
+    {
+      "loss": 0.23645257949829102,
+      "grad_norm": 1.1515614986419678,
+      "learning_rate": 0.00017380952380952383,
+      "entropy": 0.24558336716145276,
+      "num_tokens": 1466197.0,
+      "mean_token_accuracy": 0.9315326489508152,
+      "epoch": 0.43781094527363185,
+      "step": 220
+    },
+    {
+      "loss": 0.21004528999328614,
+      "grad_norm": 1.0060856342315674,
+      "learning_rate": 0.00018174603174603177,
+      "entropy": 0.23100281171500683,
+      "num_tokens": 1533135.0,
+      "mean_token_accuracy": 0.9294509522616863,
+      "epoch": 0.4577114427860697,
+      "step": 230
+    },
+    {
+      "loss": 0.23107924461364746,
+      "grad_norm": 0.8229029178619385,
+      "learning_rate": 0.0001896825396825397,
+      "entropy": 0.2551280764862895,
+      "num_tokens": 1600905.0,
+      "mean_token_accuracy": 0.9292825579643249,
+      "epoch": 0.47761194029850745,
+      "step": 240
+    },
+    {
+      "loss": 0.25817849636077883,
+      "grad_norm": 1.199968695640564,
+      "learning_rate": 0.00019761904761904763,
+      "entropy": 0.2424938028678298,
+      "num_tokens": 1666956.0,
+      "mean_token_accuracy": 0.9277816534042358,
+      "epoch": 0.4975124378109453,
+      "step": 250
+    },
+    {
+      "loss": 0.25299272537231443,
+      "grad_norm": 0.6602728962898254,
+      "learning_rate": 0.0001999989408126818,
+      "entropy": 0.27997240191325545,
+      "num_tokens": 1734782.0,
+      "mean_token_accuracy": 0.926287354528904,
+      "epoch": 0.5174129353233831,
+      "step": 260
+    },
+    {
+      "loss": 0.20403761863708497,
+      "grad_norm": 0.7651694416999817,
+      "learning_rate": 0.0001999937530104439,
+      "entropy": 0.20782412360422314,
+      "num_tokens": 1800055.0,
+      "mean_token_accuracy": 0.9310354895889759,
+      "epoch": 0.5373134328358209,
+      "step": 270
+    },
+    {
+      "loss": 0.20663719177246093,
+      "grad_norm": 1.2643989324569702,
+      "learning_rate": 0.00019998424227267588,
+      "entropy": 0.2172251005657017,
+      "num_tokens": 1866900.0,
+      "mean_token_accuracy": 0.9364220850169659,
+      "epoch": 0.5572139303482587,
+      "step": 280
+    },
+    {
+      "loss": 0.20071234703063964,
+      "grad_norm": 0.9468103051185608,
+      "learning_rate": 0.00019997040901054646,
+      "entropy": 0.19974687648937106,
+      "num_tokens": 1934548.0,
+      "mean_token_accuracy": 0.9412682101130485,
+      "epoch": 0.5771144278606966,
+      "step": 290
+    },
+    {
+      "loss": 0.18384914398193358,
+      "grad_norm": 0.8480322360992432,
+      "learning_rate": 0.000199952253822096,
+      "entropy": 0.1886322578880936,
+      "num_tokens": 2001379.0,
+      "mean_token_accuracy": 0.948433005809784,
+      "epoch": 0.5970149253731343,
+      "step": 300
+    },
+    {
+      "loss": 0.20460138320922852,
+      "grad_norm": 0.8029792308807373,
+      "learning_rate": 0.00019992977749221064,
+      "entropy": 0.21811659364029765,
+      "num_tokens": 2070001.0,
+      "mean_token_accuracy": 0.9401800245046615,
+      "epoch": 0.6169154228855721,
+      "step": 310
+    },
+    {
+      "loss": 0.17823137044906617,
+      "grad_norm": 0.8735001087188721,
+      "learning_rate": 0.0001999029809925883,
+      "entropy": 0.18392337979748846,
+      "num_tokens": 2135350.0,
+      "mean_token_accuracy": 0.9488276831805706,
+      "epoch": 0.6368159203980099,
+      "step": 320
+    },
+    {
+      "loss": 0.16862742900848388,
+      "grad_norm": 0.7449737787246704,
+      "learning_rate": 0.00019987186548169682,
+      "entropy": 0.18473064368590714,
+      "num_tokens": 2203472.0,
+      "mean_token_accuracy": 0.9462769009172917,
+      "epoch": 0.6567164179104478,
+      "step": 330
+    },
+    {
+      "loss": 0.16848835945129395,
+      "grad_norm": 0.8586801290512085,
+      "learning_rate": 0.0001998364323047236,
+      "entropy": 0.16661408836953343,
+      "num_tokens": 2269601.0,
+      "mean_token_accuracy": 0.9531193666160107,
+      "epoch": 0.6766169154228856,
+      "step": 340
+    },
+    {
+      "loss": 0.17167186737060547,
+      "grad_norm": 0.7884849309921265,
+      "learning_rate": 0.00019979668299351783,
+      "entropy": 0.1743609025143087,
+      "num_tokens": 2335815.0,
+      "mean_token_accuracy": 0.9482224509119987,
+      "epoch": 0.6965174129353234,
+      "step": 350
+    },
+    {
+      "loss": 0.18604878187179566,
+      "grad_norm": 0.941378116607666,
+      "learning_rate": 0.00019975261926652392,
+      "entropy": 0.1952204409521073,
+      "num_tokens": 2404130.0,
+      "mean_token_accuracy": 0.942904282361269,
+      "epoch": 0.7164179104477612,
+      "step": 360
+    },
+    {
+      "loss": 0.1847616672515869,
+      "grad_norm": 0.7770227789878845,
+      "learning_rate": 0.00019970424302870739,
+      "entropy": 0.19471225845627488,
+      "num_tokens": 2470744.0,
+      "mean_token_accuracy": 0.9433550946414471,
+      "epoch": 0.736318407960199,
+      "step": 370
+    },
+    {
+      "loss": 0.18451868295669555,
+      "grad_norm": 1.1318728923797607,
+      "learning_rate": 0.00019965155637147243,
+      "entropy": 0.19339222041890025,
+      "num_tokens": 2537581.0,
+      "mean_token_accuracy": 0.9457193531095982,
+      "epoch": 0.7562189054726368,
+      "step": 380
+    },
+    {
+      "loss": 0.17818082571029664,
+      "grad_norm": 0.8256890773773193,
+      "learning_rate": 0.0001995945615725716,
+      "entropy": 0.17824228820391,
+      "num_tokens": 2604045.0,
+      "mean_token_accuracy": 0.9509255833923816,
+      "epoch": 0.7761194029850746,
+      "step": 390
+    },
+    {
+      "loss": 0.15658079385757445,
+      "grad_norm": 0.7419007420539856,
+      "learning_rate": 0.00019953326109600728,
+      "entropy": 0.17659657467156648,
+      "num_tokens": 2669475.0,
+      "mean_token_accuracy": 0.9504644252359867,
+      "epoch": 0.7960199004975125,
+      "step": 400
+    },
+    {
+      "loss": 0.16461316347122193,
+      "grad_norm": 0.8381772041320801,
+      "learning_rate": 0.00019946765759192497,
+      "entropy": 0.16667078505270183,
+      "num_tokens": 2735709.0,
+      "mean_token_accuracy": 0.9532247520983219,
+      "epoch": 0.8159203980099502,
+      "step": 410
+    },
+    {
+      "loss": 0.15512030124664306,
+      "grad_norm": 0.7996618747711182,
+      "learning_rate": 0.00019939775389649916,
+      "entropy": 0.1525796527042985,
+      "num_tokens": 2804586.0,
+      "mean_token_accuracy": 0.9544322639703751,
+      "epoch": 0.835820895522388,
+      "step": 420
+    },
+    {
+      "loss": 0.1559414744377136,
+      "grad_norm": 0.6151495575904846,
+      "learning_rate": 0.00019932355303181026,
+      "entropy": 0.16498080925084652,
+      "num_tokens": 2871824.0,
+      "mean_token_accuracy": 0.956156824529171,
+      "epoch": 0.8557213930348259,
+      "step": 430
+    },
+    {
+      "loss": 0.17734644412994385,
+      "grad_norm": 0.7512227892875671,
+      "learning_rate": 0.00019924505820571425,
+      "entropy": 0.20550905396230518,
+      "num_tokens": 2937826.0,
+      "mean_token_accuracy": 0.9471527449786663,
+      "epoch": 0.8756218905472637,
+      "step": 440
+    },
+    {
+      "loss": 0.15445693731307983,
+      "grad_norm": 0.762320339679718,
+      "learning_rate": 0.0001991622728117038,
+      "entropy": 0.1697809119708836,
+      "num_tokens": 3006135.0,
+      "mean_token_accuracy": 0.9543316774070263,
+      "epoch": 0.8955223880597015,
+      "step": 450
+    },
+    {
+      "loss": 0.16953592300415038,
+      "grad_norm": 0.6466898918151855,
+      "learning_rate": 0.00019907520042876172,
+      "entropy": 0.17613516801502554,
+      "num_tokens": 3073150.0,
+      "mean_token_accuracy": 0.950883662700653,
+      "epoch": 0.9154228855721394,
+      "step": 460
+    },
+    {
+      "loss": 0.15585366487503052,
+      "grad_norm": 1.1007940769195557,
+      "learning_rate": 0.00019898384482120614,
+      "entropy": 0.15535307771060616,
+      "num_tokens": 3138638.0,
+      "mean_token_accuracy": 0.9595168434083462,
+      "epoch": 0.9353233830845771,
+      "step": 470
+    },
+    {
+      "loss": 0.15293551683425904,
+      "grad_norm": 0.6533820629119873,
+      "learning_rate": 0.0001988882099385278,
+      "entropy": 0.1557972011389211,
+      "num_tokens": 3206372.0,
+      "mean_token_accuracy": 0.9572585269808769,
+      "epoch": 0.9552238805970149,
+      "step": 480
+    },
+    {
+      "loss": 0.1253079891204834,
+      "grad_norm": 0.5975553393363953,
+      "learning_rate": 0.00019878829991521935,
+      "entropy": 0.13091885023750366,
+      "num_tokens": 3274281.0,
+      "mean_token_accuracy": 0.9635655723512173,
+      "epoch": 0.9751243781094527,
+      "step": 490
+    },
+    {
+      "loss": 0.17081425189971924,
+      "grad_norm": 0.8513726592063904,
+      "learning_rate": 0.00019868411907059645,
+      "entropy": 0.1669132244773209,
+      "num_tokens": 3340159.0,
+      "mean_token_accuracy": 0.9505244322121144,
+      "epoch": 0.9950248756218906,
+      "step": 500
+    },
+    {
+      "eval_loss": 0.24836121499538422,
+      "eval_runtime": 67.6123,
+      "eval_samples_per_second": 15.308,
+      "eval_steps_per_second": 7.661,
+      "eval_entropy": 0.23721536022024842,
+      "eval_num_tokens": 3356986.0,
+      "eval_mean_token_accuracy": 0.9339753162676764,
+      "epoch": 1.0,
+      "step": 503
+    },
+    {
+      "loss": 0.1242946743965149,
+      "grad_norm": 0.5372758507728577,
+      "learning_rate": 0.00019857567190861126,
+      "entropy": 0.1568159531605871,
+      "num_tokens": 3404334.0,
+      "mean_token_accuracy": 0.9635580613424903,
+      "epoch": 1.0139303482587065,
+      "step": 510
+    },
+    {
+      "loss": 0.1351562261581421,
+      "grad_norm": 0.7160713076591492,
+      "learning_rate": 0.00019846296311765754,
+      "entropy": 0.1316974400077015,
+      "num_tokens": 3471525.0,
+      "mean_token_accuracy": 0.9581282936036587,
+      "epoch": 1.0338308457711443,
+      "step": 520
+    },
+    {
+      "loss": 0.12300963401794433,
+      "grad_norm": 0.5991392731666565,
+      "learning_rate": 0.00019834599757036803,
+      "entropy": 0.14656153018586338,
+      "num_tokens": 3538338.0,
+      "mean_token_accuracy": 0.9646149106323719,
+      "epoch": 1.053731343283582,
+      "step": 530
+    },
+    {
+      "loss": 0.12765015363693238,
+      "grad_norm": 0.9330605864524841,
+      "learning_rate": 0.00019822478032340387,
+      "entropy": 0.15197489713318646,
+      "num_tokens": 3606642.0,
+      "mean_token_accuracy": 0.9586149267852306,
+      "epoch": 1.07363184079602,
+      "step": 540
+    },
+    {
+      "loss": 0.11172252893447876,
+      "grad_norm": 0.8762836456298828,
+      "learning_rate": 0.0001980993166172358,
+      "entropy": 0.11335502485744656,
+      "num_tokens": 3674420.0,
+      "mean_token_accuracy": 0.9656657867133618,
+      "epoch": 1.0935323383084576,
+      "step": 550
+    },
+    {
+      "loss": 0.10862302780151367,
+      "grad_norm": 0.8866952061653137,
+      "learning_rate": 0.00019796961187591781,
+      "entropy": 0.12588824331760406,
+      "num_tokens": 3741381.0,
+      "mean_token_accuracy": 0.9648959740996361,
+      "epoch": 1.1134328358208956,
+      "step": 560
+    },
+    {
+      "loss": 0.11582423448562622,
+      "grad_norm": 0.6481871604919434,
+      "learning_rate": 0.00019783567170685262,
+      "entropy": 0.11430091026704758,
+      "num_tokens": 3808318.0,
+      "mean_token_accuracy": 0.9628987669944763,
+      "epoch": 1.1333333333333333,
+      "step": 570
+    },
+    {
+      "loss": 0.1037294864654541,
+      "grad_norm": 0.5650383234024048,
+      "learning_rate": 0.00019769750190054905,
+      "entropy": 0.11517859897576273,
+      "num_tokens": 3874410.0,
+      "mean_token_accuracy": 0.9702431283891201,
+      "epoch": 1.153233830845771,
+      "step": 580
+    },
+    {
+      "loss": 0.10045719146728516,
+      "grad_norm": 0.5433067083358765,
+      "learning_rate": 0.00019755510843037191,
+      "entropy": 0.10291576159652323,
+      "num_tokens": 3941134.0,
+      "mean_token_accuracy": 0.9618785113096238,
+      "epoch": 1.173134328358209,
+      "step": 590
+    },
+    {
+      "loss": 0.12216674089431763,
+      "grad_norm": 0.7597805261611938,
+      "learning_rate": 0.00019740849745228367,
+      "entropy": 0.11075262987287715,
+      "num_tokens": 4008097.0,
+      "mean_token_accuracy": 0.9657749280333519,
+      "epoch": 1.1930348258706467,
+      "step": 600
+    },
+    {
+      "loss": 0.11509623527526855,
+      "grad_norm": 0.6820019483566284,
+      "learning_rate": 0.00019725767530457837,
+      "entropy": 0.12675938094034792,
+      "num_tokens": 4073570.0,
+      "mean_token_accuracy": 0.9657132118940354,
+      "epoch": 1.2129353233830846,
+      "step": 610
+    },
+    {
+      "loss": 0.10812582969665527,
+      "grad_norm": 0.5021085739135742,
+      "learning_rate": 0.00019710264850760756,
+      "entropy": 0.13302950132638217,
+      "num_tokens": 4140207.0,
+      "mean_token_accuracy": 0.9639534369111061,
+      "epoch": 1.2328358208955223,
+      "step": 620
+    },
+    {
+      "loss": 0.12314709424972534,
+      "grad_norm": 0.7293747067451477,
+      "learning_rate": 0.00019694342376349835,
+      "entropy": 0.12266454068012536,
+      "num_tokens": 4206997.0,
+      "mean_token_accuracy": 0.9629024133086205,
+      "epoch": 1.25273631840796,
+      "step": 630
+    },
+    {
+      "loss": 0.117351496219635,
+      "grad_norm": 0.9365243911743164,
+      "learning_rate": 0.00019678000795586386,
+      "entropy": 0.12240176484920084,
+      "num_tokens": 4277152.0,
+      "mean_token_accuracy": 0.96337865665555,
+      "epoch": 1.272636815920398,
+      "step": 640
+    },
+    {
+      "loss": 0.11279709339141845,
+      "grad_norm": 0.7311879396438599,
+      "learning_rate": 0.00019661240814950536,
+      "entropy": 0.11147555778734386,
+      "num_tokens": 4344652.0,
+      "mean_token_accuracy": 0.9658049061894417,
+      "epoch": 1.292537313432836,
+      "step": 650
+    },
+    {
+      "loss": 0.09745638966560363,
+      "grad_norm": 0.6283079981803894,
+      "learning_rate": 0.00019644063159010716,
+      "entropy": 0.11891384413465858,
+      "num_tokens": 4411916.0,
+      "mean_token_accuracy": 0.9698325954377651,
+      "epoch": 1.3124378109452737,
+      "step": 660
+    },
+    {
+      "loss": 0.1158707618713379,
+      "grad_norm": 0.6160532832145691,
+      "learning_rate": 0.00019626468570392298,
+      "entropy": 0.12134865028783678,
+      "num_tokens": 4478621.0,
+      "mean_token_accuracy": 0.9658548943698406,
+      "epoch": 1.3323383084577114,
+      "step": 670
+    },
+    {
+      "loss": 0.09168269634246826,
+      "grad_norm": 0.43357354402542114,
+      "learning_rate": 0.00019608457809745537,
+      "entropy": 0.10654389052651822,
+      "num_tokens": 4544168.0,
+      "mean_token_accuracy": 0.9723479963839055,
+      "epoch": 1.3522388059701491,
+      "step": 680
+    },
+    {
+      "loss": 0.11080507040023804,
+      "grad_norm": 0.7618773579597473,
+      "learning_rate": 0.00019590031655712631,
+      "entropy": 0.109538364992477,
+      "num_tokens": 4610518.0,
+      "mean_token_accuracy": 0.9682544745504856,
+      "epoch": 1.372139303482587,
+      "step": 690
+    },
+    {
+      "loss": 0.09991470575332642,
+      "grad_norm": 0.8631065487861633,
+      "learning_rate": 0.00019571190904894115,
+      "entropy": 0.11043523394037039,
+      "num_tokens": 4676248.0,
+      "mean_token_accuracy": 0.9694355696439743,
+      "epoch": 1.392039800995025,
+      "step": 700
+    },
+    {
+      "loss": 0.10194973945617676,
+      "grad_norm": 1.0989038944244385,
+      "learning_rate": 0.00019551936371814375,
+      "entropy": 0.1043223840300925,
+      "num_tokens": 4742628.0,
+      "mean_token_accuracy": 0.9693835198879241,
+      "epoch": 1.4119402985074627,
+      "step": 710
+    },
+    {
+      "loss": 0.10495258569717407,
+      "grad_norm": 0.9169466495513916,
+      "learning_rate": 0.0001953226888888647,
+      "entropy": 0.10138569427654147,
+      "num_tokens": 4809419.0,
+      "mean_token_accuracy": 0.970366296172142,
+      "epoch": 1.4318407960199004,
+      "step": 720
+    },
+    {
+      "loss": 0.11099306344985962,
+      "grad_norm": 0.49399080872535706,
+      "learning_rate": 0.00019512189306376118,
+      "entropy": 0.11472290018573403,
+      "num_tokens": 4877119.0,
+      "mean_token_accuracy": 0.9688441671431065,
+      "epoch": 1.4517412935323384,
+      "step": 730
+    },
+    {
+      "loss": 0.10629711151123047,
+      "grad_norm": 0.61200350522995,
+      "learning_rate": 0.0001949169849236496,
+      "entropy": 0.11205615981016308,
+      "num_tokens": 4944433.0,
+      "mean_token_accuracy": 0.966337724775076,
+      "epoch": 1.471641791044776,
+      "step": 740
+    },
+    {
+      "loss": 0.11089148521423339,
+      "grad_norm": 0.6224406361579895,
+      "learning_rate": 0.00019470797332713012,
+      "entropy": 0.11642576553858816,
+      "num_tokens": 5011634.0,
+      "mean_token_accuracy": 0.9659203454852104,
+      "epoch": 1.491542288557214,
+      "step": 750
+    },
+    {
+      "loss": 0.10163601636886596,
+      "grad_norm": 0.7650023698806763,
+      "learning_rate": 0.0001944948673102038,
+      "entropy": 0.11514911148697138,
+      "num_tokens": 5077156.0,
+      "mean_token_accuracy": 0.9675627797842026,
+      "epoch": 1.5114427860696518,
+      "step": 760
+    },
+    {
+      "loss": 0.098751300573349,
+      "grad_norm": 0.49794143438339233,
+      "learning_rate": 0.00019427767608588183,
+      "entropy": 0.10764023282099515,
+      "num_tokens": 5143991.0,
+      "mean_token_accuracy": 0.9677646860480309,
+      "epoch": 1.5313432835820895,
+      "step": 770
+    },
+    {
+      "loss": 0.11413514614105225,
+      "grad_norm": 0.6449922323226929,
+      "learning_rate": 0.0001940564090437875,
+      "entropy": 0.1094623792450875,
+      "num_tokens": 5211604.0,
+      "mean_token_accuracy": 0.9680206254124641,
+      "epoch": 1.5512437810945272,
+      "step": 780
+    },
+    {
+      "loss": 0.10809429883956909,
+      "grad_norm": 0.8340095281600952,
+      "learning_rate": 0.00019383107574974984,
+      "entropy": 0.12053416313137859,
+      "num_tokens": 5277697.0,
+      "mean_token_accuracy": 0.9674052610993386,
+      "epoch": 1.5711442786069652,
+      "step": 790
+    },
+    {
+      "loss": 0.08709208965301514,
+      "grad_norm": 0.7644860744476318,
+      "learning_rate": 0.00019360168594539055,
+      "entropy": 0.10000467539066449,
+      "num_tokens": 5342978.0,
+      "mean_token_accuracy": 0.9731013409793376,
+      "epoch": 1.591044776119403,
+      "step": 800
+    },
+    {
+      "loss": 0.09788179397583008,
+      "grad_norm": 0.6640422344207764,
+      "learning_rate": 0.0001933682495477024,
+      "entropy": 0.10800170768052339,
+      "num_tokens": 5410289.0,
+      "mean_token_accuracy": 0.9665102422237396,
+      "epoch": 1.6109452736318408,
+      "step": 810
+    },
+    {
+      "loss": 0.09605536460876465,
+      "grad_norm": 0.6755411624908447,
+      "learning_rate": 0.00019313077664862092,
+      "entropy": 0.10117872587870806,
+      "num_tokens": 5475880.0,
+      "mean_token_accuracy": 0.972070074826479,
+      "epoch": 1.6308457711442785,
+      "step": 820
+    },
+    {
+      "loss": 0.10400503873825073,
+      "grad_norm": 0.890627384185791,
+      "learning_rate": 0.00019288927751458766,
+      "entropy": 0.1069639899302274,
+      "num_tokens": 5541606.0,
+      "mean_token_accuracy": 0.9711074873805046,
+      "epoch": 1.6507462686567163,
+      "step": 830
+    },
+    {
+      "loss": 0.10754516124725341,
+      "grad_norm": 0.5509929656982422,
+      "learning_rate": 0.0001926437625861068,
+      "entropy": 0.11311845399904996,
+      "num_tokens": 5608065.0,
+      "mean_token_accuracy": 0.9695759303867817,
+      "epoch": 1.6706467661691542,
+      "step": 840
+    },
+    {
+      "loss": 0.12257307767868042,
+      "grad_norm": 0.6879218816757202,
+      "learning_rate": 0.00019239424247729345,
+      "entropy": 0.12085098770912736,
+      "num_tokens": 5674308.0,
+      "mean_token_accuracy": 0.9648775070905685,
+      "epoch": 1.6905472636815921,
+      "step": 850
+    },
+    {
+      "loss": 0.10849488973617553,
+      "grad_norm": 0.4920276999473572,
+      "learning_rate": 0.0001921407279754149,
+      "entropy": 0.11081431191414595,
+      "num_tokens": 5742414.0,
+      "mean_token_accuracy": 0.9698534436523915,
+      "epoch": 1.7104477611940299,
+      "step": 860
+    },
+    {
+      "loss": 0.0802489161491394,
+      "grad_norm": 0.7228904366493225,
+      "learning_rate": 0.00019188323004042435,
+      "entropy": 0.0929627066012472,
+      "num_tokens": 5809827.0,
+      "mean_token_accuracy": 0.9730034552514553,
+      "epoch": 1.7303482587064676,
+      "step": 870
+    },
+    {
+      "loss": 0.1109757661819458,
+      "grad_norm": 0.6558980345726013,
+      "learning_rate": 0.00019162175980448688,
+      "entropy": 0.0936126358807087,
+      "num_tokens": 5875172.0,
+      "mean_token_accuracy": 0.9671767763793468,
+      "epoch": 1.7502487562189055,
+      "step": 880
+    },
+    {
+      "loss": 0.09240159988403321,
+      "grad_norm": 0.5792869925498962,
+      "learning_rate": 0.0001913563285714984,
+      "entropy": 0.11744439310859889,
+      "num_tokens": 5942033.0,
+      "mean_token_accuracy": 0.9710902646183968,
+      "epoch": 1.7701492537313432,
+      "step": 890
+    },
+    {
+      "loss": 0.12259334325790405,
+      "grad_norm": 0.8990387320518494,
+      "learning_rate": 0.0001910869478165969,
+      "entropy": 0.10555630044545979,
+      "num_tokens": 6007977.0,
+      "mean_token_accuracy": 0.9644198954105377,
+      "epoch": 1.7900497512437812,
+      "step": 900
+    },
+    {
+      "loss": 0.10547571182250977,
+      "grad_norm": 0.7768874168395996,
+      "learning_rate": 0.00019081362918566618,
+      "entropy": 0.10993905747309327,
+      "num_tokens": 6074057.0,
+      "mean_token_accuracy": 0.9728645481169224,
+      "epoch": 1.809950248756219,
+      "step": 910
+    },
+    {
+      "loss": 0.10611696243286133,
+      "grad_norm": 0.4525396525859833,
+      "learning_rate": 0.00019053638449483259,
+      "entropy": 0.10362131035653874,
+      "num_tokens": 6141345.0,
+      "mean_token_accuracy": 0.9724654078483581,
+      "epoch": 1.8298507462686566,
+      "step": 920
+    },
+    {
+      "loss": 0.11176036596298218,
+      "grad_norm": 0.7492642998695374,
+      "learning_rate": 0.0001902552257299542,
+      "entropy": 0.11784212745260447,
+      "num_tokens": 6209124.0,
+      "mean_token_accuracy": 0.9628825642168521,
+      "epoch": 1.8497512437810946,
+      "step": 930
+    },
+    {
+      "loss": 0.10774084329605102,
+      "grad_norm": 0.5748555064201355,
+      "learning_rate": 0.00018997016504610246,
+      "entropy": 0.11072989953681826,
+      "num_tokens": 6276485.0,
+      "mean_token_accuracy": 0.9646185263991356,
+      "epoch": 1.8696517412935323,
+      "step": 940
+    },
+    {
+      "loss": 0.08848418593406678,
+      "grad_norm": 0.5937514305114746,
+      "learning_rate": 0.00018968121476703678,
+      "entropy": 0.10556984411086887,
+      "num_tokens": 6342957.0,
+      "mean_token_accuracy": 0.9734670996665955,
+      "epoch": 1.8895522388059702,
+      "step": 950
+    },
+    {
+      "loss": 0.10269320011138916,
+      "grad_norm": 1.0715358257293701,
+      "learning_rate": 0.00018938838738467184,
+      "entropy": 0.09428299731807784,
+      "num_tokens": 6408918.0,
+      "mean_token_accuracy": 0.9720249362289906,
+      "epoch": 1.909452736318408,
+      "step": 960
+    },
+    {
+      "loss": 0.0899561107158661,
+      "grad_norm": 0.5169550180435181,
+      "learning_rate": 0.00018909169555853743,
+      "entropy": 0.10850229405332357,
+      "num_tokens": 6475062.0,
+      "mean_token_accuracy": 0.9714486353099346,
+      "epoch": 1.9293532338308457,
+      "step": 970
+    },
+    {
+      "loss": 0.09649609923362731,
+      "grad_norm": 0.7642938494682312,
+      "learning_rate": 0.00018879115211523117,
+      "entropy": 0.1026058561168611,
+      "num_tokens": 6542465.0,
+      "mean_token_accuracy": 0.9703472301363945,
+      "epoch": 1.9492537313432836,
+      "step": 980
+    },
+    {
+      "loss": 0.09067035913467407,
+      "grad_norm": 0.3874703347682953,
+      "learning_rate": 0.0001884867700478641,
+      "entropy": 0.09136548589449375,
+      "num_tokens": 6608985.0,
+      "mean_token_accuracy": 0.9718083783984184,
+      "epoch": 1.9691542288557216,
+      "step": 990
+    },
+    {
+      "loss": 0.09239903688430787,
+      "grad_norm": 0.693695604801178,
+      "learning_rate": 0.00018817856251549867,
+      "entropy": 0.11761876428499818,
+      "num_tokens": 6676885.0,
+      "mean_token_accuracy": 0.9674227833747864,
+      "epoch": 1.9890547263681593,
+      "step": 1000
+    },
+    {
+      "eval_loss": 0.24770455062389374,
+      "eval_runtime": 67.3665,
+      "eval_samples_per_second": 15.364,
+      "eval_steps_per_second": 7.689,
+      "eval_entropy": 0.13267684354209502,
+      "eval_num_tokens": 6713972.0,
+      "eval_mean_token_accuracy": 0.9410935246806347,
+      "epoch": 2.0,
+      "step": 1006
+    },
+    {
+      "loss": 0.08439697623252869,
+      "grad_norm": 0.41609451174736023,
+      "learning_rate": 0.00018786654284258034,
+      "entropy": 0.09164438765545033,
+      "num_tokens": 6740189.0,
+      "mean_token_accuracy": 0.9744413871514169,
+      "epoch": 2.007960199004975,
+      "step": 1010
+    },
+    {
+      "loss": 0.06379352807998658,
+      "grad_norm": 0.7209794521331787,
+      "learning_rate": 0.00018755072451836097,
+      "entropy": 0.06243965101893991,
+      "num_tokens": 6806511.0,
+      "mean_token_accuracy": 0.9806746728718281,
+      "epoch": 2.027860696517413,
+      "step": 1020
+    },
+    {
+      "loss": 0.07251286506652832,
+      "grad_norm": 0.4035630524158478,
+      "learning_rate": 0.00018723112119631608,
+      "entropy": 0.06880665038479492,
+      "num_tokens": 6874726.0,
+      "mean_token_accuracy": 0.9811282232403755,
+      "epoch": 2.047761194029851,
+      "step": 1030
+    },
+    {
+      "loss": 0.05488339066505432,
+      "grad_norm": 0.6395288705825806,
+      "learning_rate": 0.00018690774669355442,
+      "entropy": 0.07282297083875164,
+      "num_tokens": 6940725.0,
+      "mean_token_accuracy": 0.9807328015565873,
+      "epoch": 2.0676616915422885,
+      "step": 1040
+    },
+    {
+      "loss": 0.06734220385551452,
+      "grad_norm": 0.7077370882034302,
+      "learning_rate": 0.00018658061499022055,
+      "entropy": 0.07170078799827025,
+      "num_tokens": 7008850.0,
+      "mean_token_accuracy": 0.9810515813529491,
+      "epoch": 2.0875621890547262,
+      "step": 1050
+    },
+    {
+      "loss": 0.062459665536880496,
+      "grad_norm": 1.220231294631958,
+      "learning_rate": 0.0001862497402288906,
+      "entropy": 0.05807271180674434,
+      "num_tokens": 7074982.0,
+      "mean_token_accuracy": 0.981388358771801,
+      "epoch": 2.107462686567164,
+      "step": 1060
+    },
+    {
+      "loss": 0.046548599004745485,
+      "grad_norm": 0.5530555248260498,
+      "learning_rate": 0.0001859151367139608,
+      "entropy": 0.06504726539133117,
+      "num_tokens": 7140914.0,
+      "mean_token_accuracy": 0.984452311694622,
+      "epoch": 2.127363184079602,
+      "step": 1070
+    },
+    {
+      "loss": 0.05819639563560486,
+      "grad_norm": 0.7002009749412537,
+      "learning_rate": 0.000185576818911029,
+      "entropy": 0.050316512072458866,
+      "num_tokens": 7206800.0,
+      "mean_token_accuracy": 0.985405495017767,
+      "epoch": 2.14726368159204,
+      "step": 1080
+    },
+    {
+      "loss": 0.0601584792137146,
+      "grad_norm": 0.540239691734314,
+      "learning_rate": 0.00018523480144626948,
+      "entropy": 0.07835423464421183,
+      "num_tokens": 7273522.0,
+      "mean_token_accuracy": 0.9818796373903751,
+      "epoch": 2.1671641791044776,
+      "step": 1090
+    },
+    {
+      "loss": 0.04493230581283569,
+      "grad_norm": 0.5114269256591797,
+      "learning_rate": 0.00018488909910580037,
+      "entropy": 0.05474828036967665,
+      "num_tokens": 7340979.0,
+      "mean_token_accuracy": 0.9857458151876927,
+      "epoch": 2.1870646766169153,
+      "step": 1100
+    },
+    {
+      "loss": 0.054016536474227904,
+      "grad_norm": 0.6734052300453186,
+      "learning_rate": 0.00018453972683504466,
+      "entropy": 0.05471267879474908,
+      "num_tokens": 7407885.0,
+      "mean_token_accuracy": 0.9807157158851624,
+      "epoch": 2.206965174129353,
+      "step": 1110
+    },
+    {
+      "loss": 0.06644362211227417,
+      "grad_norm": 1.0557215213775635,
+      "learning_rate": 0.00018418669973808386,
+      "entropy": 0.07309502450516447,
+      "num_tokens": 7476921.0,
+      "mean_token_accuracy": 0.9793836884200573,
+      "epoch": 2.226865671641791,
+      "step": 1120
+    },
+    {
+      "loss": 0.06655114293098449,
+      "grad_norm": 0.5192980170249939,
+      "learning_rate": 0.00018383003307700525,
+      "entropy": 0.06608295071637257,
+      "num_tokens": 7543462.0,
+      "mean_token_accuracy": 0.9790004834532737,
+      "epoch": 2.246766169154229,
+      "step": 1130
+    },
+    {
+      "loss": 0.06803213357925415,
+      "grad_norm": 0.4396572709083557,
+      "learning_rate": 0.0001834697422712419,
+      "entropy": 0.06500887103029526,
+      "num_tokens": 7610642.0,
+      "mean_token_accuracy": 0.980684906244278,
+      "epoch": 2.2666666666666666,
+      "step": 1140
+    },
+    {
+      "loss": 0.05774785876274109,
+      "grad_norm": 0.8334233164787292,
+      "learning_rate": 0.00018310584289690608,
+      "entropy": 0.058826766291167586,
+      "num_tokens": 7676890.0,
+      "mean_token_accuracy": 0.9823732137680053,
+      "epoch": 2.2865671641791043,
+      "step": 1150
+    },
+    {
+      "loss": 0.05890558958053589,
+      "grad_norm": 0.6917388439178467,
+      "learning_rate": 0.0001827383506861159,
+      "entropy": 0.0589894114760682,
+      "num_tokens": 7743110.0,
+      "mean_token_accuracy": 0.9807198375463486,
+      "epoch": 2.306467661691542,
+      "step": 1160
+    },
+    {
+      "loss": 0.04957199692726135,
+      "grad_norm": 0.351596862077713,
+      "learning_rate": 0.00018236728152631526,
+      "entropy": 0.05655237181927077,
+      "num_tokens": 7809971.0,
+      "mean_token_accuracy": 0.9838769190013409,
+      "epoch": 2.32636815920398,
+      "step": 1170
+    },
+    {
+      "loss": 0.07587432265281677,
+      "grad_norm": 0.4856368899345398,
+      "learning_rate": 0.00018199265145958678,
+      "entropy": 0.06457642229506746,
+      "num_tokens": 7876231.0,
+      "mean_token_accuracy": 0.974904265254736,
+      "epoch": 2.346268656716418,
+      "step": 1180
+    },
+    {
+      "loss": 0.057352250814437865,
+      "grad_norm": 0.6313260197639465,
+      "learning_rate": 0.00018161447668195858,
+      "entropy": 0.07011734971310943,
+      "num_tokens": 7942463.0,
+      "mean_token_accuracy": 0.9822526164352894,
+      "epoch": 2.3661691542288557,
+      "step": 1190
+    },
+    {
+      "loss": 0.07296563386917114,
+      "grad_norm": 0.6475229263305664,
+      "learning_rate": 0.00018123277354270372,
+      "entropy": 0.07394456524634734,
+      "num_tokens": 8008529.0,
+      "mean_token_accuracy": 0.9783020555973053,
+      "epoch": 2.3860696517412934,
+      "step": 1200
+    },
+    {
+      "loss": 0.06899880170822144,
+      "grad_norm": 0.5311662554740906,
+      "learning_rate": 0.00018084755854363377,
+      "entropy": 0.06762066348455846,
+      "num_tokens": 8075331.0,
+      "mean_token_accuracy": 0.9800443604588509,
+      "epoch": 2.405970149253731,
+      "step": 1210
+    },
+    {
+      "loss": 0.04785624444484711,
+      "grad_norm": 0.47831037640571594,
+      "learning_rate": 0.00018045884833838512,
+      "entropy": 0.052969011454842986,
+      "num_tokens": 8140043.0,
+      "mean_token_accuracy": 0.9850305773317813,
+      "epoch": 2.4258706467661693,
+      "step": 1220
+    },
+    {
+      "loss": 0.057527285814285276,
+      "grad_norm": 0.4341302216053009,
+      "learning_rate": 0.00018006665973169911,
+      "entropy": 0.058878783753607424,
+      "num_tokens": 8206439.0,
+      "mean_token_accuracy": 0.9826890744268895,
+      "epoch": 2.445771144278607,
+      "step": 1230
+    },
+    {
+      "loss": 0.06314617991447449,
+      "grad_norm": 0.6121116876602173,
+      "learning_rate": 0.0001796710096786956,
+      "entropy": 0.06802375400438905,
+      "num_tokens": 8274350.0,
+      "mean_token_accuracy": 0.9803676478564739,
+      "epoch": 2.4656716417910447,
+      "step": 1240
+    },
+    {
+      "loss": 0.06138876676559448,
+      "grad_norm": 0.485624760389328,
+      "learning_rate": 0.0001792719152841398,
+      "entropy": 0.06864131800248288,
+      "num_tokens": 8342676.0,
+      "mean_token_accuracy": 0.9821898102760315,
+      "epoch": 2.4855721393034824,
+      "step": 1250
+    },
+    {
+      "loss": 0.05479460954666138,
+      "grad_norm": 0.7229267358779907,
+      "learning_rate": 0.0001788693938017029,
+      "entropy": 0.06992838233709335,
+      "num_tokens": 8408688.0,
+      "mean_token_accuracy": 0.9806355632841587,
+      "epoch": 2.50547263681592,
+      "step": 1260
+    },
+    {
+      "loss": 0.05725674629211426,
+      "grad_norm": 0.6357669234275818,
+      "learning_rate": 0.00017846346263321623,
+      "entropy": 0.05212859932216816,
+      "num_tokens": 8475857.0,
+      "mean_token_accuracy": 0.982030725479126,
+      "epoch": 2.5253731343283583,
+      "step": 1270
+    },
+    {
+      "loss": 0.05330604910850525,
+      "grad_norm": 0.48565125465393066,
+      "learning_rate": 0.00017805413932791875,
+      "entropy": 0.0598757246916648,
+      "num_tokens": 8542007.0,
+      "mean_token_accuracy": 0.9829342268407345,
+      "epoch": 2.545273631840796,
+      "step": 1280
+    },
+    {
+      "loss": 0.0662715494632721,
+      "grad_norm": 0.47484299540519714,
+      "learning_rate": 0.00017764144158169856,
+      "entropy": 0.07917212916072458,
+      "num_tokens": 8609409.0,
+      "mean_token_accuracy": 0.9773714534938336,
+      "epoch": 2.5651741293532337,
+      "step": 1290
+    },
+    {
+      "loss": 0.06932964324951171,
+      "grad_norm": 0.7906941175460815,
+      "learning_rate": 0.00017722538723632773,
+      "entropy": 0.0599730534479022,
+      "num_tokens": 8676488.0,
+      "mean_token_accuracy": 0.9792346425354481,
+      "epoch": 2.585074626865672,
+      "step": 1300
+    },
+    {
+      "loss": 0.08003131747245788,
+      "grad_norm": 0.5393727421760559,
+      "learning_rate": 0.000176805994278691,
+      "entropy": 0.07814967349986546,
+      "num_tokens": 8744553.0,
+      "mean_token_accuracy": 0.9784729719161988,
+      "epoch": 2.604975124378109,
+      "step": 1310
+    },
+    {
+      "loss": 0.051309889554977416,
+      "grad_norm": 0.8460810780525208,
+      "learning_rate": 0.0001763832808400082,
+      "entropy": 0.06959494983311743,
+      "num_tokens": 8812013.0,
+      "mean_token_accuracy": 0.9829349182546139,
+      "epoch": 2.6248756218905474,
+      "step": 1320
+    },
+    {
+      "loss": 0.05681629776954651,
+      "grad_norm": 0.5110874176025391,
+      "learning_rate": 0.00017595726519505043,
+      "entropy": 0.06371988009777851,
+      "num_tokens": 8879275.0,
+      "mean_token_accuracy": 0.9808239601552486,
+      "epoch": 2.644776119402985,
+      "step": 1330
+    },
+    {
+      "loss": 0.06473686695098876,
+      "grad_norm": 0.45770350098609924,
+      "learning_rate": 0.00017552796576134985,
+      "entropy": 0.06574244437506423,
+      "num_tokens": 8946437.0,
+      "mean_token_accuracy": 0.9830326803028584,
+      "epoch": 2.664676616915423,
+      "step": 1340
+    },
+    {
+      "loss": 0.05621873736381531,
+      "grad_norm": 0.47769930958747864,
+      "learning_rate": 0.00017509540109840365,
+      "entropy": 0.06558069243910722,
+      "num_tokens": 9013311.0,
+      "mean_token_accuracy": 0.9796665169298648,
+      "epoch": 2.684577114427861,
+      "step": 1350
+    },
+    {
+      "loss": 0.05615652799606323,
+      "grad_norm": 0.48142921924591064,
+      "learning_rate": 0.00017465958990687156,
+      "entropy": 0.059144589549396186,
+      "num_tokens": 9078260.0,
+      "mean_token_accuracy": 0.9777877710759639,
+      "epoch": 2.7044776119402982,
+      "step": 1360
+    },
+    {
+      "loss": 0.07301256060600281,
+      "grad_norm": 0.6959577798843384,
+      "learning_rate": 0.0001742205510277674,
+      "entropy": 0.0731184652308002,
+      "num_tokens": 9143896.0,
+      "mean_token_accuracy": 0.9792239956557751,
+      "epoch": 2.7243781094527364,
+      "step": 1370
+    },
+    {
+      "loss": 0.05728998184204102,
+      "grad_norm": 0.4535721242427826,
+      "learning_rate": 0.00017377830344164464,
+      "entropy": 0.06650992162758484,
+      "num_tokens": 9210455.0,
+      "mean_token_accuracy": 0.9824736349284648,
+      "epoch": 2.744278606965174,
+      "step": 1380
+    },
+    {
+      "loss": 0.05624777674674988,
+      "grad_norm": 0.9010443687438965,
+      "learning_rate": 0.00017333286626777564,
+      "entropy": 0.06068479290697724,
+      "num_tokens": 9278223.0,
+      "mean_token_accuracy": 0.9795115493237972,
+      "epoch": 2.764179104477612,
+      "step": 1390
+    },
+    {
+      "loss": 0.04648939669132233,
+      "grad_norm": 0.5129737257957458,
+      "learning_rate": 0.00017288425876332527,
+      "entropy": 0.05758373744320124,
+      "num_tokens": 9346259.0,
+      "mean_token_accuracy": 0.9854638859629631,
+      "epoch": 2.78407960199005,
+      "step": 1400
+    },
+    {
+      "loss": 0.07454426288604736,
+      "grad_norm": 1.081714153289795,
+      "learning_rate": 0.00017243250032251823,
+      "entropy": 0.07305689085042104,
+      "num_tokens": 9411960.0,
+      "mean_token_accuracy": 0.9747657172381878,
+      "epoch": 2.8039800995024877,
+      "step": 1410
+    },
+    {
+      "loss": 0.052061259746551514,
+      "grad_norm": 0.668654203414917,
+      "learning_rate": 0.00017197761047580082,
+      "entropy": 0.05686979314778,
+      "num_tokens": 9479728.0,
+      "mean_token_accuracy": 0.9826227888464928,
+      "epoch": 2.8238805970149254,
+      "step": 1420
+    },
+    {
+      "loss": 0.061662870645523074,
+      "grad_norm": 0.6690983772277832,
+      "learning_rate": 0.00017151960888899627,
+      "entropy": 0.06015237307874486,
+      "num_tokens": 9544970.0,
+      "mean_token_accuracy": 0.9811851166188716,
+      "epoch": 2.843781094527363,
+      "step": 1430
+    },
+    {
+      "loss": 0.05701953172683716,
+      "grad_norm": 0.7326856255531311,
+      "learning_rate": 0.00017105851536245492,
+      "entropy": 0.07333141873823479,
+      "num_tokens": 9611720.0,
+      "mean_token_accuracy": 0.9832665704190731,
+      "epoch": 2.863681592039801,
+      "step": 1440
+    },
+    {
+      "loss": 0.07263015508651734,
+      "grad_norm": 0.7834051251411438,
+      "learning_rate": 0.0001705943498301979,
+      "entropy": 0.06394668611465022,
+      "num_tokens": 9679831.0,
+      "mean_token_accuracy": 0.9781429409980774,
+      "epoch": 2.883582089552239,
+      "step": 1450
+    },
+    {
+      "loss": 0.05510892271995545,
+      "grad_norm": 0.6546643972396851,
+      "learning_rate": 0.00017012713235905547,
+      "entropy": 0.06291348003433087,
+      "num_tokens": 9745899.0,
+      "mean_token_accuracy": 0.9804500080645084,
+      "epoch": 2.9034825870646768,
+      "step": 1460
+    },
+    {
+      "loss": 0.05505146384239197,
+      "grad_norm": 0.5361804962158203,
+      "learning_rate": 0.00016965688314779956,
+      "entropy": 0.06303218469838612,
+      "num_tokens": 9814519.0,
+      "mean_token_accuracy": 0.9801669403910637,
+      "epoch": 2.9233830845771145,
+      "step": 1470
+    },
+    {
+      "loss": 0.04610788822174072,
+      "grad_norm": 0.6162955164909363,
+      "learning_rate": 0.00016918362252627036,
+      "entropy": 0.0545346099184826,
+      "num_tokens": 9878812.0,
+      "mean_token_accuracy": 0.9822937592864036,
+      "epoch": 2.943283582089552,
+      "step": 1480
+    },
+    {
+      "loss": 0.06813795566558838,
+      "grad_norm": 0.8021186590194702,
+      "learning_rate": 0.00016870737095449754,
+      "entropy": 0.06529987451503985,
+      "num_tokens": 9946556.0,
+      "mean_token_accuracy": 0.9782890357077122,
+      "epoch": 2.96318407960199,
+      "step": 1490
+    },
+    {
+      "loss": 0.05245916843414307,
+      "grad_norm": 0.6037353277206421,
+      "learning_rate": 0.00016822814902181583,
+      "entropy": 0.06525841613765807,
+      "num_tokens": 10014536.0,
+      "mean_token_accuracy": 0.9829807795584202,
+      "epoch": 2.983084577114428,
+      "step": 1500
+    },
+    {
+      "eval_loss": 0.26412370800971985,
+      "eval_runtime": 67.3621,
+      "eval_samples_per_second": 15.365,
+      "eval_steps_per_second": 7.69,
+      "eval_entropy": 0.10301580357465993,
+      "eval_num_tokens": 10070958.0,
+      "eval_mean_token_accuracy": 0.9438181870462351,
+      "epoch": 3.0,
+      "step": 1509
+    },
+    {
+      "loss": 0.05833644866943359,
+      "grad_norm": 0.29880842566490173,
+      "learning_rate": 0.00016774597744597457,
+      "entropy": 0.053184559752576445,
+      "num_tokens": 10077424.0,
+      "mean_token_accuracy": 0.9848840275877401,
+      "epoch": 3.0019900497512437,
+      "step": 1510
+    },
+    {
+      "loss": 0.03773494362831116,
+      "grad_norm": 0.5901796817779541,
+      "learning_rate": 0.00016726087707224236,
+      "entropy": 0.04916581161960494,
+      "num_tokens": 10143828.0,
+      "mean_token_accuracy": 0.9888612225651741,
+      "epoch": 3.0218905472636814,
+      "step": 1520
+    },
+    {
+      "loss": 0.04334467053413391,
+      "grad_norm": 0.4549998342990875,
+      "learning_rate": 0.00016677286887250572,
+      "entropy": 0.04599970751442015,
+      "num_tokens": 10210498.0,
+      "mean_token_accuracy": 0.9884389974176884,
+      "epoch": 3.0417910447761196,
+      "step": 1530
+    },
+    {
+      "loss": 0.031511181592941286,
+      "grad_norm": 0.41420701146125793,
+      "learning_rate": 0.00016628197394436247,
+      "entropy": 0.03852141368552111,
+      "num_tokens": 10275990.0,
+      "mean_token_accuracy": 0.9884374618530274,
+      "epoch": 3.0616915422885573,
+      "step": 1540
+    },
+    {
+      "loss": 0.029081150889396667,
+      "grad_norm": 0.3844093978404999,
+      "learning_rate": 0.00016578821351020964,
+      "entropy": 0.032692909514298665,
+      "num_tokens": 10342514.0,
+      "mean_token_accuracy": 0.9907526269555091,
+      "epoch": 3.081592039800995,
+      "step": 1550
+    },
+    {
+      "loss": 0.037613070011138915,
+      "grad_norm": 0.3342166244983673,
+      "learning_rate": 0.00016529160891632597,
+      "entropy": 0.03535493408198818,
+      "num_tokens": 10408828.0,
+      "mean_token_accuracy": 0.9882513448596001,
+      "epoch": 3.1014925373134328,
+      "step": 1560
+    },
+    {
+      "loss": 0.03288332223892212,
+      "grad_norm": 0.5369789600372314,
+      "learning_rate": 0.00016479218163194904,
+      "entropy": 0.038720946523244495,
+      "num_tokens": 10475303.0,
+      "mean_token_accuracy": 0.9900217793881894,
+      "epoch": 3.1213930348258705,
+      "step": 1570
+    },
+    {
+      "loss": 0.03281024694442749,
+      "grad_norm": 0.3687148094177246,
+      "learning_rate": 0.00016428995324834723,
+      "entropy": 0.03382732793106698,
+      "num_tokens": 10543012.0,
+      "mean_token_accuracy": 0.9898124732077122,
+      "epoch": 3.1412935323383087,
+      "step": 1580
+    },
+    {
+      "loss": 0.032737156748771666,
+      "grad_norm": 0.40983688831329346,
+      "learning_rate": 0.00016378494547788613,
+      "entropy": 0.04149121846421622,
+      "num_tokens": 10609348.0,
+      "mean_token_accuracy": 0.9896206237375736,
+      "epoch": 3.1611940298507464,
+      "step": 1590
+    },
+    {
+      "loss": 0.04040493667125702,
+      "grad_norm": 0.6384645104408264,
+      "learning_rate": 0.00016327718015308998,
+      "entropy": 0.04162978534004651,
+      "num_tokens": 10676433.0,
+      "mean_token_accuracy": 0.987835768610239,
+      "epoch": 3.181094527363184,
+      "step": 1600
+    },
+    {
+      "loss": 0.03527785539627075,
+      "grad_norm": 0.7771252989768982,
+      "learning_rate": 0.0001627666792256977,
+      "entropy": 0.0402049986703787,
+      "num_tokens": 10744880.0,
+      "mean_token_accuracy": 0.9879353500902652,
+      "epoch": 3.200995024875622,
+      "step": 1610
+    },
+    {
+      "loss": 0.027459162473678588,
+      "grad_norm": 0.7088900208473206,
+      "learning_rate": 0.00016225346476571396,
+      "entropy": 0.03322969185537659,
+      "num_tokens": 10811704.0,
+      "mean_token_accuracy": 0.9915633283555507,
+      "epoch": 3.2208955223880595,
+      "step": 1620
+    },
+    {
+      "loss": 0.04761128127574921,
+      "grad_norm": 0.5046206712722778,
+      "learning_rate": 0.00016173755896045506,
+      "entropy": 0.04271038012811914,
+      "num_tokens": 10877771.0,
+      "mean_token_accuracy": 0.9852734059095383,
+      "epoch": 3.2407960199004977,
+      "step": 1630
+    },
+    {
+      "loss": 0.040560868382453916,
+      "grad_norm": 0.5330039262771606,
+      "learning_rate": 0.00016121898411358966,
+      "entropy": 0.056298443174455315,
+      "num_tokens": 10946056.0,
+      "mean_token_accuracy": 0.9860941573977471,
+      "epoch": 3.2606965174129354,
+      "step": 1640
+    },
+    {
+      "loss": 0.038580065965652464,
+      "grad_norm": 0.3763498067855835,
+      "learning_rate": 0.00016069776264417463,
+      "entropy": 0.040748245036229494,
+      "num_tokens": 11014117.0,
+      "mean_token_accuracy": 0.9879476867616177,
+      "epoch": 3.280597014925373,
+      "step": 1650
+    },
+    {
+      "loss": 0.031229573488235473,
+      "grad_norm": 0.5975865721702576,
+      "learning_rate": 0.00016017391708568563,
+      "entropy": 0.04134431722341105,
+      "num_tokens": 11080721.0,
+      "mean_token_accuracy": 0.9899743214249611,
+      "epoch": 3.300497512437811,
+      "step": 1660
+    },
+    {
+      "loss": 0.04345620274543762,
+      "grad_norm": 0.5342262387275696,
+      "learning_rate": 0.00015964747008504325,
+      "entropy": 0.03906176597811282,
+      "num_tokens": 11147441.0,
+      "mean_token_accuracy": 0.98561105504632,
+      "epoch": 3.3203980099502486,
+      "step": 1670
+    },
+    {
+      "loss": 0.042844048142433165,
+      "grad_norm": 0.4553395211696625,
+      "learning_rate": 0.00015911844440163371,
+      "entropy": 0.043423575052293016,
+      "num_tokens": 11212240.0,
+      "mean_token_accuracy": 0.9864377163350582,
+      "epoch": 3.3402985074626868,
+      "step": 1680
+    },
+    {
+      "loss": 0.029976919293403625,
+      "grad_norm": 0.7629397511482239,
+      "learning_rate": 0.00015858686290632493,
+      "entropy": 0.04001676997286267,
+      "num_tokens": 11277138.0,
+      "mean_token_accuracy": 0.9910943493247032,
+      "epoch": 3.3601990049751245,
+      "step": 1690
+    },
+    {
+      "loss": 0.03644349873065948,
+      "grad_norm": 0.5027875304222107,
+      "learning_rate": 0.0001580527485804779,
+      "entropy": 0.037145845251507124,
+      "num_tokens": 11343926.0,
+      "mean_token_accuracy": 0.9864401429891586,
+      "epoch": 3.380099502487562,
+      "step": 1700
+    },
+    {
+      "loss": 0.030204242467880248,
+      "grad_norm": 0.3141781687736511,
+      "learning_rate": 0.00015751612451495313,
+      "entropy": 0.03934923965134658,
+      "num_tokens": 11411256.0,
+      "mean_token_accuracy": 0.9906957238912583,
+      "epoch": 3.4,
+      "step": 1710
+    },
+    {
+      "loss": 0.03447907567024231,
+      "grad_norm": 0.2602730989456177,
+      "learning_rate": 0.00015697701390911218,
+      "entropy": 0.03915706583939027,
+      "num_tokens": 11477632.0,
+      "mean_token_accuracy": 0.9882164128124714,
+      "epoch": 3.4199004975124376,
+      "step": 1720
+    },
+    {
+      "loss": 0.03258863389492035,
+      "grad_norm": 0.5404685735702515,
+      "learning_rate": 0.00015643544006981505,
+      "entropy": 0.04417289613047615,
+      "num_tokens": 11542182.0,
+      "mean_token_accuracy": 0.9894180931150913,
+      "epoch": 3.439800995024876,
+      "step": 1730
+    },
+    {
+      "loss": 0.03976099193096161,
+      "grad_norm": 0.4809146821498871,
+      "learning_rate": 0.0001558914264104122,
+      "entropy": 0.039228807145264,
+      "num_tokens": 11608358.0,
+      "mean_token_accuracy": 0.9867617316544056,
+      "epoch": 3.4597014925373135,
+      "step": 1740
+    },
+    {
+      "loss": 0.03436765670776367,
+      "grad_norm": 0.7439927458763123,
+      "learning_rate": 0.00015534499644973267,
+      "entropy": 0.03574614835088141,
+      "num_tokens": 11675284.0,
+      "mean_token_accuracy": 0.9876792296767235,
+      "epoch": 3.4796019900497512,
+      "step": 1750
+    },
+    {
+      "loss": 0.038963159918785094,
+      "grad_norm": 1.2817922830581665,
+      "learning_rate": 0.00015479617381106711,
+      "entropy": 0.04119858265621588,
+      "num_tokens": 11743101.0,
+      "mean_token_accuracy": 0.9876352168619633,
+      "epoch": 3.499502487562189,
+      "step": 1760
+    },
+    {
+      "loss": 0.031986075639724734,
+      "grad_norm": 0.5344582796096802,
+      "learning_rate": 0.00015424498222114662,
+      "entropy": 0.039489572704769674,
+      "num_tokens": 11811048.0,
+      "mean_token_accuracy": 0.9890580669045448,
+      "epoch": 3.5194029850746267,
+      "step": 1770
+    },
+    {
+      "loss": 0.043090081214904784,
+      "grad_norm": 0.3375915288925171,
+      "learning_rate": 0.00015369144550911678,
+      "entropy": 0.039671140746213494,
+      "num_tokens": 11879773.0,
+      "mean_token_accuracy": 0.9859576515853405,
+      "epoch": 3.539303482587065,
+      "step": 1780
+    },
+    {
+      "loss": 0.03574670851230621,
+      "grad_norm": 0.5646845698356628,
+      "learning_rate": 0.0001531355876055078,
+      "entropy": 0.04234636231994955,
+      "num_tokens": 11946882.0,
+      "mean_token_accuracy": 0.9891765132546425,
+      "epoch": 3.5592039800995026,
+      "step": 1790
+    },
+    {
+      "loss": 0.03617365062236786,
+      "grad_norm": 0.5281543731689453,
+      "learning_rate": 0.00015257743254119973,
+      "entropy": 0.03838658424501773,
+      "num_tokens": 12014198.0,
+      "mean_token_accuracy": 0.9879163481295109,
+      "epoch": 3.5791044776119403,
+      "step": 1800
+    },
+    {
+      "loss": 0.02715664207935333,
+      "grad_norm": 0.4327303171157837,
+      "learning_rate": 0.00015201700444638348,
+      "entropy": 0.035956174423336054,
+      "num_tokens": 12080056.0,
+      "mean_token_accuracy": 0.9888209789991379,
+      "epoch": 3.599004975124378,
+      "step": 1810
+    },
+    {
+      "loss": 0.03283187747001648,
+      "grad_norm": 0.6245374083518982,
+      "learning_rate": 0.00015145432754951784,
+      "entropy": 0.03530628806038294,
+      "num_tokens": 12146900.0,
+      "mean_token_accuracy": 0.9889325089752674,
+      "epoch": 3.6189054726368157,
+      "step": 1820
+    },
+    {
+      "loss": 0.03508978486061096,
+      "grad_norm": 0.8539674878120422,
+      "learning_rate": 0.000150889426176282,
+      "entropy": 0.03957471833855379,
+      "num_tokens": 12213024.0,
+      "mean_token_accuracy": 0.9876590810716153,
+      "epoch": 3.638805970149254,
+      "step": 1830
+    },
+    {
+      "loss": 0.0289535254240036,
+      "grad_norm": 0.44117090106010437,
+      "learning_rate": 0.00015032232474852371,
+      "entropy": 0.03894171481369994,
+      "num_tokens": 12278205.0,
+      "mean_token_accuracy": 0.9898740701377392,
+      "epoch": 3.6587064676616916,
+      "step": 1840
+    },
+    {
+      "loss": 0.03545762598514557,
+      "grad_norm": 0.5383766889572144,
+      "learning_rate": 0.00014975304778320364,
+      "entropy": 0.04185012882517185,
+      "num_tokens": 12346281.0,
+      "mean_token_accuracy": 0.987272110581398,
+      "epoch": 3.6786069651741293,
+      "step": 1850
+    },
+    {
+      "loss": 0.036569598317146304,
+      "grad_norm": 0.43862003087997437,
+      "learning_rate": 0.00014918161989133552,
+      "entropy": 0.039183757436694576,
+      "num_tokens": 12416040.0,
+      "mean_token_accuracy": 0.9884304039180278,
+      "epoch": 3.698507462686567,
+      "step": 1860
+    },
+    {
+      "loss": 0.046497902274131774,
+      "grad_norm": 0.24663235247135162,
+      "learning_rate": 0.0001486080657769219,
+      "entropy": 0.04875118255149573,
+      "num_tokens": 12483739.0,
+      "mean_token_accuracy": 0.9872626729309559,
+      "epoch": 3.7184079601990048,
+      "step": 1870
+    },
+    {
+      "loss": 0.0350829690694809,
+      "grad_norm": 0.27893203496932983,
+      "learning_rate": 0.00014803241023588637,
+      "entropy": 0.039400185630074705,
+      "num_tokens": 12550264.0,
+      "mean_token_accuracy": 0.9897375635802745,
+      "epoch": 3.738308457711443,
+      "step": 1880
+    },
+    {
+      "loss": 0.048194342851638795,
+      "grad_norm": 0.5404137372970581,
+      "learning_rate": 0.00014745467815500157,
+      "entropy": 0.045675123430555686,
+      "num_tokens": 12615635.0,
+      "mean_token_accuracy": 0.9822327814996242,
+      "epoch": 3.7582089552238807,
+      "step": 1890
+    },
+    {
+      "loss": 0.041299638152122495,
+      "grad_norm": 0.22043873369693756,
+      "learning_rate": 0.0001468748945108131,
+      "entropy": 0.0417529508471489,
+      "num_tokens": 12682762.0,
+      "mean_token_accuracy": 0.9876552060246467,
+      "epoch": 3.7781094527363184,
+      "step": 1900
+    },
+    {
+      "loss": 0.03528775572776795,
+      "grad_norm": 0.38801971077919006,
+      "learning_rate": 0.00014629308436856,
+      "entropy": 0.0380144338007085,
+      "num_tokens": 12749704.0,
+      "mean_token_accuracy": 0.988120187819004,
+      "epoch": 3.798009950248756,
+      "step": 1910
+    },
+    {
+      "loss": 0.028483673930168152,
+      "grad_norm": 0.7580603957176208,
+      "learning_rate": 0.0001457092728810909,
+      "entropy": 0.036843240825692194,
+      "num_tokens": 12815360.0,
+      "mean_token_accuracy": 0.9913376808166504,
+      "epoch": 3.817910447761194,
+      "step": 1920
+    },
+    {
+      "loss": 0.034877949953079225,
+      "grad_norm": 0.41240543127059937,
+      "learning_rate": 0.00014512348528777675,
+      "entropy": 0.03646660551312379,
+      "num_tokens": 12883173.0,
+      "mean_token_accuracy": 0.991103533655405,
+      "epoch": 3.837810945273632,
+      "step": 1930
+    },
+    {
+      "loss": 0.02729986608028412,
+      "grad_norm": 0.431193470954895,
+      "learning_rate": 0.00014453574691341957,
+      "entropy": 0.03418905301514315,
+      "num_tokens": 12949048.0,
+      "mean_token_accuracy": 0.9920941665768623,
+      "epoch": 3.8577114427860697,
+      "step": 1940
+    },
+    {
+      "loss": 0.03583741784095764,
+      "grad_norm": 0.8341835737228394,
+      "learning_rate": 0.00014394608316715764,
+      "entropy": 0.0317019106762018,
+      "num_tokens": 13017021.0,
+      "mean_token_accuracy": 0.9879241831600666,
+      "epoch": 3.8776119402985074,
+      "step": 1950
+    },
+    {
+      "loss": 0.04165915548801422,
+      "grad_norm": 0.54258793592453,
+      "learning_rate": 0.00014335451954136712,
+      "entropy": 0.04781383575173095,
+      "num_tokens": 13084247.0,
+      "mean_token_accuracy": 0.9850596696138382,
+      "epoch": 3.897512437810945,
+      "step": 1960
+    },
+    {
+      "loss": 0.023919902741909027,
+      "grad_norm": 1.1518906354904175,
+      "learning_rate": 0.00014276108161055977,
+      "entropy": 0.03323271934641525,
+      "num_tokens": 13149886.0,
+      "mean_token_accuracy": 0.9917417526245117,
+      "epoch": 3.917412935323383,
+      "step": 1970
+    },
+    {
+      "loss": 0.041414502263069156,
+      "grad_norm": 0.4409993588924408,
+      "learning_rate": 0.00014216579503027748,
+      "entropy": 0.04212225944211241,
+      "num_tokens": 13216848.0,
+      "mean_token_accuracy": 0.987069496512413,
+      "epoch": 3.937313432835821,
+      "step": 1980
+    },
+    {
+      "loss": 0.026138466596603394,
+      "grad_norm": 0.4671242833137512,
+      "learning_rate": 0.000141568685535983,
+      "entropy": 0.03499636381748132,
+      "num_tokens": 13282491.0,
+      "mean_token_accuracy": 0.9915538854897022,
+      "epoch": 3.9572139303482587,
+      "step": 1990
+    },
+    {
+      "loss": 0.03617530465126038,
+      "grad_norm": 0.6693828105926514,
+      "learning_rate": 0.00014096977894194741,
+      "entropy": 0.03698443787143333,
+      "num_tokens": 13351431.0,
+      "mean_token_accuracy": 0.9893846169114113,
+      "epoch": 3.9771144278606965,
+      "step": 2000
+    },
+    {
+      "loss": 0.04186029434204101,
+      "grad_norm": 0.6782835721969604,
+      "learning_rate": 0.0001403691011401342,
+      "entropy": 0.04769496822264045,
+      "num_tokens": 13418199.0,
+      "mean_token_accuracy": 0.9833127044141292,
+      "epoch": 3.997014925373134,
+      "step": 2010
+    },
+    {
+      "eval_loss": 0.29686856269836426,
+      "eval_runtime": 67.8045,
+      "eval_samples_per_second": 15.264,
+      "eval_steps_per_second": 7.64,
+      "eval_entropy": 0.08914197777698184,
+      "eval_num_tokens": 13427944.0,
+      "eval_mean_token_accuracy": 0.9404666641274014,
+      "epoch": 4.0,
+      "step": 2012
+    },
+    {
+      "loss": 0.022541260719299315,
+      "grad_norm": 0.330778032541275,
+      "learning_rate": 0.00013976667809907967,
+      "entropy": 0.03496060195673061,
+      "num_tokens": 13480426.0,
+      "mean_token_accuracy": 0.992155635827466,
+      "epoch": 4.01592039800995,
+      "step": 2020
+    },
+    {
+      "loss": 0.020323292911052705,
+      "grad_norm": 0.6635566353797913,
+      "learning_rate": 0.00013916253586277046,
+      "entropy": 0.025642355805030093,
+      "num_tokens": 13548017.0,
+      "mean_token_accuracy": 0.9932406023144722,
+      "epoch": 4.035820895522388,
+      "step": 2030
+    },
+    {
+      "loss": 0.018497467041015625,
+      "grad_norm": 0.3545978367328644,
+      "learning_rate": 0.00013855670054951764,
+      "entropy": 0.02235944996937178,
+      "num_tokens": 13615907.0,
+      "mean_token_accuracy": 0.9940000854432582,
+      "epoch": 4.055721393034826,
+      "step": 2040
+    },
+    {
+      "loss": 0.016379858553409576,
+      "grad_norm": 0.333023339509964,
+      "learning_rate": 0.00013794919835082733,
+      "entropy": 0.02085629914072342,
+      "num_tokens": 13682334.0,
+      "mean_token_accuracy": 0.9944917172193527,
+      "epoch": 4.075621890547263,
+      "step": 2050
+    },
+    {
+      "loss": 0.025261417031288147,
+      "grad_norm": 0.4259703457355499,
+      "learning_rate": 0.00013734005553026863,
+      "entropy": 0.022779430758964735,
+      "num_tokens": 13750009.0,
+      "mean_token_accuracy": 0.9915114663541317,
+      "epoch": 4.095522388059702,
+      "step": 2060
+    },
+    {
+      "loss": 0.020985141396522522,
+      "grad_norm": 0.5059815645217896,
+      "learning_rate": 0.00013672929842233807,
+      "entropy": 0.030157123084063642,
+      "num_tokens": 13816528.0,
+      "mean_token_accuracy": 0.9944588914513588,
+      "epoch": 4.115422885572139,
+      "step": 2070
+    },
+    {
+      "loss": 0.019122378528118135,
+      "grad_norm": 0.27580463886260986,
+      "learning_rate": 0.00013611695343132118,
+      "entropy": 0.023731828895688523,
+      "num_tokens": 13883127.0,
+      "mean_token_accuracy": 0.9933131754398346,
+      "epoch": 4.135323383084577,
+      "step": 2080
+    },
+    {
+      "loss": 0.018794278800487518,
+      "grad_norm": 0.4173099994659424,
+      "learning_rate": 0.00013550304703015083,
+      "entropy": 0.02200460991152795,
+      "num_tokens": 13949694.0,
+      "mean_token_accuracy": 0.993462772667408,
+      "epoch": 4.155223880597015,
+      "step": 2090
+    },
+    {
+      "loss": 0.022590236365795137,
+      "grad_norm": 0.2273045778274536,
+      "learning_rate": 0.000134887605759263,
+      "entropy": 0.028880356659647076,
+      "num_tokens": 14016869.0,
+      "mean_token_accuracy": 0.9921673700213433,
+      "epoch": 4.1751243781094525,
+      "step": 2100
+    },
+    {
+      "loss": 0.028038790822029112,
+      "grad_norm": 0.3150612413883209,
+      "learning_rate": 0.00013427065622544914,
+      "entropy": 0.029227956954855472,
+      "num_tokens": 14082895.0,
+      "mean_token_accuracy": 0.9920774199068546,
+      "epoch": 4.195024875621891,
+      "step": 2110
+    },
+    {
+      "loss": 0.02197175920009613,
+      "grad_norm": 0.4716366231441498,
+      "learning_rate": 0.0001336522251007061,
+      "entropy": 0.026419853966217488,
+      "num_tokens": 14148483.0,
+      "mean_token_accuracy": 0.9921677350997925,
+      "epoch": 4.214925373134328,
+      "step": 2120
+    },
+    {
+      "loss": 0.019603276252746583,
+      "grad_norm": 0.6820113062858582,
+      "learning_rate": 0.00013303233912108283,
+      "entropy": 0.023932285644696093,
+      "num_tokens": 14214410.0,
+      "mean_token_accuracy": 0.9929466463625432,
+      "epoch": 4.234825870646766,
+      "step": 2130
+    },
+    {
+      "loss": 0.02161734998226166,
+      "grad_norm": 0.5260307788848877,
+      "learning_rate": 0.00013241102508552475,
+      "entropy": 0.023090845490514766,
+      "num_tokens": 14281433.0,
+      "mean_token_accuracy": 0.9935616083443165,
+      "epoch": 4.254726368159204,
+      "step": 2140
+    },
+    {
+      "loss": 0.020520062744617464,
+      "grad_norm": 0.5381481051445007,
+      "learning_rate": 0.00013178830985471505,
+      "entropy": 0.0253336504829349,
+      "num_tokens": 14347059.0,
+      "mean_token_accuracy": 0.9924183614552021,
+      "epoch": 4.2746268656716415,
+      "step": 2150
+    },
+    {
+      "loss": 0.02292156219482422,
+      "grad_norm": 0.4821164906024933,
+      "learning_rate": 0.00013116422034991347,
+      "entropy": 0.023389648927695815,
+      "num_tokens": 14412949.0,
+      "mean_token_accuracy": 0.9910940513014793,
+      "epoch": 4.29452736318408,
+      "step": 2160
+    },
+    {
+      "loss": 0.020588286221027374,
+      "grad_norm": 0.314314067363739,
+      "learning_rate": 0.00013053878355179244,
+      "entropy": 0.028456786752212793,
+      "num_tokens": 14479732.0,
+      "mean_token_accuracy": 0.9933287680149079,
+      "epoch": 4.314427860696517,
+      "step": 2170
+    },
+    {
+      "loss": 0.022340704500675202,
+      "grad_norm": 0.1762777715921402,
+      "learning_rate": 0.00012991202649927056,
+      "entropy": 0.024215608432132284,
+      "num_tokens": 14547229.0,
+      "mean_token_accuracy": 0.993473107367754,
+      "epoch": 4.334328358208955,
+      "step": 2180
+    },
+    {
+      "loss": 0.01929643303155899,
+      "grad_norm": 0.5308461785316467,
+      "learning_rate": 0.00012928397628834395,
+      "entropy": 0.022466996918956282,
+      "num_tokens": 14615321.0,
+      "mean_token_accuracy": 0.9933658391237259,
+      "epoch": 4.354228855721393,
+      "step": 2190
+    },
+    {
+      "loss": 0.01675943285226822,
+      "grad_norm": 0.2166927307844162,
+      "learning_rate": 0.0001286546600709144,
+      "entropy": 0.02382153325015679,
+      "num_tokens": 14681484.0,
+      "mean_token_accuracy": 0.9929957866668702,
+      "epoch": 4.374129353233831,
+      "step": 2200
+    },
+    {
+      "loss": 0.01616147756576538,
+      "grad_norm": 0.31190088391304016,
+      "learning_rate": 0.00012802410505361592,
+      "entropy": 0.021566898842866066,
+      "num_tokens": 14748246.0,
+      "mean_token_accuracy": 0.9953425668179989,
+      "epoch": 4.394029850746269,
+      "step": 2210
+    },
+    {
+      "loss": 0.020143616199493408,
+      "grad_norm": 0.5401660799980164,
+      "learning_rate": 0.0001273923384966383,
+      "entropy": 0.019891528951120563,
+      "num_tokens": 14815008.0,
+      "mean_token_accuracy": 0.9928821548819542,
+      "epoch": 4.413930348258706,
+      "step": 2220
+    },
+    {
+      "loss": 0.023275299370288847,
+      "grad_norm": 0.5358554720878601,
+      "learning_rate": 0.00012675938771254872,
+      "entropy": 0.027829346724320202,
+      "num_tokens": 14880257.0,
+      "mean_token_accuracy": 0.9924751475453377,
+      "epoch": 4.433830845771144,
+      "step": 2230
+    },
+    {
+      "loss": 0.018383045494556428,
+      "grad_norm": 0.30791008472442627,
+      "learning_rate": 0.00012612528006511093,
+      "entropy": 0.01990911388711538,
+      "num_tokens": 14946824.0,
+      "mean_token_accuracy": 0.9936320453882217,
+      "epoch": 4.453731343283582,
+      "step": 2240
+    },
+    {
+      "loss": 0.018728886544704438,
+      "grad_norm": 0.4665677547454834,
+      "learning_rate": 0.0001254900429681023,
+      "entropy": 0.02076636096899165,
+      "num_tokens": 15012580.0,
+      "mean_token_accuracy": 0.9947231650352478,
+      "epoch": 4.47363184079602,
+      "step": 2250
+    },
+    {
+      "loss": 0.02189120650291443,
+      "grad_norm": 0.22175736725330353,
+      "learning_rate": 0.0001248537038841285,
+      "entropy": 0.025514183560881067,
+      "num_tokens": 15078417.0,
+      "mean_token_accuracy": 0.9927590176463127,
+      "epoch": 4.493532338308458,
+      "step": 2260
+    },
+    {
+      "loss": 0.018778012692928316,
+      "grad_norm": 0.37904441356658936,
+      "learning_rate": 0.0001242162903234365,
+      "entropy": 0.024176929768873378,
+      "num_tokens": 15145267.0,
+      "mean_token_accuracy": 0.9923839196562767,
+      "epoch": 4.513432835820895,
+      "step": 2270
+    },
+    {
+      "loss": 0.021106557548046113,
+      "grad_norm": 0.4318683445453644,
+      "learning_rate": 0.00012357782984272504,
+      "entropy": 0.017634534194803562,
+      "num_tokens": 15211816.0,
+      "mean_token_accuracy": 0.9937898091971874,
+      "epoch": 4.533333333333333,
+      "step": 2280
+    },
+    {
+      "loss": 0.015814051032066345,
+      "grad_norm": 0.24642682075500488,
+      "learning_rate": 0.0001229383500439534,
+      "entropy": 0.020605951044126415,
+      "num_tokens": 15278462.0,
+      "mean_token_accuracy": 0.9958261914551259,
+      "epoch": 4.553233830845771,
+      "step": 2290
+    },
+    {
+      "loss": 0.01898970454931259,
+      "grad_norm": 0.3638794422149658,
+      "learning_rate": 0.00012229787857314801,
+      "entropy": 0.020515447400248375,
+      "num_tokens": 15346966.0,
+      "mean_token_accuracy": 0.9943824775516987,
+      "epoch": 4.573134328358209,
+      "step": 2300
+    },
+    {
+      "loss": 0.01712157726287842,
+      "grad_norm": 0.287197470664978,
+      "learning_rate": 0.00012165644311920741,
+      "entropy": 0.019921001550392246,
+      "num_tokens": 15414059.0,
+      "mean_token_accuracy": 0.9939340233802796,
+      "epoch": 4.593034825870647,
+      "step": 2310
+    },
+    {
+      "loss": 0.029004442691802978,
+      "grad_norm": 0.613610565662384,
+      "learning_rate": 0.000121014071412705,
+      "entropy": 0.03135428504901938,
+      "num_tokens": 15479736.0,
+      "mean_token_accuracy": 0.9910656772553921,
+      "epoch": 4.612935323383084,
+      "step": 2320
+    },
+    {
+      "loss": 0.02000347375869751,
+      "grad_norm": 0.5387656092643738,
+      "learning_rate": 0.00012037079122469044,
+      "entropy": 0.02796420800877968,
+      "num_tokens": 15546436.0,
+      "mean_token_accuracy": 0.9945271402597428,
+      "epoch": 4.632835820895522,
+      "step": 2330
+    },
+    {
+      "loss": 0.02363658994436264,
+      "grad_norm": 0.3635019361972809,
+      "learning_rate": 0.00011972663036548884,
+      "entropy": 0.022651451456476936,
+      "num_tokens": 15612938.0,
+      "mean_token_accuracy": 0.9910245075821876,
+      "epoch": 4.65273631840796,
+      "step": 2340
+    },
+    {
+      "loss": 0.023246921598911285,
+      "grad_norm": 0.25204190611839294,
+      "learning_rate": 0.0001190816166834985,
+      "entropy": 0.027166575644514523,
+      "num_tokens": 15681309.0,
+      "mean_token_accuracy": 0.9916775986552239,
+      "epoch": 4.672636815920398,
+      "step": 2350
+    },
+    {
+      "loss": 0.020346690714359284,
+      "grad_norm": 0.37856990098953247,
+      "learning_rate": 0.00011843577806398705,
+      "entropy": 0.025465619296301156,
+      "num_tokens": 15749150.0,
+      "mean_token_accuracy": 0.9934266820549965,
+      "epoch": 4.692537313432836,
+      "step": 2360
+    },
+    {
+      "loss": 0.022995509207248688,
+      "grad_norm": 0.3728092610836029,
+      "learning_rate": 0.00011778914242788588,
+      "entropy": 0.02361324623489054,
+      "num_tokens": 15815098.0,
+      "mean_token_accuracy": 0.9924490824341774,
+      "epoch": 4.712437810945273,
+      "step": 2370
+    },
+    {
+      "loss": 0.024149328470230103,
+      "grad_norm": 0.44309478998184204,
+      "learning_rate": 0.00011714173773058304,
+      "entropy": 0.023943239758955313,
+      "num_tokens": 15882589.0,
+      "mean_token_accuracy": 0.9915248282253742,
+      "epoch": 4.732338308457711,
+      "step": 2380
+    },
+    {
+      "loss": 0.02506372332572937,
+      "grad_norm": 0.4280659556388855,
+      "learning_rate": 0.00011649359196071459,
+      "entropy": 0.027685758270672524,
+      "num_tokens": 15947514.0,
+      "mean_token_accuracy": 0.9927876427769661,
+      "epoch": 4.7522388059701495,
+      "step": 2390
+    },
+    {
+      "loss": 0.01892734318971634,
+      "grad_norm": 0.1368793547153473,
+      "learning_rate": 0.00011584473313895483,
+      "entropy": 0.02604678466304904,
+      "num_tokens": 16015196.0,
+      "mean_token_accuracy": 0.9935356914997101,
+      "epoch": 4.772139303482587,
+      "step": 2400
+    },
+    {
+      "loss": 0.022761589288711546,
+      "grad_norm": 0.2650626301765442,
+      "learning_rate": 0.00011519518931680463,
+      "entropy": 0.02288266723626293,
+      "num_tokens": 16084048.0,
+      "mean_token_accuracy": 0.9923152230679989,
+      "epoch": 4.792039800995025,
+      "step": 2410
+    },
+    {
+      "loss": 0.02219443619251251,
+      "grad_norm": 0.28764352202415466,
+      "learning_rate": 0.00011454498857537893,
+      "entropy": 0.021520700085238785,
+      "num_tokens": 16151465.0,
+      "mean_token_accuracy": 0.9929785504937172,
+      "epoch": 4.811940298507462,
+      "step": 2420
+    },
+    {
+      "loss": 0.01702689528465271,
+      "grad_norm": 0.265876442193985,
+      "learning_rate": 0.00011389415902419251,
+      "entropy": 0.022896296047838403,
+      "num_tokens": 16219886.0,
+      "mean_token_accuracy": 0.9943966299295426,
+      "epoch": 4.8318407960199,
+      "step": 2430
+    },
+    {
+      "loss": 0.02124347984790802,
+      "grad_norm": 0.4632825255393982,
+      "learning_rate": 0.00011324272879994509,
+      "entropy": 0.024235972843598574,
+      "num_tokens": 16286732.0,
+      "mean_token_accuracy": 0.9925971522927284,
+      "epoch": 4.8517412935323385,
+      "step": 2440
+    },
+    {
+      "loss": 0.02112184017896652,
+      "grad_norm": 0.7702906131744385,
+      "learning_rate": 0.00011259072606530452,
+      "entropy": 0.02376203608873766,
+      "num_tokens": 16353324.0,
+      "mean_token_accuracy": 0.9932762250304222,
+      "epoch": 4.871641791044776,
+      "step": 2450
+    },
+    {
+      "loss": 0.020696014165878296,
+      "grad_norm": 0.43466606736183167,
+      "learning_rate": 0.0001119381790076896,
+      "entropy": 0.020602873369352893,
+      "num_tokens": 16420045.0,
+      "mean_token_accuracy": 0.9940371952950955,
+      "epoch": 4.891542288557214,
+      "step": 2460
+    },
+    {
+      "loss": 0.02507336139678955,
+      "grad_norm": 0.7223814725875854,
+      "learning_rate": 0.00011128511583805122,
+      "entropy": 0.026875402212681365,
+      "num_tokens": 16487489.0,
+      "mean_token_accuracy": 0.9920289881527424,
+      "epoch": 4.911442786069652,
+      "step": 2470
+    },
+    {
+      "loss": 0.026432156562805176,
+      "grad_norm": 0.47006839513778687,
+      "learning_rate": 0.00011063156478965293,
+      "entropy": 0.03200415483734105,
+      "num_tokens": 16556385.0,
+      "mean_token_accuracy": 0.9915605559945107,
+      "epoch": 4.931343283582089,
+      "step": 2480
+    },
+    {
+      "loss": 0.019228325784206392,
+      "grad_norm": 0.6098710298538208,
+      "learning_rate": 0.00010997755411685022,
+      "entropy": 0.021561289162491448,
+      "num_tokens": 16622218.0,
+      "mean_token_accuracy": 0.9942485235631466,
+      "epoch": 4.951243781094528,
+      "step": 2490
+    },
+    {
+      "loss": 0.02225508540868759,
+      "grad_norm": 0.6580213904380798,
+      "learning_rate": 0.0001093231120938692,
+      "entropy": 0.025235205114586278,
+      "num_tokens": 16688229.0,
+      "mean_token_accuracy": 0.992252241820097,
+      "epoch": 4.971144278606965,
+      "step": 2500
+    },
+    {
+      "loss": 0.02393328994512558,
+      "grad_norm": 0.2273155003786087,
+      "learning_rate": 0.000108668267013584,
+      "entropy": 0.02115868393520941,
+      "num_tokens": 16755861.0,
+      "mean_token_accuracy": 0.9941031068563462,
+      "epoch": 4.991044776119403,
+      "step": 2510
+    },
+    {
+      "eval_loss": 0.3079036474227905,
+      "eval_runtime": 67.1765,
+      "eval_samples_per_second": 15.407,
+      "eval_steps_per_second": 7.711,
+      "eval_entropy": 0.07137546143402403,
+      "eval_num_tokens": 16784930.0,
+      "eval_mean_token_accuracy": 0.9430849159776474,
+      "epoch": 5.0,
+      "step": 2515
+    },
+    {
+      "train_runtime": 8534.0451,
+      "train_samples_per_second": 9.421,
+      "train_steps_per_second": 0.589,
+      "total_flos": 3.4880192385122304e+16,
+      "train_loss": 0.10587940257242612,
+      "epoch": 5.0,
+      "step": 2515
+    },
+    {
+      "eval_loss": 0.24770455062389374,
+      "eval_runtime": 67.4582,
+      "eval_samples_per_second": 15.343,
+      "eval_steps_per_second": 7.679,
+      "eval_entropy": 0.13267684354209502,
+      "eval_num_tokens": 16784930.0,
+      "eval_mean_token_accuracy": 0.9410935246806347,
+      "epoch": 5.0,
+      "step": 2515
+    }
+  ]
+}

tokenizer.json ADDED Viewed

The diff for this file is too large to render. See raw diff

tokenizer_config.json ADDED Viewed

	@@ -0,0 +1,11 @@

+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "is_local": true,
+  "model_max_length": 2048,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "TokenizersBackend",
+  "unk_token": "<|endoftext|>"
+}

training_args.bin ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:2350592d6176e1137e9e5000ddaae8bcace4a32263a7e4abe5cae8ea3d5bbe86
+size 5841