{
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "collapsed": false,
        "scrolled": true
      },
      "source": [
        "%uv pip install tokenizers datasets huggingface_hub"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[2mUsing Python 3.12.6 environment at: /usr/local\u001b[0m\r\n",
            "\u001b[37m\u280b\u001b[0m \u001b[2mResolving dependencies...                                                     \u001b[0m\r\u001b[2K\u001b[37m\u2819\u001b[0m \u001b[2mResolving dependencies...                                                     \u001b[0m\r\u001b[2K\u001b[37m\u280b\u001b[0m \u001b[2mResolving dependencies...                                                     \u001b[0m\r\u001b[2K\u001b[37m\u2819\u001b[0m \u001b[2mResolving dependencies...                                                     \u001b[0m\r\u001b[2K\u001b[37m\u2819\u001b[0m \u001b[2mtokenizers==0.22.0                                                            \u001b[0m\r\u001b[2K\u001b[37m\u2819\u001b[0m \u001b[2mdatasets==5.0.0                                                               \u001b[0m\r\u001b[2K\u001b[37m\u2819\u001b[0m \u001b[2mhuggingface-hub==0.34.4                                                       \u001b[0m\r\u001b[2K\u001b[37m\u2819\u001b[0m \u001b[2mfilelock==3.13.1                                                              \u001b[0m\r\u001b[2K\u001b[37m\u2819\u001b[0m \u001b[2mnumpy==2.1.2                                                                  \u001b[0m\r\u001b[2K\u001b[37m\u2819\u001b[0m \u001b[2mpyarrow==25.0.0                                                               \u001b[0m\r\u001b[2K\u001b[37m\u2819\u001b[0m \u001b[2mdill==0.4.1                                                                   \u001b[0m\r\u001b[2K\u001b[37m\u2819\u001b[0m \u001b[2mpandas==2.3.2                                                                 \u001b[0m\r\u001b[2K\u001b[37m\u2819\u001b[0m \u001b[2mnumpy==2.1.2                                                                  \u001b[0m\r\u001b[2K\u001b[37m\u2819\u001b[0m \u001b[2mrequests==2.32.5                                                              \u001b[0m\r\u001b[2K\u001b[37m\u2819\u001b[0m \u001b[2mhttpx==0.28.1                                                                 \u001b[0m\r\u001b[2K\u001b[37m\u2819\u001b[0m \u001b[2mtqdm==4.67.1                                                                  \u001b[0m\r\u001b[2K\u001b[37m\u2819\u001b[0m \u001b[2mxxhash==3.8.1                                                                 \u001b[0m\r\u001b[2K\u001b[37m\u2819\u001b[0m \u001b[2mmultiprocess==0.70.19                                                         \u001b[0m\r\u001b[2K\u001b[37m\u2819\u001b[0m \u001b[2mfsspec==2024.6.1                                                              \u001b[0m\r\u001b[2K\u001b[37m\u2819\u001b[0m \u001b[2mfsspec==2024.6.1                                                              \u001b[0m\r\u001b[2K\u001b[37m\u2819\u001b[0m \u001b[2mpackaging==25.0                                                               \u001b[0m\r\u001b[2K\u001b[37m\u2819\u001b[0m \u001b[2mpyyaml==6.0.2                                                                 \u001b[0m\r\u001b[2K\u001b[37m\u2819\u001b[0m \u001b[2mtyping-extensions==4.12.2                                                     \u001b[0m\r\u001b[2K\u001b[37m\u2819\u001b[0m \u001b[2mhf-xet==1.1.9                                                                 \u001b[0m\r\u001b[2K\u001b[37m\u2819\u001b[0m \u001b[2mhf-xet==1.1.9                                                                 \u001b[0m\r\u001b[2K\u001b[2mResolved \u001b[1m37 packages\u001b[0m \u001b[2min 197ms\u001b[0m\u001b[0m\r\n",
            "\u001b[37m\u280b\u001b[0m \u001b[2mPreparing packages...\u001b[0m (0/0)                                                   \r\u001b[2K\u001b[37m\u280b\u001b[0m \u001b[2mPreparing packages...\u001b[0m (0/5)                                                   \r\u001b[2K\u001b[37m\u2819\u001b[0m \u001b[2mPreparing packages...\u001b[0m (0/5)                                                   \r\u001b[2K\u001b[37m\u2819\u001b[0m \u001b[2mPreparing packages...\u001b[0m (0/5)\r\n",
            "\u001b[2mmultiprocess\u001b[0m \u001b[32m\u001b[2m------------------------------\u001b[0m\u001b[0m     0 B/146.76 KiB                  \u001b[1A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1A\u001b[37m\u2819\u001b[0m \u001b[2mPreparing packages...\u001b[0m (0/5)\r\n",
            "\u001b[2mmultiprocess\u001b[0m \u001b[32m----\u001b[2m--------------------------\u001b[0m\u001b[0m 16.00 KiB/146.76 KiB                \u001b[1A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1A\u001b[37m\u2819\u001b[0m \u001b[2mPreparing packages...\u001b[0m (0/5)\r\n",
            "\u001b[2mmultiprocess\u001b[0m \u001b[32m----\u001b[2m--------------------------\u001b[0m\u001b[0m 16.00 KiB/146.76 KiB\r\n",
            "\u001b[2mdatasets  \u001b[0m \u001b[32m\u001b[2m------------------------------\u001b[0m\u001b[0m     0 B/542.07 KiB                    \u001b[2A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[2A\u001b[37m\u2819\u001b[0m \u001b[2mPreparing packages...\u001b[0m (0/5)\r\n",
            "\u001b[2mmultiprocess\u001b[0m \u001b[32m----\u001b[2m--------------------------\u001b[0m\u001b[0m 16.00 KiB/146.76 KiB\r\n",
            "\u001b[2mdatasets  \u001b[0m \u001b[32m-\u001b[2m-----------------------------\u001b[0m\u001b[0m 14.87 KiB/542.07 KiB                  \u001b[2A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[2A\u001b[37m\u2819\u001b[0m \u001b[2mPreparing packages...\u001b[0m (0/5)\r\n",
            "\u001b[2mmultiprocess\u001b[0m \u001b[32m----\u001b[2m--------------------------\u001b[0m\u001b[0m 16.00 KiB/146.76 KiB\r\n",
            "\u001b[2mdatasets  \u001b[0m \u001b[32m--\u001b[2m----------------------------\u001b[0m\u001b[0m 30.87 KiB/542.07 KiB                  \u001b[2A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[2A\u001b[37m\u2819\u001b[0m \u001b[2mPreparing packages...\u001b[0m (0/5)\r\n",
            "\u001b[2mmultiprocess\u001b[0m \u001b[32m-------\u001b[2m-----------------------\u001b[0m\u001b[0m 32.00 KiB/146.76 KiB\r\n",
            "\u001b[2mdatasets  \u001b[0m \u001b[32m--\u001b[2m----------------------------\u001b[0m\u001b[0m 30.87 KiB/542.07 KiB                  \u001b[2A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[2A\u001b[37m\u2819\u001b[0m \u001b[2mPreparing packages...\u001b[0m (0/5)\r\n",
            "\u001b[2mmultiprocess\u001b[0m \u001b[32m----------\u001b[2m--------------------\u001b[0m\u001b[0m 48.00 KiB/146.76 KiB\r\n",
            "\u001b[2mdatasets  \u001b[0m \u001b[32m--\u001b[2m----------------------------\u001b[0m\u001b[0m 30.87 KiB/542.07 KiB                  \u001b[2A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[2A\u001b[37m\u2819\u001b[0m \u001b[2mPreparing packages...\u001b[0m (0/5)\r\n",
            "\u001b[2mmultiprocess\u001b[0m \u001b[32m----------\u001b[2m--------------------\u001b[0m\u001b[0m 48.00 KiB/146.76 KiB\r\n",
            "\u001b[2mdatasets  \u001b[0m \u001b[32m---\u001b[2m---------------------------\u001b[0m\u001b[0m 46.87 KiB/542.07 KiB                  \u001b[2A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[2A\u001b[37m\u2819\u001b[0m \u001b[2mPreparing packages...\u001b[0m (0/5)\r\n",
            "\u001b[2mmultiprocess\u001b[0m \u001b[32m----------\u001b[2m--------------------\u001b[0m\u001b[0m 48.00 KiB/146.76 KiB\r\n",
            "\u001b[2mdatasets  \u001b[0m \u001b[32m----\u001b[2m--------------------------\u001b[0m\u001b[0m 62.87 KiB/542.07 KiB                  \u001b[2A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[2A\u001b[37m\u2819\u001b[0m \u001b[2mPreparing packages...\u001b[0m (0/5)\r\n",
            "\u001b[2mmultiprocess\u001b[0m \u001b[32m----------\u001b[2m--------------------\u001b[0m\u001b[0m 48.00 KiB/146.76 KiB\r\n",
            "\u001b[2mdatasets  \u001b[0m \u001b[32m-----\u001b[2m-------------------------\u001b[0m\u001b[0m 76.23 KiB/542.07 KiB                  \u001b[2A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[2A\u001b[37m\u2819\u001b[0m \u001b[2mPreparing packages...\u001b[0m (0/5)\r\n",
            "\u001b[2mmultiprocess\u001b[0m \u001b[32m----------\u001b[2m--------------------\u001b[0m\u001b[0m 48.00 KiB/146.76 KiB\r\n",
            "\u001b[2mdatasets  \u001b[0m \u001b[32m-----\u001b[2m-------------------------\u001b[0m\u001b[0m 76.23 KiB/542.07 KiB\r\n",
            "\u001b[2mpyarrow   \u001b[0m \u001b[32m\u001b[2m------------------------------\u001b[0m\u001b[0m     0 B/47.77 MiB                     \u001b[3A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[3A\u001b[37m\u2819\u001b[0m \u001b[2mPreparing packages...\u001b[0m (0/5)\r\n",
            "\u001b[2mmultiprocess\u001b[0m \u001b[32m----------\u001b[2m--------------------\u001b[0m\u001b[0m 48.00 KiB/146.76 KiB\r\n",
            "\u001b[2mdatasets  \u001b[0m \u001b[32m-----\u001b[2m-------------------------\u001b[0m\u001b[0m 76.23 KiB/542.07 KiB\r\n",
            "\u001b[2mpyarrow   \u001b[0m \u001b[32m-\u001b[2m-----------------------------\u001b[0m\u001b[0m 14.81 KiB/47.77 MiB                   \u001b[3A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[3A\u001b[37m\u2819\u001b[0m \u001b[2mPreparing packages...\u001b[0m (0/5)\r\n",
            "\u001b[2mmultiprocess\u001b[0m \u001b[32m----------\u001b[2m--------------------\u001b[0m\u001b[0m 48.00 KiB/146.76 KiB\r\n",
            "\u001b[2mxxhash    \u001b[0m \u001b[32m\u001b[2m------------------------------\u001b[0m\u001b[0m     0 B/215.30 KiB\r\n",
            "\u001b[2mdatasets  \u001b[0m \u001b[32m-----\u001b[2m-------------------------\u001b[0m\u001b[0m 76.23 KiB/542.07 KiB\r\n",
            "\u001b[2mpyarrow   \u001b[0m \u001b[32m-\u001b[2m-----------------------------\u001b[0m\u001b[0m 14.81 KiB/47.77 MiB                   \u001b[4A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[4A\u001b[37m\u2819\u001b[0m \u001b[2mPreparing packages...\u001b[0m (0/5)\r\n",
            "\u001b[2mmultiprocess\u001b[0m \u001b[32m----------\u001b[2m--------------------\u001b[0m\u001b[0m 48.00 KiB/146.76 KiB\r\n",
            "\u001b[2mxxhash    \u001b[0m \u001b[32m---\u001b[2m---------------------------\u001b[0m\u001b[0m 14.85 KiB/215.30 KiB\r\n",
            "\u001b[2mdatasets  \u001b[0m \u001b[32m-----\u001b[2m-------------------------\u001b[0m\u001b[0m 76.23 KiB/542.07 KiB\r\n",
            "\u001b[2mpyarrow   \u001b[0m \u001b[32m-\u001b[2m-----------------------------\u001b[0m\u001b[0m 14.81 KiB/47.77 MiB                   \u001b[4A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[4A\u001b[37m\u2819\u001b[0m \u001b[2mPreparing packages...\u001b[0m (0/5)\r\n",
            "\u001b[2mdill      \u001b[0m \u001b[32m\u001b[2m------------------------------\u001b[0m\u001b[0m     0 B/117.21 KiB\r\n",
            "\u001b[2mmultiprocess\u001b[0m \u001b[32m----------\u001b[2m--------------------\u001b[0m\u001b[0m 48.00 KiB/146.76 KiB\r\n",
            "\u001b[2mxxhash    \u001b[0m \u001b[32m---\u001b[2m---------------------------\u001b[0m\u001b[0m 14.85 KiB/215.30 KiB\r\n",
            "\u001b[2mdatasets  \u001b[0m \u001b[32m-----\u001b[2m-------------------------\u001b[0m\u001b[0m 76.23 KiB/542.07 KiB\r\n",
            "\u001b[2mpyarrow   \u001b[0m \u001b[32m-\u001b[2m-----------------------------\u001b[0m\u001b[0m 14.81 KiB/47.77 MiB                   \u001b[5A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[5A\u001b[37m\u2819\u001b[0m \u001b[2mPreparing packages...\u001b[0m (0/5)\r\n",
            "\u001b[2mdill      \u001b[0m \u001b[32m----\u001b[2m--------------------------\u001b[0m\u001b[0m 14.84 KiB/117.21 KiB\r\n",
            "\u001b[2mmultiprocess\u001b[0m \u001b[32m----------\u001b[2m--------------------\u001b[0m\u001b[0m 48.00 KiB/146.76 KiB\r\n",
            "\u001b[2mxxhash    \u001b[0m \u001b[32m---\u001b[2m---------------------------\u001b[0m\u001b[0m 14.85 KiB/215.30 KiB\r\n",
            "\u001b[2mdatasets  \u001b[0m \u001b[32m-----\u001b[2m-------------------------\u001b[0m\u001b[0m 76.23 KiB/542.07 KiB\r\n",
            "\u001b[2mpyarrow   \u001b[0m \u001b[32m-\u001b[2m-----------------------------\u001b[0m\u001b[0m 14.81 KiB/47.77 MiB                   \u001b[5A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[5A\u001b[37m\u2819\u001b[0m \u001b[2mPreparing packages...\u001b[0m (0/5)\r\n",
            "\u001b[2mdill      \u001b[0m \u001b[32m----\u001b[2m--------------------------\u001b[0m\u001b[0m 14.84 KiB/117.21 KiB\r\n",
            "\u001b[2mmultiprocess\u001b[0m \u001b[32m----------\u001b[2m--------------------\u001b[0m\u001b[0m 48.00 KiB/146.76 KiB\r\n",
            "\u001b[2mxxhash    \u001b[0m \u001b[32m---\u001b[2m---------------------------\u001b[0m\u001b[0m 14.85 KiB/215.30 KiB\r\n",
            "\u001b[2mdatasets  \u001b[0m \u001b[32m------\u001b[2m------------------------\u001b[0m\u001b[0m 92.23 KiB/542.07 KiB\r\n",
            "\u001b[2mpyarrow   \u001b[0m \u001b[32m-\u001b[2m-----------------------------\u001b[0m\u001b[0m 14.81 KiB/47.77 MiB                   \u001b[5A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[5A\u001b[37m\u2819\u001b[0m \u001b[2mPreparing packages...\u001b[0m (0/5)\r\n",
            "\u001b[2mdill      \u001b[0m \u001b[32m-----------------\u001b[2m-------------\u001b[0m\u001b[0m 62.84 KiB/117.21 KiB\r\n",
            "\u001b[2mmultiprocess\u001b[0m \u001b[32m----------------\u001b[2m--------------\u001b[0m\u001b[0m 76.96 KiB/146.76 KiB\r\n",
            "\u001b[2mdatasets  \u001b[0m \u001b[32m---------\u001b[2m---------------------\u001b[0m\u001b[0m 156.23 KiB/542.07 KiB\r\n",
            "\u001b[2mpyarrow   \u001b[0m \u001b[32m-\u001b[2m-----------------------------\u001b[0m\u001b[0m 735.89 KiB/47.77 MiB                  \u001b[4A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[4A\u001b[37m\u2819\u001b[0m \u001b[2mPreparing packages...\u001b[0m (0/5)\r\n",
            "\u001b[2mdill      \u001b[0m \u001b[32m---------------------\u001b[2m---------\u001b[0m\u001b[0m 78.84 KiB/117.21 KiB\r\n",
            "\u001b[2mmultiprocess\u001b[0m \u001b[32m--------------------\u001b[2m----------\u001b[0m\u001b[0m 92.96 KiB/146.76 KiB\r\n",
            "\u001b[2mdatasets  \u001b[0m \u001b[32m--------------\u001b[2m----------------\u001b[0m\u001b[0m 236.12 KiB/542.07 KiB\r\n",
            "\u001b[2mpyarrow   \u001b[0m \u001b[32m-\u001b[2m-----------------------------\u001b[0m\u001b[0m 1.07 MiB/47.77 MiB                    \u001b[4A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[4A\u001b[37m\u2819\u001b[0m \u001b[2mPreparing packages...\u001b[0m (0/5)\r\n",
            "\u001b[2mdill      \u001b[0m \u001b[32m-----------------------------\u001b[2m-\u001b[0m\u001b[0m 110.84 KiB/117.21 KiB\r\n",
            "\u001b[2mdatasets  \u001b[0m \u001b[32m------------------\u001b[2m------------\u001b[0m\u001b[0m 316.23 KiB/542.07 KiB\r\n",
            "\u001b[2mpyarrow   \u001b[0m \u001b[32m--\u001b[2m----------------------------\u001b[0m\u001b[0m 2.12 MiB/47.77 MiB                    \u001b[3A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[3A\u001b[37m\u2819\u001b[0m \u001b[2mPreparing packages...\u001b[0m (0/5)\r\n",
            "\u001b[2mdatasets  \u001b[0m \u001b[32m-------------------\u001b[2m-----------\u001b[0m\u001b[0m 332.23 KiB/542.07 KiB\r\n",
            "\u001b[2mpyarrow   \u001b[0m \u001b[32m--\u001b[2m----------------------------\u001b[0m\u001b[0m 2.62 MiB/47.77 MiB                    \u001b[2A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[2A\u001b[37m\u2819\u001b[0m \u001b[2mPreparing packages...\u001b[0m (0/5)\r\n",
            "\u001b[2mdatasets  \u001b[0m \u001b[32m--------------------\u001b[2m----------\u001b[0m\u001b[0m 348.23 KiB/542.07 KiB\r\n",
            "\u001b[2mpyarrow   \u001b[0m \u001b[32m--\u001b[2m----------------------------\u001b[0m\u001b[0m 2.80 MiB/47.77 MiB                    \u001b[2A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[2A\u001b[37m\u2819\u001b[0m \u001b[2mPreparing packages...\u001b[0m (0/5)\r\n",
            "\u001b[2mdatasets  \u001b[0m \u001b[32m---------------------\u001b[2m---------\u001b[0m\u001b[0m 364.23 KiB/542.07 KiB\r\n",
            "\u001b[2mpyarrow   \u001b[0m \u001b[32m----\u001b[2m--------------------------\u001b[0m\u001b[0m 6.25 MiB/47.77 MiB                    \u001b[2A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[2A\u001b[37m\u2819\u001b[0m \u001b[2mPreparing packages...\u001b[0m (0/5)\r\n",
            "\u001b[2mdatasets  \u001b[0m \u001b[32m-----------------------\u001b[2m-------\u001b[0m\u001b[0m 412.23 KiB/542.07 KiB\r\n",
            "\u001b[2mpyarrow   \u001b[0m \u001b[32m------\u001b[2m------------------------\u001b[0m\u001b[0m 8.42 MiB/47.77 MiB                    \u001b[2A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[2A\u001b[37m\u2839\u001b[0m \u001b[2mPreparing packages...\u001b[0m (3/5)\r\n",
            "\u001b[2mdatasets  \u001b[0m \u001b[32m------------------------\u001b[2m------\u001b[0m\u001b[0m 428.23 KiB/542.07 KiB\r\n",
            "\u001b[2mpyarrow   \u001b[0m \u001b[32m--------\u001b[2m----------------------\u001b[0m\u001b[0m 11.58 MiB/47.77 MiB                   \u001b[2A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[2A\u001b[37m\u2839\u001b[0m \u001b[2mPreparing packages...\u001b[0m (3/5)\r\n",
            "\u001b[2mdatasets  \u001b[0m \u001b[32m------------------------\u001b[2m------\u001b[0m\u001b[0m 428.23 KiB/542.07 KiB\r\n",
            "\u001b[2mpyarrow   \u001b[0m \u001b[32m----------\u001b[2m--------------------\u001b[0m\u001b[0m 14.45 MiB/47.77 MiB                   \u001b[2A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[2A\u001b[37m\u2839\u001b[0m \u001b[2mPreparing packages...\u001b[0m (3/5)\r\n",
            "\u001b[2mdatasets  \u001b[0m \u001b[32m--------------------------\u001b[2m----\u001b[0m\u001b[0m 460.23 KiB/542.07 KiB\r\n",
            "\u001b[2mpyarrow   \u001b[0m \u001b[32m-----------\u001b[2m-------------------\u001b[0m\u001b[0m 16.89 MiB/47.77 MiB                   \u001b[2A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[2A\u001b[37m\u2839\u001b[0m \u001b[2mPreparing packages...\u001b[0m (3/5)\r\n",
            "\u001b[2mdatasets  \u001b[0m \u001b[32m------------------------------\u001b[2m\u001b[0m\u001b[0m 524.23 KiB/542.07 KiB\r\n",
            "\u001b[2mpyarrow   \u001b[0m \u001b[32m-------------\u001b[2m-----------------\u001b[0m\u001b[0m 19.98 MiB/47.77 MiB                   \u001b[2A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[2A\u001b[37m\u2838\u001b[0m \u001b[2mPreparing packages...\u001b[0m (3/5)\r\n",
            "\u001b[2mpyarrow   \u001b[0m \u001b[32m--------------\u001b[2m----------------\u001b[0m\u001b[0m 20.87 MiB/47.77 MiB                   \u001b[1A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1A\u001b[37m\u2838\u001b[0m \u001b[2mPreparing packages...\u001b[0m (3/5)\r\n",
            "\u001b[2mpyarrow   \u001b[0m \u001b[32m---------------\u001b[2m---------------\u001b[0m\u001b[0m 22.70 MiB/47.77 MiB                   \u001b[1A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1A\u001b[37m\u2838\u001b[0m \u001b[2mPreparing packages...\u001b[0m (3/5)\r\n",
            "\u001b[2mpyarrow   \u001b[0m \u001b[32m----------------\u001b[2m--------------\u001b[0m\u001b[0m 25.08 MiB/47.77 MiB                   \u001b[1A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1A\u001b[37m\u2838\u001b[0m \u001b[2mPreparing packages...\u001b[0m (3/5)\r\n",
            "\u001b[2mpyarrow   \u001b[0m \u001b[32m------------------\u001b[2m------------\u001b[0m\u001b[0m 27.73 MiB/47.77 MiB                   \u001b[1A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1A\u001b[37m\u2838\u001b[0m \u001b[2mPreparing packages...\u001b[0m (3/5)\r\n",
            "\u001b[2mpyarrow   \u001b[0m \u001b[32m--------------------\u001b[2m----------\u001b[0m\u001b[0m 30.47 MiB/47.77 MiB                   \u001b[1A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1A\u001b[37m\u283c\u001b[0m \u001b[2mPreparing packages...\u001b[0m (4/5)\r\n",
            "\u001b[2mpyarrow   \u001b[0m \u001b[32m----------------------\u001b[2m--------\u001b[0m\u001b[0m 33.67 MiB/47.77 MiB                   \u001b[1A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1A\u001b[37m\u283c\u001b[0m \u001b[2mPreparing packages...\u001b[0m (4/5)\r\n",
            "\u001b[2mpyarrow   \u001b[0m \u001b[32m-----------------------\u001b[2m-------\u001b[0m\u001b[0m 36.00 MiB/47.77 MiB                   \u001b[1A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1A\u001b[37m\u283c\u001b[0m \u001b[2mPreparing packages...\u001b[0m (4/5)\r\n",
            "\u001b[2mpyarrow   \u001b[0m \u001b[32m-------------------------\u001b[2m-----\u001b[0m\u001b[0m 38.79 MiB/47.77 MiB                   \u001b[1A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1A\u001b[37m\u2834\u001b[0m \u001b[2mPreparing packages...\u001b[0m (4/5)\r\n",
            "\u001b[2mpyarrow   \u001b[0m \u001b[32m---------------------------\u001b[2m---\u001b[0m\u001b[0m 41.76 MiB/47.77 MiB                   \u001b[1A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1A\u001b[37m\u2834\u001b[0m \u001b[2mPreparing packages...\u001b[0m (4/5)\r\n",
            "\u001b[2mpyarrow   \u001b[0m \u001b[32m----------------------------\u001b[2m--\u001b[0m\u001b[0m 44.22 MiB/47.77 MiB                   \u001b[1A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1A\u001b[37m\u2834\u001b[0m \u001b[2mPreparing packages...\u001b[0m (4/5)\r\n",
            "\u001b[2mpyarrow   \u001b[0m \u001b[32m-----------------------------\u001b[2m-\u001b[0m\u001b[0m 45.86 MiB/47.77 MiB                   \u001b[1A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1A\u001b[37m\u2834\u001b[0m \u001b[2mPreparing packages...\u001b[0m (4/5)\r\n",
            "\u001b[2mpyarrow   \u001b[0m \u001b[32m------------------------------\u001b[2m\u001b[0m\u001b[0m 46.45 MiB/47.77 MiB                   \u001b[1A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1A\u001b[37m\u2834\u001b[0m \u001b[2mPreparing packages...\u001b[0m (4/5)\r\n",
            "\u001b[2mpyarrow   \u001b[0m \u001b[32m------------------------------\u001b[2m\u001b[0m\u001b[0m 47.33 MiB/47.77 MiB                   \u001b[1A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1A\u001b[37m\u2826\u001b[0m \u001b[2mPreparing packages...\u001b[0m (4/5)                                                   \r\u001b[2K\u001b[2mPrepared \u001b[1m5 packages\u001b[0m \u001b[2min 1.02s\u001b[0m\u001b[0m\r\n",
            "\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591 [0/0] \u001b[2mInstalling wheels...                                 \u001b[0m\r\u001b[2K\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591 [0/5] \u001b[2mInstalling wheels...                                 \u001b[0m\r\u001b[2K\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591 [0/5] \u001b[2mxxhash==3.8.1                                        \u001b[0m\r\u001b[2K\u2588\u2588\u2588\u2588\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591 [1/5] \u001b[2mxxhash==3.8.1                                        \u001b[0m\r\u001b[2K\u2588\u2588\u2588\u2588\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591 [1/5] \u001b[2mmultiprocess==0.70.19                                \u001b[0m\r\u001b[2K\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591 [2/5] \u001b[2mmultiprocess==0.70.19                                \u001b[0m\r\u001b[2K\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591 [2/5] \u001b[2mdill==0.4.1                                          \u001b[0m\r\u001b[2K\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591 [3/5] \u001b[2mdill==0.4.1                                          \u001b[0m\r\u001b[2K\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591 [3/5] \u001b[2mdatasets==5.0.0                                      \u001b[0m\r\u001b[2K\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2591\u2591\u2591\u2591 [4/5] \u001b[2mdatasets==5.0.0                                      \u001b[0m\r\u001b[2K\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2591\u2591\u2591\u2591 [4/5] \u001b[2mpyarrow==25.0.0                                      \u001b[0m\r\u001b[2K\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588 [5/5] \u001b[2mpyarrow==25.0.0                                      \u001b[0m\r\u001b[2K\u001b[2mInstalled \u001b[1m5 packages\u001b[0m \u001b[2min 83ms\u001b[0m\u001b[0m\r\n",
            " \u001b[32m+\u001b[39m \u001b[1mdatasets\u001b[0m\u001b[2m==5.0.0\u001b[0m\r\n",
            " \u001b[32m+\u001b[39m \u001b[1mdill\u001b[0m\u001b[2m==0.4.1\u001b[0m\r\n",
            " \u001b[32m+\u001b[39m \u001b[1mmultiprocess\u001b[0m\u001b[2m==0.70.19\u001b[0m\r\n",
            " \u001b[32m+\u001b[39m \u001b[1mpyarrow\u001b[0m\u001b[2m==25.0.0\u001b[0m\r\n",
            " \u001b[32m+\u001b[39m \u001b[1mxxhash\u001b[0m\u001b[2m==3.8.1\u001b[0m\r\n",
            "Note: you may need to restart the kernel to use updated packages.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "collapsed": false,
        "scrolled": true
      },
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.optim as optim\n",
        "from datasets import Dataset, load_dataset, concatenate_datasets\n",
        "from tokenizers import Tokenizer, Regex\n",
        "from tokenizers.models import BPE\n",
        "from tokenizers.pre_tokenizers import Sequence, Whitespace, Punctuation, Split\n",
        "from tokenizers.trainers import BpeTrainer\n",
        "from huggingface_hub import PyTorchModelHubMixin, notebook_login, HfApi"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "collapsed": false,
        "scrolled": true
      },
      "source": [
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "default_config = dict(\n",
        "        context_window=2,\n",
        "        vocab_size=0,\n",
        "        embedding_dim=200,\n",
        "\n",
        "        batch_size=8192,\n",
        "        num_negatives=5\n",
        ")"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "collapsed": false,
        "scrolled": true
      },
      "source": [
        "ds_train = load_dataset(\"bobox/OpenbookQA-4ST\", data_dir=\"filtered\", split=\"train+validation\", revision=\"refs/convert/parquet\")\n",
        "ds_test = load_dataset(\"bobox/OpenbookQA-4ST\", data_dir=\"filtered\", split=\"test\", revision=\"refs/convert/parquet\")\n",
        "ds = concatenate_datasets([ds_train, ds_test])\n",
        "ds_len = len(ds_train)+len(ds_test)"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c84e223fda7b40dface4157869fb589b",
              "version_minor": 0.0,
              "version_major": 2.0
            },
            "text/plain": "filtered/train/0000.parquet:   0%|          | 0.00/328k [00:00<?, ?B/s]"
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "06420636fba64f92a6870b33da17e20e",
              "version_minor": 0.0,
              "version_major": 2.0
            },
            "text/plain": "filtered/validation/0000.parquet:   0%|          | 0.00/41.8k [00:00<?, ?B/s]"
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "027daf714a8240f98189eded8f4edec6",
              "version_minor": 0.0,
              "version_major": 2.0
            },
            "text/plain": "filtered/test/0000.parquet:   0%|          | 0.00/48.2k [00:00<?, ?B/s]"
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "7f0903bfd5d24caea5033dda18c0e77a",
              "version_minor": 0.0,
              "version_major": 2.0
            },
            "text/plain": "Generating train split: 0 examples [00:00, ? examples/s]"
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "fefa92bf3e604b35949fbde0b24ce40c",
              "version_minor": 0.0,
              "version_major": 2.0
            },
            "text/plain": "Generating validation split: 0 examples [00:00, ? examples/s]"
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "cc77d154ff814d7bba05e723f43331f3",
              "version_minor": 0.0,
              "version_major": 2.0
            },
            "text/plain": "Generating test split: 0 examples [00:00, ? examples/s]"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "collapsed": false,
        "scrolled": true
      },
      "source": [
        "tokenizer = Tokenizer(BPE(unk_token=\"[UNK]\"))\n",
        "tokenizer.pre_tokenizer = Sequence([\n",
        "        Whitespace(),\n",
        "        Punctuation(behavior=\"removed\"),\n",
        "        Split(Regex(r\"\\d+\"), behavior=\"removed\")\n",
        "])\n",
        "\n",
        "tokenizer_trainer = BpeTrainer(special_tokens=[\"[UNK]\"], max_token_length=4)\n",
        "\n",
        "def batcher(n_batch=1000):\n",
        "        for i in range(0, ds_len, n_batch):\n",
        "                yield ds[\"fact\"][i : i + n_batch]\n",
        "\n",
        "tokenizer.train_from_iterator(batcher(), trainer=tokenizer_trainer, length=ds_len)\n",
        "\n",
        "tokenizer.save(\"tokenizer.json\")"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "collapsed": false,
        "scrolled": true
      },
      "source": [
        "class FastText(nn.Module, PyTorchModelHubMixin):\n",
        "        def __init__(self, config):\n",
        "                nn.Module.__init__(self)\n",
        "\n",
        "                self.context_window = config[\"context_window\"]\n",
        "                self.vocab_size = config[\"vocab_size\"]\n",
        "                self.embedding_dim = config[\"embedding_dim\"]\n",
        "\n",
        "                self.embedding = nn.EmbeddingBag(self.vocab_size, self.embedding_dim, mode=\"sum\", sparse=True)\n",
        "                self.context_embedding = nn.Embedding(self.vocab_size, self.embedding_dim, sparse=True)\n",
        "\n",
        "        def forward(self, input_ids, offsets, context_ids, negative_ids):\n",
        "                u_wt = self.embedding(input_ids, offsets)\n",
        "                v_wc = self.context_embedding(context_ids)\n",
        "\n",
        "                pos_score = (u_wt * v_wc).sum(dim=1)\n",
        "\n",
        "                v_neg = self.context_embedding(negative_ids)\n",
        "                neg_score = torch.bmm(v_neg, u_wt.unsqueeze(2)).squeeze(2)\n",
        "\n",
        "                log1p_exp = lambda x: torch.log1p(torch.exp(-x.abs())) + torch.relu(x)\n",
        "                loss = log1p_exp(-pos_score).mean() + log1p_exp(neg_score).sum(dim=1).mean()\n",
        "                return loss"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "collapsed": false,
        "scrolled": true
      },
      "source": [
        "default_config[\"vocab_size\"] = tokenizer.get_vocab_size()\n",
        "PurpleFastText = FastText(config=default_config).to(device)"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "collapsed": false,
        "scrolled": true
      },
      "source": [
        "def format_data(batch):\n",
        "        input_ids = []\n",
        "        offsets = []\n",
        "        context_ids = []\n",
        "        negative_ids = []\n",
        "\n",
        "        for sentence in batch[\"fact\"]:\n",
        "                tokens = tokenizer.encode(sentence)\n",
        "                toks = tokens.ids\n",
        "\n",
        "                for i in range(len(toks)):\n",
        "                        start = max(0, i - default_config[\"context_window\"])\n",
        "                        end = min(len(toks), i + default_config[\"context_window\"] + 1)\n",
        "\n",
        "                        for j in range(start, end):\n",
        "                                if i != j:\n",
        "                                        input_ids.append(toks[i])\n",
        "                                        context_ids.append(toks[j])\n",
        "\n",
        "        negative_ids = (\n",
        "                torch.randint(\n",
        "                        low=0,\n",
        "                        high=default_config[\"vocab_size\"],\n",
        "                        size=(len(input_ids), default_config[\"num_negatives\"]),\n",
        "                        dtype=torch.long\n",
        "                ).tolist()\n",
        "        )\n",
        "\n",
        "        return dict(\n",
        "                input_ids=input_ids,\n",
        "                context_ids=context_ids,\n",
        "                negative_ids=negative_ids\n",
        "        )\n",
        "\n",
        "def collator(batch):\n",
        "        batch_dict = torch.utils.data.default_collate(batch)\n",
        "\n",
        "        input_ids = batch_dict[\"input_ids\"]\n",
        "        context_ids = batch_dict[\"context_ids\"]\n",
        "        negative_ids = batch_dict[\"negative_ids\"]\n",
        "\n",
        "        offsets = torch.arange(len(input_ids), dtype=torch.long)\n",
        "\n",
        "        return input_ids, offsets, context_ids, negative_ids\n",
        "\n",
        "dl = torch.utils.data.DataLoader(\n",
        "        ds_train.map(format_data, batched=True, remove_columns=ds_train.column_names).with_format(\"torch\"), # should've used IterableDataset from torch\n",
        "        batch_size=default_config[\"batch_size\"], shuffle=True,\n",
        "        collate_fn=collator,\n",
        "        num_workers=8, persistent_workers=True, prefetch_factor=2,\n",
        "        pin_memory=True if device.type == \"cuda\" else False\n",
        ")"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "collapsed": false,
        "scrolled": true
      },
      "source": [
        "optimizer = optim.SparseAdam(PurpleFastText.parameters(), lr=1e-4)\n",
        "\n",
        "for epoch in range(200):\n",
        "        for (input_ids, offsets, context_ids, negative_ids) in dl:\n",
        "                optimizer.zero_grad()\n",
        "                loss = PurpleFastText(input_ids.to(device), offsets.to(device), context_ids.to(device), negative_ids.to(device))\n",
        "                loss.backward()\n",
        "                optimizer.step()\n",
        "\n",
        "        loss = loss.item()\n",
        "        print(f\"{epoch = } | {loss = }\")"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "epoch = 0 | loss = 33.62425231933594\n",
            "epoch = 1 | loss = 33.472225189208984\n",
            "epoch = 2 | loss = 33.398258209228516\n",
            "epoch = 3 | loss = 33.84187316894531\n",
            "epoch = 4 | loss = 32.96810531616211\n",
            "epoch = 5 | loss = 32.768104553222656\n",
            "epoch = 6 | loss = 32.597679138183594\n",
            "epoch = 7 | loss = 32.50499725341797\n",
            "epoch = 8 | loss = 32.46271514892578\n",
            "epoch = 9 | loss = 31.799774169921875\n",
            "epoch = 10 | loss = 31.96149253845215\n",
            "epoch = 11 | loss = 31.556041717529297\n",
            "epoch = 12 | loss = 31.90681266784668\n",
            "epoch = 13 | loss = 31.102127075195312\n",
            "epoch = 14 | loss = 31.5416316986084\n",
            "epoch = 15 | loss = 31.22119140625\n",
            "epoch = 16 | loss = 31.240070343017578\n",
            "epoch = 17 | loss = 30.6594181060791\n",
            "epoch = 18 | loss = 30.212684631347656\n",
            "epoch = 19 | loss = 30.307228088378906\n",
            "epoch = 20 | loss = 30.078458786010742\n",
            "epoch = 21 | loss = 29.99860382080078\n",
            "epoch = 22 | loss = 29.850513458251953\n",
            "epoch = 23 | loss = 29.601530075073242\n",
            "epoch = 24 | loss = 29.231857299804688\n",
            "epoch = 25 | loss = 29.260231018066406\n",
            "epoch = 26 | loss = 29.491256713867188\n",
            "epoch = 27 | loss = 28.56077003479004\n",
            "epoch = 28 | loss = 29.046772003173828\n",
            "epoch = 29 | loss = 28.38245964050293\n",
            "epoch = 30 | loss = 28.465112686157227\n",
            "epoch = 31 | loss = 28.316547393798828\n",
            "epoch = 32 | loss = 28.164907455444336\n",
            "epoch = 33 | loss = 27.77496910095215\n",
            "epoch = 34 | loss = 27.681516647338867\n",
            "epoch = 35 | loss = 27.743240356445312\n",
            "epoch = 36 | loss = 27.441387176513672\n",
            "epoch = 37 | loss = 27.788480758666992\n",
            "epoch = 38 | loss = 27.45903778076172\n",
            "epoch = 39 | loss = 27.11681365966797\n",
            "epoch = 40 | loss = 27.122961044311523\n",
            "epoch = 41 | loss = 26.90481948852539\n",
            "epoch = 42 | loss = 26.61617088317871\n",
            "epoch = 43 | loss = 26.463600158691406\n",
            "epoch = 44 | loss = 26.532682418823242\n",
            "epoch = 45 | loss = 26.35540008544922\n",
            "epoch = 46 | loss = 26.15418243408203\n",
            "epoch = 47 | loss = 26.277740478515625\n",
            "epoch = 48 | loss = 26.20408058166504\n",
            "epoch = 49 | loss = 25.71695327758789\n",
            "epoch = 50 | loss = 25.684810638427734\n",
            "epoch = 51 | loss = 25.397275924682617\n",
            "epoch = 52 | loss = 25.11101531982422\n",
            "epoch = 53 | loss = 25.443233489990234\n",
            "epoch = 54 | loss = 24.976665496826172\n",
            "epoch = 55 | loss = 24.98261070251465\n",
            "epoch = 56 | loss = 24.903249740600586\n",
            "epoch = 57 | loss = 24.44039535522461\n",
            "epoch = 58 | loss = 24.10711669921875\n",
            "epoch = 59 | loss = 24.02779769897461\n",
            "epoch = 60 | loss = 24.355669021606445\n",
            "epoch = 61 | loss = 23.761302947998047\n",
            "epoch = 62 | loss = 24.263946533203125\n",
            "epoch = 63 | loss = 23.884416580200195\n",
            "epoch = 64 | loss = 23.76875877380371\n",
            "epoch = 65 | loss = 23.63066864013672\n",
            "epoch = 66 | loss = 23.35961151123047\n",
            "epoch = 67 | loss = 23.55689239501953\n",
            "epoch = 68 | loss = 23.246044158935547\n",
            "epoch = 69 | loss = 22.76032257080078\n",
            "epoch = 70 | loss = 23.042325973510742\n",
            "epoch = 71 | loss = 22.743942260742188\n",
            "epoch = 72 | loss = 22.37074089050293\n",
            "epoch = 73 | loss = 22.6041259765625\n",
            "epoch = 74 | loss = 22.25589370727539\n",
            "epoch = 75 | loss = 22.096574783325195\n",
            "epoch = 76 | loss = 22.071853637695312\n",
            "epoch = 77 | loss = 22.31040382385254\n",
            "epoch = 78 | loss = 22.118852615356445\n",
            "epoch = 79 | loss = 21.45325469970703\n",
            "epoch = 80 | loss = 21.61875343322754\n",
            "epoch = 81 | loss = 21.439592361450195\n",
            "epoch = 82 | loss = 21.3583927154541\n",
            "epoch = 83 | loss = 21.06571388244629\n",
            "epoch = 84 | loss = 20.578998565673828\n",
            "epoch = 85 | loss = 20.79303550720215\n",
            "epoch = 86 | loss = 20.762752532958984\n",
            "epoch = 87 | loss = 20.799985885620117\n",
            "epoch = 88 | loss = 20.517189025878906\n",
            "epoch = 89 | loss = 20.281986236572266\n",
            "epoch = 90 | loss = 20.166234970092773\n",
            "epoch = 91 | loss = 20.12177848815918\n",
            "epoch = 92 | loss = 20.381624221801758\n",
            "epoch = 93 | loss = 19.79176902770996\n",
            "epoch = 94 | loss = 19.528615951538086\n",
            "epoch = 95 | loss = 19.95457649230957\n",
            "epoch = 96 | loss = 19.54012680053711\n",
            "epoch = 97 | loss = 19.215112686157227\n",
            "epoch = 98 | loss = 19.4979305267334\n",
            "epoch = 99 | loss = 19.16718864440918\n",
            "epoch = 100 | loss = 18.99817657470703\n",
            "epoch = 101 | loss = 18.84443473815918\n",
            "epoch = 102 | loss = 18.61274528503418\n",
            "epoch = 103 | loss = 18.64920425415039\n",
            "epoch = 104 | loss = 18.58812141418457\n",
            "epoch = 105 | loss = 18.30907440185547\n",
            "epoch = 106 | loss = 18.271347045898438\n",
            "epoch = 107 | loss = 18.45148468017578\n",
            "epoch = 108 | loss = 18.378337860107422\n",
            "epoch = 109 | loss = 18.269376754760742\n",
            "epoch = 110 | loss = 17.706981658935547\n",
            "epoch = 111 | loss = 17.62091064453125\n",
            "epoch = 112 | loss = 17.225004196166992\n",
            "epoch = 113 | loss = 17.723031997680664\n",
            "epoch = 114 | loss = 17.18803596496582\n",
            "epoch = 115 | loss = 17.075786590576172\n",
            "epoch = 116 | loss = 16.95713233947754\n",
            "epoch = 117 | loss = 16.833356857299805\n",
            "epoch = 118 | loss = 16.814830780029297\n",
            "epoch = 119 | loss = 17.00945472717285\n",
            "epoch = 120 | loss = 16.623493194580078\n",
            "epoch = 121 | loss = 16.313987731933594\n",
            "epoch = 122 | loss = 16.316606521606445\n",
            "epoch = 123 | loss = 16.264816284179688\n",
            "epoch = 124 | loss = 15.747238159179688\n",
            "epoch = 125 | loss = 15.969701766967773\n",
            "epoch = 126 | loss = 16.052602767944336\n",
            "epoch = 127 | loss = 16.00865936279297\n",
            "epoch = 128 | loss = 15.772383689880371\n",
            "epoch = 129 | loss = 15.535850524902344\n",
            "epoch = 130 | loss = 15.065999031066895\n",
            "epoch = 131 | loss = 15.313947677612305\n",
            "epoch = 132 | loss = 15.149383544921875\n",
            "epoch = 133 | loss = 15.007120132446289\n",
            "epoch = 134 | loss = 15.279787063598633\n",
            "epoch = 135 | loss = 14.991166114807129\n",
            "epoch = 136 | loss = 14.836761474609375\n",
            "epoch = 137 | loss = 14.972814559936523\n",
            "epoch = 138 | loss = 14.291553497314453\n",
            "epoch = 139 | loss = 14.598798751831055\n",
            "epoch = 140 | loss = 14.328827857971191\n",
            "epoch = 141 | loss = 14.343247413635254\n",
            "epoch = 142 | loss = 14.173320770263672\n",
            "epoch = 143 | loss = 14.070396423339844\n",
            "epoch = 144 | loss = 13.911269187927246\n",
            "epoch = 145 | loss = 13.662429809570312\n",
            "epoch = 146 | loss = 13.669416427612305\n",
            "epoch = 147 | loss = 13.736848831176758\n",
            "epoch = 148 | loss = 13.631596565246582\n",
            "epoch = 149 | loss = 13.368789672851562\n",
            "epoch = 150 | loss = 13.092583656311035\n",
            "epoch = 151 | loss = 13.113190650939941\n",
            "epoch = 152 | loss = 13.169113159179688\n",
            "epoch = 153 | loss = 13.064599990844727\n",
            "epoch = 154 | loss = 12.857305526733398\n",
            "epoch = 155 | loss = 12.743917465209961\n",
            "epoch = 156 | loss = 12.533823013305664\n",
            "epoch = 157 | loss = 12.30309772491455\n",
            "epoch = 158 | loss = 12.425459861755371\n",
            "epoch = 159 | loss = 12.448675155639648\n",
            "epoch = 160 | loss = 12.417452812194824\n",
            "epoch = 161 | loss = 12.414138793945312\n",
            "epoch = 162 | loss = 12.402454376220703\n",
            "epoch = 163 | loss = 12.141651153564453\n",
            "epoch = 164 | loss = 12.028185844421387\n",
            "epoch = 165 | loss = 11.94282341003418\n",
            "epoch = 166 | loss = 11.632282257080078\n",
            "epoch = 167 | loss = 11.308302879333496\n",
            "epoch = 168 | loss = 11.397911071777344\n",
            "epoch = 169 | loss = 11.540604591369629\n",
            "epoch = 170 | loss = 11.422112464904785\n",
            "epoch = 171 | loss = 11.166105270385742\n",
            "epoch = 172 | loss = 11.410856246948242\n",
            "epoch = 173 | loss = 11.141489028930664\n",
            "epoch = 174 | loss = 11.055648803710938\n",
            "epoch = 175 | loss = 11.138644218444824\n",
            "epoch = 176 | loss = 10.659797668457031\n",
            "epoch = 177 | loss = 10.853754043579102\n",
            "epoch = 178 | loss = 10.80173110961914\n",
            "epoch = 179 | loss = 10.684595108032227\n",
            "epoch = 180 | loss = 10.418027877807617\n",
            "epoch = 181 | loss = 10.73669719696045\n",
            "epoch = 182 | loss = 10.225930213928223\n",
            "epoch = 183 | loss = 10.185312271118164\n",
            "epoch = 184 | loss = 10.047532081604004\n",
            "epoch = 185 | loss = 10.030939102172852\n",
            "epoch = 186 | loss = 10.077144622802734\n",
            "epoch = 187 | loss = 9.813972473144531\n",
            "epoch = 188 | loss = 9.702218055725098\n",
            "epoch = 189 | loss = 9.979422569274902\n",
            "epoch = 190 | loss = 9.73726749420166\n",
            "epoch = 191 | loss = 9.609981536865234\n",
            "epoch = 192 | loss = 9.520135879516602\n",
            "epoch = 193 | loss = 9.530278205871582\n",
            "epoch = 194 | loss = 9.442426681518555\n",
            "epoch = 195 | loss = 9.331952095031738\n",
            "epoch = 196 | loss = 9.179746627807617\n",
            "epoch = 197 | loss = 9.125386238098145\n",
            "epoch = 198 | loss = 9.122198104858398\n",
            "epoch = 199 | loss = 9.065939903259277\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "collapsed": false,
        "scrolled": true
      },
      "source": [
        "notebook_login()"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "dc1ce00109ba423a9a47d08ef1c453d7",
              "version_minor": 0.0,
              "version_major": 2.0
            },
            "text/plain": "VBox(children=(HTML(value='<center> <img\\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.sv\u2026"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "collapsed": false,
        "scrolled": true
      },
      "source": [
        "PurpleFastText.push_to_hub(\"LocalWisdom/PurpleFastText\")"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "cce21a8db91c44bca9595f6e81a7db96",
              "version_minor": 0.0,
              "version_major": 2.0
            },
            "text/plain": "Processing Files (0 / 0)                : |          |  0.00B /  0.00B            "
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "ba50891c5c514f208daf6febc9f34d6f",
              "version_minor": 0.0,
              "version_major": 2.0
            },
            "text/plain": "New Data Upload                         : |          |  0.00B /  0.00B            "
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b016ed071f6c451dbb9ca978a6663975",
              "version_minor": 0.0,
              "version_major": 2.0
            },
            "text/plain": "  ...om/PurpleFastText/model.safetensors: 100%|##########| 1.43MB / 1.43MB            "
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "execution_count": 15,
          "data": {
            "text/plain": "CommitInfo(commit_url='https://huggingface.co/LocalWisdom/PurpleFastText/commit/6b71cc46afb2660032e44f86fd8ced34b032c3a1', commit_message='Push model using huggingface_hub.', commit_description='', oid='6b71cc46afb2660032e44f86fd8ced34b032c3a1', pr_url=None, repo_url=RepoUrl('https://huggingface.co/LocalWisdom/PurpleFastText', endpoint='https://huggingface.co', repo_type='model', repo_id='LocalWisdom/PurpleFastText'), pr_revision=None, pr_num=None)"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "collapsed": false,
        "scrolled": true
      },
      "source": [
        "api = HfApi()\n",
        "\n",
        "api.upload_file(\n",
        "    path_or_fileobj=\"tokenizer.json\",\n",
        "    path_in_repo=\"tokenizer.json\",\n",
        "    repo_id=\"LocalWisdom/PurpleFastText\",\n",
        "    repo_type=\"model\"\n",
        ")"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "execute_result",
          "execution_count": 19,
          "data": {
            "text/plain": "CommitInfo(commit_url='https://huggingface.co/LocalWisdom/PurpleFastText/commit/afb9be250708a9d86719ae4a1e74330776abd9f0', commit_message='Upload tokenizer.json with huggingface_hub', commit_description='', oid='afb9be250708a9d86719ae4a1e74330776abd9f0', pr_url=None, repo_url=RepoUrl('https://huggingface.co/LocalWisdom/PurpleFastText', endpoint='https://huggingface.co', repo_type='model', repo_id='LocalWisdom/PurpleFastText'), pr_revision=None, pr_num=None)"
          },
          "metadata": {}
        }
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 5
}