{ "cells": [ { "cell_type": "code", "source": [ "import torch\n", "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", "device" ], "outputs": [ { "output_type": "stream", "name": "stderr", "text": "/anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n from .autonotebook import tqdm as notebook_tqdm\n" }, { "output_type": "execute_result", "execution_count": 1, "data": { "text/plain": "device(type='cpu')" }, "metadata": {} } ], "execution_count": 1, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "jFv37fzUD_8e", "executionInfo": { "status": "ok", "timestamp": 1716314364319, "user_tz": -60, "elapsed": 333, "user": { "displayName": "Massimo Frasson", "userId": "09747843240273532020" } }, "outputId": "0b64d28b-9575-4d6a-8327-abdfbf6c05ad", "gather": { "logged": 1716385135085 } } }, { "cell_type": "markdown", "source": [ "# Install dependencies" ], "metadata": { "id": "N8DkDmkh7w2n" } }, { "cell_type": "code", "source": [ "%pip install datasets transformers" ], "outputs": [ { "output_type": "stream", "name": "stdout", "text": "Requirement already satisfied: datasets in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (2.19.1)\nRequirement already satisfied: transformers in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (4.41.0)\nRequirement already satisfied: tqdm>=4.62.1 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from datasets) (4.65.0)\nRequirement already satisfied: multiprocess in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from datasets) (0.70.16)\nRequirement already satisfied: pandas in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from datasets) (2.0.2)\nRequirement already satisfied: xxhash in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from datasets) (3.4.1)\nRequirement already satisfied: dill<0.3.9,>=0.3.0 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from datasets) (0.3.8)\nRequirement already satisfied: filelock in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from datasets) (3.14.0)\nRequirement already satisfied: fsspec[http]<=2024.3.1,>=2023.1.0 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from datasets) (2024.3.1)\nRequirement already satisfied: aiohttp in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from datasets) (3.9.5)\nRequirement already satisfied: numpy>=1.17 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from datasets) (1.23.5)\nRequirement already satisfied: pyarrow>=12.0.0 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from datasets) (16.1.0)\nRequirement already satisfied: requests>=2.19.0 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from datasets) (2.31.0)\nRequirement already satisfied: huggingface-hub>=0.21.2 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from datasets) (0.23.1)\nRequirement already satisfied: pyarrow-hotfix in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from datasets) (0.6)\nRequirement already satisfied: packaging in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from datasets) (23.1)\nRequirement already satisfied: pyyaml>=5.1 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from datasets) (6.0)\nRequirement already satisfied: regex!=2019.12.17 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from transformers) (2024.5.15)\nRequirement already satisfied: tokenizers<0.20,>=0.19 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from transformers) (0.19.1)\nRequirement already satisfied: safetensors>=0.4.1 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from transformers) (0.4.3)\nRequirement already satisfied: async-timeout<5.0,>=4.0 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from aiohttp->datasets) (4.0.3)\nRequirement already satisfied: aiosignal>=1.1.2 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from aiohttp->datasets) (1.3.1)\nRequirement already satisfied: yarl<2.0,>=1.0 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from aiohttp->datasets) (1.9.4)\nRequirement already satisfied: attrs>=17.3.0 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from aiohttp->datasets) (23.1.0)\nRequirement already satisfied: multidict<7.0,>=4.5 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from aiohttp->datasets) (6.0.5)\nRequirement already satisfied: frozenlist>=1.1.1 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from aiohttp->datasets) (1.4.1)\nRequirement already satisfied: typing-extensions>=3.7.4.3 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from huggingface-hub>=0.21.2->datasets) (4.6.3)\nRequirement already satisfied: charset-normalizer<4,>=2 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from requests>=2.19.0->datasets) (3.1.0)\nRequirement already satisfied: urllib3<3,>=1.21.1 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from requests>=2.19.0->datasets) (1.26.16)\nRequirement already satisfied: idna<4,>=2.5 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from requests>=2.19.0->datasets) (3.4)\nRequirement already satisfied: certifi>=2017.4.17 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from requests>=2.19.0->datasets) (2023.5.7)\nRequirement already satisfied: python-dateutil>=2.8.2 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from pandas->datasets) (2.8.2)\nRequirement already satisfied: tzdata>=2022.1 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from pandas->datasets) (2023.3)\nRequirement already satisfied: pytz>=2020.1 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from pandas->datasets) (2023.3)\nRequirement already satisfied: six>=1.5 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from python-dateutil>=2.8.2->pandas->datasets) (1.16.0)\nNote: you may need to restart the kernel to use updated packages.\n" } ], "execution_count": 2, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "jNoKtDn97c96", "executionInfo": { "status": "ok", "timestamp": 1716313623682, "user_tz": -60, "elapsed": 11725, "user": { "displayName": "Massimo Frasson", "userId": "09747843240273532020" } }, "outputId": "0e2c77dc-9b8b-4319-a271-48d2f05225a5" } }, { "cell_type": "markdown", "source": [ "# Named Entity Recogntion (Token Classification)" ], "metadata": { "id": "w0y8z4xm-ONA" } }, { "cell_type": "markdown", "source": [ "Given a dictionary $D$ and a label set $L$, token classification is: $D^s \\to L^s$, where $s$ is the size of the input sequence.\n", "\n", "As you probably know, the dictionary depends on the tokenization strategy we adopt.\n", "For instance: \"My name is Massimo\" could become:\n", "- [\"My, \"name\", \"is\", \"Massimo\"]\n", "- [\"my, \"name\", \"is\", \"massimo\"]\n", "- [\"my, \"na\", \"##me\", \"is\", \"ma\", \"##ss\", \"##imo\"]" ], "metadata": { "id": "kLrd-d0LBZ6f" } }, { "cell_type": "markdown", "source": [ "## Load the dataset" ], "metadata": { "id": "7AvpMZZ470Fs" } }, { "cell_type": "code", "source": [ "from datasets import load_dataset\n", "\n", "dataset = load_dataset('eriktks/conll2003')\n", "dataset" ], "outputs": [ { "output_type": "stream", "name": "stderr", "text": "/anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages/datasets/load.py:1486: FutureWarning: The repository for eriktks/conll2003 contains custom code which must be executed to correctly load the dataset. You can inspect the repository content at https://hf.co/datasets/eriktks/conll2003\nYou can avoid this message in future by passing the argument `trust_remote_code=True`.\nPassing `trust_remote_code=True` will be mandatory to load this dataset from the next major release of `datasets`.\n warnings.warn(\n" }, { "output_type": "execute_result", "execution_count": 3, "data": { "text/plain": "DatasetDict({\n train: Dataset({\n features: ['id', 'tokens', 'pos_tags', 'chunk_tags', 'ner_tags'],\n num_rows: 14041\n })\n validation: Dataset({\n features: ['id', 'tokens', 'pos_tags', 'chunk_tags', 'ner_tags'],\n num_rows: 3250\n })\n test: Dataset({\n features: ['id', 'tokens', 'pos_tags', 'chunk_tags', 'ner_tags'],\n num_rows: 3453\n })\n})" }, "metadata": {} } ], "execution_count": 3, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "xhYPltmU7Uxz", "executionInfo": { "status": "ok", "timestamp": 1716313627884, "user_tz": -60, "elapsed": 4205, "user": { "displayName": "Massimo Frasson", "userId": "09747843240273532020" } }, "outputId": "2b546b75-53a5-40ef-af84-6881ac0bedc4", "gather": { "logged": 1716385241429 } } }, { "cell_type": "markdown", "source": [ "As we can see this dataset comes with predefined train/test splits. Its common practice in research to allow reproducibility of experiments." ], "metadata": { "id": "C6hLoS2d92Yh" } }, { "cell_type": "code", "source": [ "row = dataset['test'][30]\n", "row" ], "outputs": [ { "output_type": "execute_result", "execution_count": 4, "data": { "text/plain": "{'id': '30',\n 'tokens': ['\"',\n 'He',\n 'ended',\n 'the',\n 'World',\n 'Cup',\n 'on',\n 'the',\n 'wrong',\n 'note',\n ',',\n '\"',\n 'Coste',\n 'said',\n '.'],\n 'pos_tags': [0, 28, 38, 12, 22, 22, 15, 12, 16, 21, 6, 0, 22, 38, 7],\n 'chunk_tags': [0, 11, 21, 11, 12, 12, 13, 11, 12, 12, 0, 0, 11, 21, 0],\n 'ner_tags': [0, 0, 0, 0, 7, 8, 0, 0, 0, 0, 0, 0, 1, 0, 0]}" }, "metadata": {} } ], "execution_count": 4, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "dlC6mWnc7Zdj", "executionInfo": { "status": "ok", "timestamp": 1716313627885, "user_tz": -60, "elapsed": 10, "user": { "displayName": "Massimo Frasson", "userId": "09747843240273532020" } }, "outputId": "5ffbfb77-459b-4eb7-8b0b-894ea620a6f4", "gather": { "logged": 1716385277857 } } }, { "cell_type": "code", "source": [ "import pandas as pd\n", "\n", "pd.DataFrame([\n", " row['tokens'],\n", " row['pos_tags'],\n", " row['chunk_tags'],\n", " row['ner_tags']\n", "], index=['tokens', 'pos_tags', 'chunk_tags', 'ner_tags'])" ], "outputs": [ { "output_type": "execute_result", "execution_count": 5, "data": { "text/plain": " 0 1 2 3 4 5 6 7 8 9 10 11 12 \\\ntokens \" He ended the World Cup on the wrong note , \" Coste \npos_tags 0 28 38 12 22 22 15 12 16 21 6 0 22 \nchunk_tags 0 11 21 11 12 12 13 11 12 12 0 0 11 \nner_tags 0 0 0 0 7 8 0 0 0 0 0 0 1 \n\n 13 14 \ntokens said . \npos_tags 38 7 \nchunk_tags 21 0 \nner_tags 0 0 ", "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
01234567891011121314
tokens\"HeendedtheWorldCuponthewrongnote,\"Costesaid.
pos_tags02838122222151216216022387
chunk_tags01121111212131112120011210
ner_tags000078000000100
\n
" }, "metadata": {} } ], "execution_count": 5, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 175 }, "id": "5N-vxUp69h3H", "executionInfo": { "status": "ok", "timestamp": 1716313627885, "user_tz": -60, "elapsed": 7, "user": { "displayName": "Massimo Frasson", "userId": "09747843240273532020" } }, "outputId": "1c27ddcd-e7c2-45ba-b0dc-ef25b0fe886f", "gather": { "logged": 1716385295161 } } }, { "cell_type": "markdown", "source": [ "We can see the meaning of each of those tags in the docs: https://huggingface.co/datasets/eriktks/conll2003\n", "\n", "We are interested in the ner ones for today." ], "metadata": { "id": "RnQBaFCP_fZj" } }, { "cell_type": "code", "source": [ "tag_to_id = {'O': 0, 'B-PER': 1, 'I-PER': 2, 'B-ORG': 3, 'I-ORG': 4, 'B-LOC': 5, 'I-LOC': 6, 'B-MISC': 7, 'I-MISC': 8}\n", "id_to_tag = {v: k for k, v in tag_to_id.items()}\n", "id_to_tag" ], "outputs": [ { "output_type": "execute_result", "execution_count": 6, "data": { "text/plain": "{0: 'O',\n 1: 'B-PER',\n 2: 'I-PER',\n 3: 'B-ORG',\n 4: 'I-ORG',\n 5: 'B-LOC',\n 6: 'I-LOC',\n 7: 'B-MISC',\n 8: 'I-MISC'}" }, "metadata": {} } ], "execution_count": 6, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "2oApQP1x-q9T", "executionInfo": { "status": "ok", "timestamp": 1716313627885, "user_tz": -60, "elapsed": 6, "user": { "displayName": "Massimo Frasson", "userId": "09747843240273532020" } }, "outputId": "0f39436b-4b3d-4d6f-b673-6887cd7feb1a", "gather": { "logged": 1716385348456 } } }, { "cell_type": "code", "source": [ "pd.DataFrame([\n", " row['tokens'],\n", " [id_to_tag[id] for id in row['ner_tags']]\n", "], index=['tokens', 'ner_tags'])" ], "outputs": [ { "output_type": "execute_result", "execution_count": 7, "data": { "text/plain": " 0 1 2 3 4 5 6 7 8 9 10 11 \\\ntokens \" He ended the World Cup on the wrong note , \" \nner_tags O O O O B-MISC I-MISC O O O O O O \n\n 12 13 14 \ntokens Coste said . \nner_tags B-PER O O ", "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
01234567891011121314
tokens\"HeendedtheWorldCuponthewrongnote,\"Costesaid.
ner_tagsOOOOB-MISCI-MISCOOOOOOB-PEROO
\n
" }, "metadata": {} } ], "execution_count": 7, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 112 }, "id": "FMldpLAU_69J", "executionInfo": { "status": "ok", "timestamp": 1716313628264, "user_tz": -60, "elapsed": 383, "user": { "displayName": "Massimo Frasson", "userId": "09747843240273532020" } }, "outputId": "b15b50d8-bf45-453e-97d2-88ffd7f6c810", "gather": { "logged": 1716385350845 } } }, { "cell_type": "markdown", "source": [ "What are those B and I prefixes? [IOB2 labels](https://en.wikipedia.org/wiki/Inside%E2%80%93outside%E2%80%93beginning_(tagging)).\n", "Since a word can be split in two by the tokenizer, or it could just be a multiword entity like \"Word Cup\", we have a way to tag the initial token of the word and the following ones." ], "metadata": { "id": "gsBqttq9AJ6B" } }, { "cell_type": "markdown", "source": [ "## Let's quickly try a model from HuggingFace" ], "metadata": { "id": "mDjmI7rEDfx9" } }, { "cell_type": "code", "source": [ "from transformers import pipeline\n", "\n", "#model_name = 'dslim/bert-base-NER'\n", "model_name = 'dslim/bert-large-NER'\n", "\n", "pipe = pipeline('token-classification', model=model_name, device=device)" ], "outputs": [ { "output_type": "stream", "name": "stderr", "text": "2024-05-22 13:44:48.860927: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\nTo enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n2024-05-22 13:44:59.136843: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory\n2024-05-22 13:44:59.137041: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory\n2024-05-22 13:44:59.137054: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.\nSome weights of the model checkpoint at dslim/bert-large-NER were not used when initializing BertForTokenClassification: ['bert.pooler.dense.bias', 'bert.pooler.dense.weight']\n- This IS expected if you are initializing BertForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n- This IS NOT expected if you are initializing BertForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n" } ], "execution_count": 8, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "D9Hq4W4HAFmV", "executionInfo": { "status": "ok", "timestamp": 1716313639937, "user_tz": -60, "elapsed": 11675, "user": { "displayName": "Massimo Frasson", "userId": "09747843240273532020" } }, "outputId": "2e8a29bd-d697-457a-f3d0-c5246a8ac2ce", "gather": { "logged": 1716385518686 } } }, { "cell_type": "code", "source": [ "' '.join(row['tokens'])" ], "outputs": [ { "output_type": "execute_result", "execution_count": 9, "data": { "text/plain": "'\" He ended the World Cup on the wrong note , \" Coste said .'" }, "metadata": {} } ], "execution_count": 9, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 35 }, "id": "rWhOgV8JFE8I", "executionInfo": { "status": "ok", "timestamp": 1716313639938, "user_tz": -60, "elapsed": 14, "user": { "displayName": "Massimo Frasson", "userId": "09747843240273532020" } }, "outputId": "cb410a69-076d-4e33-9706-06678e9bbd25", "gather": { "logged": 1716385518976 } } }, { "cell_type": "code", "source": [ "pipe(row['tokens'])" ], "outputs": [ { "output_type": "execute_result", "execution_count": 10, "data": { "text/plain": "[[],\n [],\n [],\n [],\n [{'entity': 'B-MISC',\n 'score': 0.58489937,\n 'index': 1,\n 'word': 'World',\n 'start': 0,\n 'end': 5}],\n [{'entity': 'B-MISC',\n 'score': 0.9501208,\n 'index': 1,\n 'word': 'Cup',\n 'start': 0,\n 'end': 3}],\n [],\n [],\n [],\n [],\n [],\n [],\n [{'entity': 'B-PER',\n 'score': 0.9929893,\n 'index': 1,\n 'word': 'Co',\n 'start': 0,\n 'end': 2},\n {'entity': 'I-PER',\n 'score': 0.41799864,\n 'index': 2,\n 'word': '##ste',\n 'start': 2,\n 'end': 5}],\n [],\n []]" }, "metadata": {} } ], "execution_count": 10, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "wDTWzm8tEFch", "executionInfo": { "status": "ok", "timestamp": 1716313642548, "user_tz": -60, "elapsed": 2623, "user": { "displayName": "Massimo Frasson", "userId": "09747843240273532020" } }, "outputId": "cb512a7a-c7a5-4210-abe6-4f66d353ded8", "gather": { "logged": 1716385520632 } } }, { "cell_type": "markdown", "source": [ "We have a few problems here:\n", "- the results aren't great\n", "- we are struggling to build the input\n", "\n", "Let's go down one step of the abstraction layer, let's download both the tokenizer and the model." ], "metadata": { "id": "gyfcQIeqFqC3" } }, { "cell_type": "markdown", "source": [ "## The Auto Classes" ], "metadata": { "id": "Dg5D7kR-Ho7P" } }, { "cell_type": "code", "source": [ "from transformers import AutoTokenizer, AutoModelForTokenClassification, AutoConfig\n", "\n", "tokenizer = AutoTokenizer.from_pretrained(model_name)\n", "tokenizer" ], "outputs": [ { "output_type": "execute_result", "execution_count": 11, "data": { "text/plain": "BertTokenizerFast(name_or_path='dslim/bert-large-NER', vocab_size=28996, model_max_length=512, is_fast=True, padding_side='right', truncation_side='right', special_tokens={'unk_token': '[UNK]', 'sep_token': '[SEP]', 'pad_token': '[PAD]', 'cls_token': '[CLS]', 'mask_token': '[MASK]'}, clean_up_tokenization_spaces=True), added_tokens_decoder={\n\t0: AddedToken(\"[PAD]\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n\t100: AddedToken(\"[UNK]\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n\t101: AddedToken(\"[CLS]\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n\t102: AddedToken(\"[SEP]\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n\t103: AddedToken(\"[MASK]\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n}" }, "metadata": {} } ], "execution_count": 11, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "jeo4eRSRGhu9", "executionInfo": { "status": "ok", "timestamp": 1716313642987, "user_tz": -60, "elapsed": 441, "user": { "displayName": "Massimo Frasson", "userId": "09747843240273532020" } }, "outputId": "f8607732-9fd1-4da2-d99c-1dfe2dec3107", "gather": { "logged": 1716385648094 } } }, { "cell_type": "code", "source": [ "tokens = tokenizer(row['tokens'], is_split_into_words=True)\n", "tokens" ], "outputs": [ { "output_type": "execute_result", "execution_count": 12, "data": { "text/plain": "{'input_ids': [101, 107, 1124, 2207, 1103, 1291, 1635, 1113, 1103, 2488, 3805, 117, 107, 3291, 13894, 1163, 119, 102], 'token_type_ids': [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]}" }, "metadata": {} } ], "execution_count": 12, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "V7LFyVJ7ItSt", "executionInfo": { "status": "ok", "timestamp": 1716313642987, "user_tz": -60, "elapsed": 6, "user": { "displayName": "Massimo Frasson", "userId": "09747843240273532020" } }, "outputId": "a3b703d2-ab72-4289-a1a7-d1e321bbe917", "gather": { "logged": 1716385676253 } } }, { "cell_type": "code", "source": [ "tokenizer.decode(tokens['input_ids'], skip_special_tokens=True)" ], "outputs": [ { "output_type": "execute_result", "execution_count": 14, "data": { "text/plain": "'\" He ended the World Cup on the wrong note, \" Coste said.'" }, "metadata": {} } ], "execution_count": 14, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 35 }, "id": "lmcW2enzJC2P", "executionInfo": { "status": "ok", "timestamp": 1716313642988, "user_tz": -60, "elapsed": 5, "user": { "displayName": "Massimo Frasson", "userId": "09747843240273532020" } }, "outputId": "bc54b7f5-23f9-4031-f557-f436b419312a", "gather": { "logged": 1716385731317 } } }, { "cell_type": "code", "source": [ "tokenizer.convert_ids_to_tokens(tokens['input_ids'])" ], "outputs": [ { "output_type": "execute_result", "execution_count": 15, "data": { "text/plain": "['[CLS]',\n '\"',\n 'He',\n 'ended',\n 'the',\n 'World',\n 'Cup',\n 'on',\n 'the',\n 'wrong',\n 'note',\n ',',\n '\"',\n 'Co',\n '##ste',\n 'said',\n '.',\n '[SEP]']" }, "metadata": {} } ], "execution_count": 15, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "cu4VMY8KJOpD", "executionInfo": { "status": "ok", "timestamp": 1716313642988, "user_tz": -60, "elapsed": 5, "user": { "displayName": "Massimo Frasson", "userId": "09747843240273532020" } }, "outputId": "c98ce68c-3c8e-4682-b508-68247c56aa88", "gather": { "logged": 1716385742760 } } }, { "cell_type": "code", "source": [ "model = AutoModelForTokenClassification.from_pretrained(model_name)\n", "model.to(device)" ], "outputs": [ { "output_type": "stream", "name": "stderr", "text": "Some weights of the model checkpoint at dslim/bert-large-NER were not used when initializing BertForTokenClassification: ['bert.pooler.dense.bias', 'bert.pooler.dense.weight']\n- This IS expected if you are initializing BertForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n- This IS NOT expected if you are initializing BertForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n" }, { "output_type": "execute_result", "execution_count": 16, "data": { "text/plain": "BertForTokenClassification(\n (bert): BertModel(\n (embeddings): BertEmbeddings(\n (word_embeddings): Embedding(28996, 1024, padding_idx=0)\n (position_embeddings): Embedding(512, 1024)\n (token_type_embeddings): Embedding(2, 1024)\n (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n (encoder): BertEncoder(\n (layer): ModuleList(\n (0): BertLayer(\n (attention): BertAttention(\n (self): BertSelfAttention(\n (query): Linear(in_features=1024, out_features=1024, bias=True)\n (key): Linear(in_features=1024, out_features=1024, bias=True)\n (value): Linear(in_features=1024, out_features=1024, bias=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n (output): BertSelfOutput(\n (dense): Linear(in_features=1024, out_features=1024, bias=True)\n (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n )\n (intermediate): BertIntermediate(\n (dense): Linear(in_features=1024, out_features=4096, bias=True)\n (intermediate_act_fn): GELUActivation()\n )\n (output): BertOutput(\n (dense): Linear(in_features=4096, out_features=1024, bias=True)\n (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n )\n (1): BertLayer(\n (attention): BertAttention(\n (self): BertSelfAttention(\n (query): Linear(in_features=1024, out_features=1024, bias=True)\n (key): Linear(in_features=1024, out_features=1024, bias=True)\n (value): Linear(in_features=1024, out_features=1024, bias=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n (output): BertSelfOutput(\n (dense): Linear(in_features=1024, out_features=1024, bias=True)\n (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n )\n (intermediate): BertIntermediate(\n (dense): Linear(in_features=1024, out_features=4096, bias=True)\n (intermediate_act_fn): GELUActivation()\n )\n (output): BertOutput(\n (dense): Linear(in_features=4096, out_features=1024, bias=True)\n (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n )\n (2): BertLayer(\n (attention): BertAttention(\n (self): BertSelfAttention(\n (query): Linear(in_features=1024, out_features=1024, bias=True)\n (key): Linear(in_features=1024, out_features=1024, bias=True)\n (value): Linear(in_features=1024, out_features=1024, bias=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n (output): BertSelfOutput(\n (dense): Linear(in_features=1024, out_features=1024, bias=True)\n (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n )\n (intermediate): BertIntermediate(\n (dense): Linear(in_features=1024, out_features=4096, bias=True)\n (intermediate_act_fn): GELUActivation()\n )\n (output): BertOutput(\n (dense): Linear(in_features=4096, out_features=1024, bias=True)\n (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n )\n (3): BertLayer(\n (attention): BertAttention(\n (self): BertSelfAttention(\n (query): Linear(in_features=1024, out_features=1024, bias=True)\n (key): Linear(in_features=1024, out_features=1024, bias=True)\n (value): Linear(in_features=1024, out_features=1024, bias=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n (output): BertSelfOutput(\n (dense): Linear(in_features=1024, out_features=1024, bias=True)\n (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n )\n (intermediate): BertIntermediate(\n (dense): Linear(in_features=1024, out_features=4096, bias=True)\n (intermediate_act_fn): GELUActivation()\n )\n (output): BertOutput(\n (dense): Linear(in_features=4096, out_features=1024, bias=True)\n (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n )\n (4): BertLayer(\n (attention): BertAttention(\n (self): BertSelfAttention(\n (query): Linear(in_features=1024, out_features=1024, bias=True)\n (key): Linear(in_features=1024, out_features=1024, bias=True)\n (value): Linear(in_features=1024, out_features=1024, bias=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n (output): BertSelfOutput(\n (dense): Linear(in_features=1024, out_features=1024, bias=True)\n (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n )\n (intermediate): BertIntermediate(\n (dense): Linear(in_features=1024, out_features=4096, bias=True)\n (intermediate_act_fn): GELUActivation()\n )\n (output): BertOutput(\n (dense): Linear(in_features=4096, out_features=1024, bias=True)\n (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n )\n (5): BertLayer(\n (attention): BertAttention(\n (self): BertSelfAttention(\n (query): Linear(in_features=1024, out_features=1024, bias=True)\n (key): Linear(in_features=1024, out_features=1024, bias=True)\n (value): Linear(in_features=1024, out_features=1024, bias=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n (output): BertSelfOutput(\n (dense): Linear(in_features=1024, out_features=1024, bias=True)\n (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n )\n (intermediate): BertIntermediate(\n (dense): Linear(in_features=1024, out_features=4096, bias=True)\n (intermediate_act_fn): GELUActivation()\n )\n (output): BertOutput(\n (dense): Linear(in_features=4096, out_features=1024, bias=True)\n (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n )\n (6): BertLayer(\n (attention): BertAttention(\n (self): BertSelfAttention(\n (query): Linear(in_features=1024, out_features=1024, bias=True)\n (key): Linear(in_features=1024, out_features=1024, bias=True)\n (value): Linear(in_features=1024, out_features=1024, bias=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n (output): BertSelfOutput(\n (dense): Linear(in_features=1024, out_features=1024, bias=True)\n (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n )\n (intermediate): BertIntermediate(\n (dense): Linear(in_features=1024, out_features=4096, bias=True)\n (intermediate_act_fn): GELUActivation()\n )\n (output): BertOutput(\n (dense): Linear(in_features=4096, out_features=1024, bias=True)\n (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n )\n (7): BertLayer(\n (attention): BertAttention(\n (self): BertSelfAttention(\n (query): Linear(in_features=1024, out_features=1024, bias=True)\n (key): Linear(in_features=1024, out_features=1024, bias=True)\n (value): Linear(in_features=1024, out_features=1024, bias=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n (output): BertSelfOutput(\n (dense): Linear(in_features=1024, out_features=1024, bias=True)\n (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n )\n (intermediate): BertIntermediate(\n (dense): Linear(in_features=1024, out_features=4096, bias=True)\n (intermediate_act_fn): GELUActivation()\n )\n (output): BertOutput(\n (dense): Linear(in_features=4096, out_features=1024, bias=True)\n (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n )\n (8): BertLayer(\n (attention): BertAttention(\n (self): BertSelfAttention(\n (query): Linear(in_features=1024, out_features=1024, bias=True)\n (key): Linear(in_features=1024, out_features=1024, bias=True)\n (value): Linear(in_features=1024, out_features=1024, bias=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n (output): BertSelfOutput(\n (dense): Linear(in_features=1024, out_features=1024, bias=True)\n (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n )\n (intermediate): BertIntermediate(\n (dense): Linear(in_features=1024, out_features=4096, bias=True)\n (intermediate_act_fn): GELUActivation()\n )\n (output): BertOutput(\n (dense): Linear(in_features=4096, out_features=1024, bias=True)\n (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n )\n (9): BertLayer(\n (attention): BertAttention(\n (self): BertSelfAttention(\n (query): Linear(in_features=1024, out_features=1024, bias=True)\n (key): Linear(in_features=1024, out_features=1024, bias=True)\n (value): Linear(in_features=1024, out_features=1024, bias=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n (output): BertSelfOutput(\n (dense): Linear(in_features=1024, out_features=1024, bias=True)\n (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n )\n (intermediate): BertIntermediate(\n (dense): Linear(in_features=1024, out_features=4096, bias=True)\n (intermediate_act_fn): GELUActivation()\n )\n (output): BertOutput(\n (dense): Linear(in_features=4096, out_features=1024, bias=True)\n (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n )\n (10): BertLayer(\n (attention): BertAttention(\n (self): BertSelfAttention(\n (query): Linear(in_features=1024, out_features=1024, bias=True)\n (key): Linear(in_features=1024, out_features=1024, bias=True)\n (value): Linear(in_features=1024, out_features=1024, bias=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n (output): BertSelfOutput(\n (dense): Linear(in_features=1024, out_features=1024, bias=True)\n (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n )\n (intermediate): BertIntermediate(\n (dense): Linear(in_features=1024, out_features=4096, bias=True)\n (intermediate_act_fn): GELUActivation()\n )\n (output): BertOutput(\n (dense): Linear(in_features=4096, out_features=1024, bias=True)\n (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n )\n (11): BertLayer(\n (attention): BertAttention(\n (self): BertSelfAttention(\n (query): Linear(in_features=1024, out_features=1024, bias=True)\n (key): Linear(in_features=1024, out_features=1024, bias=True)\n (value): Linear(in_features=1024, out_features=1024, bias=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n (output): BertSelfOutput(\n (dense): Linear(in_features=1024, out_features=1024, bias=True)\n (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n )\n (intermediate): BertIntermediate(\n (dense): Linear(in_features=1024, out_features=4096, bias=True)\n (intermediate_act_fn): GELUActivation()\n )\n (output): BertOutput(\n (dense): Linear(in_features=4096, out_features=1024, bias=True)\n (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n )\n (12): BertLayer(\n (attention): BertAttention(\n (self): BertSelfAttention(\n (query): Linear(in_features=1024, out_features=1024, bias=True)\n (key): Linear(in_features=1024, out_features=1024, bias=True)\n (value): Linear(in_features=1024, out_features=1024, bias=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n (output): BertSelfOutput(\n (dense): Linear(in_features=1024, out_features=1024, bias=True)\n (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n )\n (intermediate): BertIntermediate(\n (dense): Linear(in_features=1024, out_features=4096, bias=True)\n (intermediate_act_fn): GELUActivation()\n )\n (output): BertOutput(\n (dense): Linear(in_features=4096, out_features=1024, bias=True)\n (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n )\n (13): BertLayer(\n (attention): BertAttention(\n (self): BertSelfAttention(\n (query): Linear(in_features=1024, out_features=1024, bias=True)\n (key): Linear(in_features=1024, out_features=1024, bias=True)\n (value): Linear(in_features=1024, out_features=1024, bias=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n (output): BertSelfOutput(\n (dense): Linear(in_features=1024, out_features=1024, bias=True)\n (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n )\n (intermediate): BertIntermediate(\n (dense): Linear(in_features=1024, out_features=4096, bias=True)\n (intermediate_act_fn): GELUActivation()\n )\n (output): BertOutput(\n (dense): Linear(in_features=4096, out_features=1024, bias=True)\n (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n )\n (14): BertLayer(\n (attention): BertAttention(\n (self): BertSelfAttention(\n (query): Linear(in_features=1024, out_features=1024, bias=True)\n (key): Linear(in_features=1024, out_features=1024, bias=True)\n (value): Linear(in_features=1024, out_features=1024, bias=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n (output): BertSelfOutput(\n (dense): Linear(in_features=1024, out_features=1024, bias=True)\n (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n )\n (intermediate): BertIntermediate(\n (dense): Linear(in_features=1024, out_features=4096, bias=True)\n (intermediate_act_fn): GELUActivation()\n )\n (output): BertOutput(\n (dense): Linear(in_features=4096, out_features=1024, bias=True)\n (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n )\n (15): BertLayer(\n (attention): BertAttention(\n (self): BertSelfAttention(\n (query): Linear(in_features=1024, out_features=1024, bias=True)\n (key): Linear(in_features=1024, out_features=1024, bias=True)\n (value): Linear(in_features=1024, out_features=1024, bias=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n (output): BertSelfOutput(\n (dense): Linear(in_features=1024, out_features=1024, bias=True)\n (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n )\n (intermediate): BertIntermediate(\n (dense): Linear(in_features=1024, out_features=4096, bias=True)\n (intermediate_act_fn): GELUActivation()\n )\n (output): BertOutput(\n (dense): Linear(in_features=4096, out_features=1024, bias=True)\n (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n )\n (16): BertLayer(\n (attention): BertAttention(\n (self): BertSelfAttention(\n (query): Linear(in_features=1024, out_features=1024, bias=True)\n (key): Linear(in_features=1024, out_features=1024, bias=True)\n (value): Linear(in_features=1024, out_features=1024, bias=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n (output): BertSelfOutput(\n (dense): Linear(in_features=1024, out_features=1024, bias=True)\n (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n )\n (intermediate): BertIntermediate(\n (dense): Linear(in_features=1024, out_features=4096, bias=True)\n (intermediate_act_fn): GELUActivation()\n )\n (output): BertOutput(\n (dense): Linear(in_features=4096, out_features=1024, bias=True)\n (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n )\n (17): BertLayer(\n (attention): BertAttention(\n (self): BertSelfAttention(\n (query): Linear(in_features=1024, out_features=1024, bias=True)\n (key): Linear(in_features=1024, out_features=1024, bias=True)\n (value): Linear(in_features=1024, out_features=1024, bias=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n (output): BertSelfOutput(\n (dense): Linear(in_features=1024, out_features=1024, bias=True)\n (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n )\n (intermediate): BertIntermediate(\n (dense): Linear(in_features=1024, out_features=4096, bias=True)\n (intermediate_act_fn): GELUActivation()\n )\n (output): BertOutput(\n (dense): Linear(in_features=4096, out_features=1024, bias=True)\n (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n )\n (18): BertLayer(\n (attention): BertAttention(\n (self): BertSelfAttention(\n (query): Linear(in_features=1024, out_features=1024, bias=True)\n (key): Linear(in_features=1024, out_features=1024, bias=True)\n (value): Linear(in_features=1024, out_features=1024, bias=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n (output): BertSelfOutput(\n (dense): Linear(in_features=1024, out_features=1024, bias=True)\n (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n )\n (intermediate): BertIntermediate(\n (dense): Linear(in_features=1024, out_features=4096, bias=True)\n (intermediate_act_fn): GELUActivation()\n )\n (output): BertOutput(\n (dense): Linear(in_features=4096, out_features=1024, bias=True)\n (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n )\n (19): BertLayer(\n (attention): BertAttention(\n (self): BertSelfAttention(\n (query): Linear(in_features=1024, out_features=1024, bias=True)\n (key): Linear(in_features=1024, out_features=1024, bias=True)\n (value): Linear(in_features=1024, out_features=1024, bias=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n (output): BertSelfOutput(\n (dense): Linear(in_features=1024, out_features=1024, bias=True)\n (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n )\n (intermediate): BertIntermediate(\n (dense): Linear(in_features=1024, out_features=4096, bias=True)\n (intermediate_act_fn): GELUActivation()\n )\n (output): BertOutput(\n (dense): Linear(in_features=4096, out_features=1024, bias=True)\n (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n )\n (20): BertLayer(\n (attention): BertAttention(\n (self): BertSelfAttention(\n (query): Linear(in_features=1024, out_features=1024, bias=True)\n (key): Linear(in_features=1024, out_features=1024, bias=True)\n (value): Linear(in_features=1024, out_features=1024, bias=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n (output): BertSelfOutput(\n (dense): Linear(in_features=1024, out_features=1024, bias=True)\n (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n )\n (intermediate): BertIntermediate(\n (dense): Linear(in_features=1024, out_features=4096, bias=True)\n (intermediate_act_fn): GELUActivation()\n )\n (output): BertOutput(\n (dense): Linear(in_features=4096, out_features=1024, bias=True)\n (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n )\n (21): BertLayer(\n (attention): BertAttention(\n (self): BertSelfAttention(\n (query): Linear(in_features=1024, out_features=1024, bias=True)\n (key): Linear(in_features=1024, out_features=1024, bias=True)\n (value): Linear(in_features=1024, out_features=1024, bias=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n (output): BertSelfOutput(\n (dense): Linear(in_features=1024, out_features=1024, bias=True)\n (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n )\n (intermediate): BertIntermediate(\n (dense): Linear(in_features=1024, out_features=4096, bias=True)\n (intermediate_act_fn): GELUActivation()\n )\n (output): BertOutput(\n (dense): Linear(in_features=4096, out_features=1024, bias=True)\n (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n )\n (22): BertLayer(\n (attention): BertAttention(\n (self): BertSelfAttention(\n (query): Linear(in_features=1024, out_features=1024, bias=True)\n (key): Linear(in_features=1024, out_features=1024, bias=True)\n (value): Linear(in_features=1024, out_features=1024, bias=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n (output): BertSelfOutput(\n (dense): Linear(in_features=1024, out_features=1024, bias=True)\n (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n )\n (intermediate): BertIntermediate(\n (dense): Linear(in_features=1024, out_features=4096, bias=True)\n (intermediate_act_fn): GELUActivation()\n )\n (output): BertOutput(\n (dense): Linear(in_features=4096, out_features=1024, bias=True)\n (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n )\n (23): BertLayer(\n (attention): BertAttention(\n (self): BertSelfAttention(\n (query): Linear(in_features=1024, out_features=1024, bias=True)\n (key): Linear(in_features=1024, out_features=1024, bias=True)\n (value): Linear(in_features=1024, out_features=1024, bias=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n (output): BertSelfOutput(\n (dense): Linear(in_features=1024, out_features=1024, bias=True)\n (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n )\n (intermediate): BertIntermediate(\n (dense): Linear(in_features=1024, out_features=4096, bias=True)\n (intermediate_act_fn): GELUActivation()\n )\n (output): BertOutput(\n (dense): Linear(in_features=4096, out_features=1024, bias=True)\n (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n (dropout): Dropout(p=0.1, inplace=False)\n )\n )\n )\n )\n )\n (dropout): Dropout(p=0.1, inplace=False)\n (classifier): Linear(in_features=1024, out_features=9, bias=True)\n)" }, "metadata": {} } ], "execution_count": 16, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "Qo911dGCJgXP", "executionInfo": { "status": "ok", "timestamp": 1716313645505, "user_tz": -60, "elapsed": 2520, "user": { "displayName": "Massimo Frasson", "userId": "09747843240273532020" } }, "outputId": "4c24f552-9fa4-4a5d-f45e-7ca75a773b82", "gather": { "logged": 1716385749395 } } }, { "cell_type": "code", "source": [ "config = AutoConfig.from_pretrained(model_name)\n", "config" ], "outputs": [ { "output_type": "stream", "name": "stderr", "text": "/anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n warnings.warn(\n" }, { "output_type": "execute_result", "execution_count": 17, "data": { "text/plain": "BertConfig {\n \"_name_or_path\": \"dslim/bert-large-NER\",\n \"architectures\": [\n \"BertForTokenClassification\"\n ],\n \"attention_probs_dropout_prob\": 0.1,\n \"classifier_dropout\": null,\n \"directionality\": \"bidi\",\n \"eos_token_ids\": null,\n \"hidden_act\": \"gelu\",\n \"hidden_dropout_prob\": 0.1,\n \"hidden_size\": 1024,\n \"id2label\": {\n \"0\": \"O\",\n \"1\": \"B-MISC\",\n \"2\": \"I-MISC\",\n \"3\": \"B-PER\",\n \"4\": \"I-PER\",\n \"5\": \"B-ORG\",\n \"6\": \"I-ORG\",\n \"7\": \"B-LOC\",\n \"8\": \"I-LOC\"\n },\n \"initializer_range\": 0.02,\n \"intermediate_size\": 4096,\n \"label2id\": {\n \"B-LOC\": 7,\n \"B-MISC\": 1,\n \"B-ORG\": 5,\n \"B-PER\": 3,\n \"I-LOC\": 8,\n \"I-MISC\": 2,\n \"I-ORG\": 6,\n \"I-PER\": 4,\n \"O\": 0\n },\n \"layer_norm_eps\": 1e-12,\n \"max_position_embeddings\": 512,\n \"model_type\": \"bert\",\n \"num_attention_heads\": 16,\n \"num_hidden_layers\": 24,\n \"output_past\": true,\n \"pad_token_id\": 0,\n \"pooler_fc_size\": 768,\n \"pooler_num_attention_heads\": 12,\n \"pooler_num_fc_layers\": 3,\n \"pooler_size_per_head\": 128,\n \"pooler_type\": \"first_token_transform\",\n \"position_embedding_type\": \"absolute\",\n \"transformers_version\": \"4.41.0\",\n \"type_vocab_size\": 2,\n \"use_cache\": true,\n \"vocab_size\": 28996\n}" }, "metadata": {} } ], "execution_count": 17, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "1BBlkIVdJ72u", "executionInfo": { "status": "ok", "timestamp": 1716313645957, "user_tz": -60, "elapsed": 454, "user": { "displayName": "Massimo Frasson", "userId": "09747843240273532020" } }, "outputId": "40afdbad-eff8-4d3e-9504-46e763db88c8", "gather": { "logged": 1716385853851 } } }, { "cell_type": "code", "source": [ "tokens = tokenizer(row['tokens'], is_split_into_words=True, return_tensors='pt', return_special_tokens_mask=True)\n", "tokens" ], "outputs": [ { "output_type": "execute_result", "execution_count": 18, "data": { "text/plain": "{'input_ids': tensor([[ 101, 107, 1124, 2207, 1103, 1291, 1635, 1113, 1103, 2488,\n 3805, 117, 107, 3291, 13894, 1163, 119, 102]]), 'token_type_ids': tensor([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]), 'special_tokens_mask': tensor([[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1]])}" }, "metadata": {} } ], "execution_count": 18, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "8w_TCHfzKYOr", "executionInfo": { "status": "ok", "timestamp": 1716313645957, "user_tz": -60, "elapsed": 5, "user": { "displayName": "Massimo Frasson", "userId": "09747843240273532020" } }, "outputId": "a87fb730-42e3-4780-cf53-570593448633", "gather": { "logged": 1716385915392 } } }, { "cell_type": "code", "source": [ "input_ids, attention_mask = tokens.input_ids.to(device), tokens.attention_mask.to(device)\n", "input_ids.shape" ], "outputs": [ { "output_type": "execute_result", "execution_count": 20, "data": { "text/plain": "torch.Size([1, 18])" }, "metadata": {} } ], "execution_count": 20, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "oCCzwFQNKzbm", "executionInfo": { "status": "ok", "timestamp": 1716313645958, "user_tz": -60, "elapsed": 3, "user": { "displayName": "Massimo Frasson", "userId": "09747843240273532020" } }, "outputId": "a9d8eb07-6cf8-4e24-95b7-61d3c2eddd30", "gather": { "logged": 1716385945872 } } }, { "cell_type": "code", "source": [ "predictions = model(input_ids, attention_mask=attention_mask)\n", "predictions" ], "outputs": [ { "output_type": "execute_result", "execution_count": 21, "data": { "text/plain": "TokenClassifierOutput(loss=None, logits=tensor([[[ 7.9187, -0.4113, -1.5611, -0.6204, -0.2018, -1.0338, -0.9577,\n -1.6814, -0.1193],\n [11.5741, -1.8944, -1.9833, -0.5482, -2.2003, -0.9254, -0.7738,\n -1.0076, -1.1121],\n [11.0746, -1.3051, -2.2110, 0.0273, -2.0430, -0.8386, -0.8197,\n -0.9676, -1.6945],\n [11.8623, -0.9233, -1.8007, -0.6327, -1.9620, -0.9431, -1.2096,\n -1.1030, -1.7021],\n [11.3466, 0.2537, -1.6077, -0.8082, -2.1432, -0.9979, -1.2625,\n -1.0296, -1.6366],\n [-0.7741, 7.4085, 1.3665, -1.2613, -1.6619, -0.7622, -1.9616,\n -0.2719, -2.0894],\n [-0.7371, 0.9987, 7.7832, -2.1687, -0.4202, -1.8552, -0.9724,\n -2.0760, 0.0263],\n [11.9569, -1.5610, -1.8399, -0.7455, -1.8140, -1.0242, -0.9104,\n -1.4859, -1.3755],\n [11.9736, -1.3492, -2.0106, -0.5690, -1.7965, -1.1176, -0.9273,\n -1.3721, -1.4524],\n [11.6905, -1.2874, -1.9174, -0.5654, -2.0186, -0.9152, -1.1881,\n -1.3111, -1.6265],\n [11.5362, -1.2028, -1.7666, -0.7362, -1.8882, -1.1173, -0.9502,\n -1.2905, -1.1622],\n [11.8898, -1.7819, -1.7713, -0.8864, -1.8619, -1.2269, -0.4725,\n -1.3367, -1.0853],\n [11.7229, -1.6347, -2.0659, -0.4383, -1.8983, -1.1207, -0.6516,\n -1.2266, -1.0870],\n [ 0.4603, -0.2236, -2.9311, 8.6894, -0.1335, 0.0543, -2.2655,\n -0.3906, -2.3858],\n [ 0.8514, -1.9610, -1.1588, 1.3823, 4.3713, -1.0212, -0.1433,\n -0.5193, -0.9105],\n [11.5773, -2.0366, -1.8260, -0.7636, -1.7054, -1.2445, -0.4836,\n -1.4063, -0.8646],\n [11.5503, -1.7444, -1.6724, -0.7700, -1.8704, -1.2549, -0.7990,\n -1.2080, -0.9925],\n [ 7.9187, -0.4113, -1.5611, -0.6204, -0.2018, -1.0338, -0.9577,\n -1.6814, -0.1193]]], grad_fn=), hidden_states=None, attentions=None)" }, "metadata": {} } ], "execution_count": 21, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "EnTF_djiLaHf", "executionInfo": { "status": "ok", "timestamp": 1716313646306, "user_tz": -60, "elapsed": 350, "user": { "displayName": "Massimo Frasson", "userId": "09747843240273532020" } }, "outputId": "026ac7ba-887c-4f4b-df2b-a34b2ed16750", "gather": { "logged": 1716385948208 } } }, { "cell_type": "code", "source": [ "predictions.logits.shape" ], "outputs": [ { "output_type": "execute_result", "execution_count": 22, "data": { "text/plain": "torch.Size([1, 18, 9])" }, "metadata": {} } ], "execution_count": 22, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "8en0mRncLxPD", "executionInfo": { "status": "ok", "timestamp": 1716313646306, "user_tz": -60, "elapsed": 7, "user": { "displayName": "Massimo Frasson", "userId": "09747843240273532020" } }, "outputId": "75fa5161-9d30-40f0-eae9-55062389717e", "gather": { "logged": 1716385964513 } } }, { "cell_type": "code", "source": [ "y_pred = torch.argmax(predictions.logits, dim=-1)\n", "y_pred.shape" ], "outputs": [ { "output_type": "execute_result", "execution_count": 23, "data": { "text/plain": "torch.Size([1, 18])" }, "metadata": {} } ], "execution_count": 23, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "O0KxrTZDN5WB", "executionInfo": { "status": "ok", "timestamp": 1716313646306, "user_tz": -60, "elapsed": 5, "user": { "displayName": "Massimo Frasson", "userId": "09747843240273532020" } }, "outputId": "e073c4b6-28a6-4a22-9828-a1eb451e297d", "gather": { "logged": 1716386010622 } } }, { "cell_type": "code", "source": [ "pd.DataFrame([\n", " tokenizer.convert_ids_to_tokens(input_ids[0]),\n", " [config.id2label[id.item()] for id in y_pred[0]]\n", "], index=['tokens', 'predictions'])" ], "outputs": [ { "output_type": "execute_result", "execution_count": 24, "data": { "text/plain": " 0 1 2 3 4 5 6 7 8 9 10 \\\ntokens [CLS] \" He ended the World Cup on the wrong note \npredictions O O O O O B-MISC I-MISC O O O O \n\n 11 12 13 14 15 16 17 \ntokens , \" Co ##ste said . [SEP] \npredictions O O B-PER I-PER O O O ", "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
01234567891011121314151617
tokens[CLS]\"HeendedtheWorldCuponthewrongnote,\"Co##stesaid.[SEP]
predictionsOOOOOB-MISCI-MISCOOOOOOB-PERI-PEROOO
\n
" }, "metadata": {} } ], "execution_count": 24, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 112 }, "id": "k_kBGrpYOAWE", "executionInfo": { "status": "ok", "timestamp": 1716313646306, "user_tz": -60, "elapsed": 3, "user": { "displayName": "Massimo Frasson", "userId": "09747843240273532020" } }, "outputId": "485fdd4b-f04c-4223-8fa7-ca5733f8561b", "gather": { "logged": 1716386015610 } } }, { "cell_type": "markdown", "source": [ "Let's compare this with the grount truth" ], "metadata": { "id": "ed0KP--4oEbK" } }, { "cell_type": "code", "source": [ "ground_truth = [id_to_tag[id] for id in row['ner_tags']]\n", "preds = [config.id2label[id.item()] for id in y_pred[0]]\n", "\n", "pd.DataFrame([\n", " tokenizer.convert_ids_to_tokens(input_ids[0]),\n", " preds,\n", " row['tokens'],\n", " ground_truth\n", "], index=['tokens', 'predictions', 'input_tokens', 'labels'])" ], "outputs": [ { "output_type": "execute_result", "execution_count": 25, "data": { "text/plain": " 0 1 2 3 4 5 6 7 8 \\\ntokens [CLS] \" He ended the World Cup on the \npredictions O O O O O B-MISC I-MISC O O \ninput_tokens \" He ended the World Cup on the wrong \nlabels O O O O B-MISC I-MISC O O O \n\n 9 10 11 12 13 14 15 16 17 \ntokens wrong note , \" Co ##ste said . [SEP] \npredictions O O O O B-PER I-PER O O O \ninput_tokens note , \" Coste said . None None None \nlabels O O O B-PER O O None None None ", "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
01234567891011121314151617
tokens[CLS]\"HeendedtheWorldCuponthewrongnote,\"Co##stesaid.[SEP]
predictionsOOOOOB-MISCI-MISCOOOOOOB-PERI-PEROOO
input_tokens\"HeendedtheWorldCuponthewrongnote,\"Costesaid.NoneNoneNone
labelsOOOOB-MISCI-MISCOOOOOOB-PEROONoneNoneNone
\n
" }, "metadata": {} } ], "execution_count": 25, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 175 }, "id": "4BDgoimvPjHi", "executionInfo": { "status": "ok", "timestamp": 1716313646580, "user_tz": -60, "elapsed": 277, "user": { "displayName": "Massimo Frasson", "userId": "09747843240273532020" } }, "outputId": "f315f660-c9f2-4001-8df5-80038bc1b132", "gather": { "logged": 1716386051269 } } }, { "cell_type": "code", "source": [ "new_truth = []\n", "i = 0\n", "for t, special_token in zip(tokenizer.convert_ids_to_tokens(input_ids[0]), tokens['special_tokens_mask'][0]):\n", " if special_token == 1:\n", " new_truth.append('O')\n", " elif '##' in t:\n", " new_truth.append(ground_truth[i-1].replace('B-', 'I-'))\n", " else:\n", " new_truth.append(ground_truth[i])\n", " i+=1" ], "outputs": [], "execution_count": 26, "metadata": { "id": "rT7RzGpLoUnY", "executionInfo": { "status": "ok", "timestamp": 1716313646580, "user_tz": -60, "elapsed": 3, "user": { "displayName": "Massimo Frasson", "userId": "09747843240273532020" } }, "gather": { "logged": 1716386101162 } } }, { "cell_type": "code", "source": [ "pd.DataFrame([\n", " tokenizer.convert_ids_to_tokens(input_ids[0]),\n", " preds,\n", " new_truth\n", "], index=['tokens', 'predictions', 'labels'])" ], "outputs": [ { "output_type": "execute_result", "execution_count": 27, "data": { "text/plain": " 0 1 2 3 4 5 6 7 8 9 10 \\\ntokens [CLS] \" He ended the World Cup on the wrong note \npredictions O O O O O B-MISC I-MISC O O O O \nlabels O O O O O B-MISC I-MISC O O O O \n\n 11 12 13 14 15 16 17 \ntokens , \" Co ##ste said . [SEP] \npredictions O O B-PER I-PER O O O \nlabels O O B-PER I-PER O O O ", "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
01234567891011121314151617
tokens[CLS]\"HeendedtheWorldCuponthewrongnote,\"Co##stesaid.[SEP]
predictionsOOOOOB-MISCI-MISCOOOOOOB-PERI-PEROOO
labelsOOOOOB-MISCI-MISCOOOOOOB-PERI-PEROOO
\n
" }, "metadata": {} } ], "execution_count": 27, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 143 }, "id": "R1p72lTYtYOj", "executionInfo": { "status": "ok", "timestamp": 1716313646580, "user_tz": -60, "elapsed": 2, "user": { "displayName": "Massimo Frasson", "userId": "09747843240273532020" } }, "outputId": "059735df-abb5-4601-b97b-f45be0ef80f5", "gather": { "logged": 1716386104079 } } }, { "cell_type": "markdown", "source": [ "## Let's formally evaluate the prediction" ], "metadata": { "id": "_gXLw6x9vouj" } }, { "cell_type": "code", "source": [ "%pip install seqeval" ], "outputs": [ { "output_type": "stream", "name": "stdout", "text": "Requirement already satisfied: seqeval in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (1.2.2)\nRequirement already satisfied: scikit-learn>=0.21.3 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from seqeval) (1.2.2)\nRequirement already satisfied: numpy>=1.14.0 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from seqeval) (1.23.5)\nRequirement already satisfied: threadpoolctl>=2.0.0 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from scikit-learn>=0.21.3->seqeval) (3.1.0)\nRequirement already satisfied: joblib>=1.1.1 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from scikit-learn>=0.21.3->seqeval) (1.2.0)\nRequirement already satisfied: scipy>=1.3.2 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from scikit-learn>=0.21.3->seqeval) (1.10.1)\nNote: you may need to restart the kernel to use updated packages.\n" } ], "execution_count": 28, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "Za5TLFxCt1AZ", "executionInfo": { "status": "ok", "timestamp": 1716313658082, "user_tz": -60, "elapsed": 11504, "user": { "displayName": "Massimo Frasson", "userId": "09747843240273532020" } }, "outputId": "09a53f3e-fb40-4224-a154-763efe3f6ddb" } }, { "cell_type": "code", "source": [ "import seqeval.metrics as met\n", "\n", "print(met.classification_report([new_truth], [preds]))" ], "outputs": [ { "output_type": "stream", "name": "stdout", "text": " precision recall f1-score support\n\n MISC 1.00 1.00 1.00 1\n PER 1.00 1.00 1.00 1\n\n micro avg 1.00 1.00 1.00 2\n macro avg 1.00 1.00 1.00 2\nweighted avg 1.00 1.00 1.00 2\n\n" } ], "execution_count": 29, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "VMFOMWllvtL3", "executionInfo": { "status": "ok", "timestamp": 1716313658083, "user_tz": -60, "elapsed": 8, "user": { "displayName": "Massimo Frasson", "userId": "09747843240273532020" } }, "outputId": "07f980b1-abe5-45cd-8574-e80f7fbeb9ce", "gather": { "logged": 1716386184062 } } }, { "cell_type": "markdown", "source": [ "## Let's put all toghether and evaluate on the whole dataset!" ], "metadata": { "id": "6H8-MXAyw1Qr" } }, { "cell_type": "code", "source": [ "# 1. First step: fix the label format for each record\n", "\n", "data = dataset['test']\n", "\n", "# TODO we could use tokenizer without map\n", "data = data.map(lambda x: tokenizer(x, is_split_into_words=True, padding='max_length', return_tensors='pt', return_special_tokens_mask=True), input_columns='tokens', batched=True)\n", "data" ], "outputs": [ { "output_type": "execute_result", "execution_count": 30, "data": { "text/plain": "Dataset({\n features: ['id', 'tokens', 'pos_tags', 'chunk_tags', 'ner_tags', 'input_ids', 'token_type_ids', 'attention_mask', 'special_tokens_mask'],\n num_rows: 3453\n})" }, "metadata": {} } ], "execution_count": 30, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "fuINXdJExcKL", "executionInfo": { "status": "ok", "timestamp": 1716313658457, "user_tz": -60, "elapsed": 378, "user": { "displayName": "Massimo Frasson", "userId": "09747843240273532020" } }, "outputId": "5342503e-5053-40b9-ff1f-5b616b5fb378", "gather": { "logged": 1716386352738 } } }, { "cell_type": "code", "source": [ "data.set_format('torch', columns=['input_ids', 'token_type_ids', 'attention_mask', 'special_tokens_mask'], output_all_columns=True, device=device)" ], "outputs": [], "execution_count": 31, "metadata": { "id": "LIkw-rVi0yyW", "executionInfo": { "status": "ok", "timestamp": 1716313658457, "user_tz": -60, "elapsed": 4, "user": { "displayName": "Massimo Frasson", "userId": "09747843240273532020" } }, "gather": { "logged": 1716386448857 } } }, { "cell_type": "code", "source": [ "# Far from perfect, but enough for today\n", "# Exercise: remove the try except and ensure the ground truth changes format correctly. The new list should be 512 labels long.\n", "def format_ground_truth(row):\n", " ground_truth = [id_to_tag[id.item()] for id in row['ner_tags']]\n", " new_truth = []\n", " i = 0\n", " for t, special_token in zip(tokenizer.convert_ids_to_tokens(row['input_ids']), row['special_tokens_mask']):\n", " if special_token.item() == 1:\n", " new_truth.append('O')\n", " elif '##' in t:\n", " new_truth.append(ground_truth[i-1].replace('B-', 'I-'))\n", " else:\n", " try:\n", " new_truth.append(ground_truth[i])\n", " i+=1\n", " except:\n", " return {'new_ner_tags': []}\n", "\n", " return {'new_ner_tags': new_truth}" ], "outputs": [], "execution_count": 32, "metadata": { "id": "SFK4kEMvzjVP", "executionInfo": { "status": "ok", "timestamp": 1716313658457, "user_tz": -60, "elapsed": 3, "user": { "displayName": "Massimo Frasson", "userId": "09747843240273532020" } }, "gather": { "logged": 1716386503834 } } }, { "cell_type": "code", "source": [ "data = data.map(format_ground_truth, batched=False)\n", "data" ], "outputs": [ { "output_type": "stream", "name": "stderr", "text": "Map: 100%|██████████| 3453/3453 [00:07<00:00, 435.57 examples/s]\n" }, { "output_type": "execute_result", "execution_count": 33, "data": { "text/plain": "Dataset({\n features: ['id', 'tokens', 'pos_tags', 'chunk_tags', 'ner_tags', 'input_ids', 'token_type_ids', 'attention_mask', 'special_tokens_mask', 'new_ner_tags'],\n num_rows: 3453\n})" }, "metadata": {} } ], "execution_count": 33, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 118, "referenced_widgets": [ "5ccdbff604ff4c409400e59eb822b88b", "4f97030ded284faf87eca885f13e1f50", "3db02263a7db4f1a98b03dc6eee0d952", "b73ba775d6da4725b23f436fb0bd59fc", "74c63a2eccac4a4ba4a827394d4949e2", "eded6e09f1b44837b5329ea8b9d6def7", "d4a22a9fe3404264b25a402bcf06ffc5", "8de7f7ae5a0f4244a3b5224a94b66c6f", "2c0eee863d0f417bb4d6ec897e63df5d", "9416fed3b8ff476398f1e2a7cb62b373", "faf15e47c1534cbd96df6b8eb8bede58" ] }, "id": "GVrKt5FS5qHc", "executionInfo": { "status": "ok", "timestamp": 1716313716649, "user_tz": -60, "elapsed": 58195, "user": { "displayName": "Massimo Frasson", "userId": "09747843240273532020" } }, "outputId": "72651fa6-d944-4412-b7e5-8362fe911eaa", "gather": { "logged": 1716386517770 } } }, { "cell_type": "code", "source": [ "data = data.filter(lambda x: len(x['new_ner_tags']) > 0)" ], "outputs": [ { "output_type": "stream", "name": "stderr", "text": "Filter: 100%|██████████| 3453/3453 [00:01<00:00, 1862.80 examples/s]\n" } ], "execution_count": 34, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 49, "referenced_widgets": [ "0aaca52c035a40229141f05d87e84fdc", "aa62030eaff8413ba033be17b0f7091e", "1647febe28bb4b15bfdabc12c593203d", "1d13a0774c0043518a2bc2ec9605f53a", "967031e6dc87423fbeb9e87588ddd90b", "80d720e56c5642f6b66c77085f87d5a7", "9e525d1aeea8419f80a7c8fb26972f8c", "7652b205fb414ddc8fb4bcf6242ee4e8", "a9091e3163cb4cbcbe2845a2e19550ef", "f82722f4671741d7b99985751971579f", "20fd7555c8e74cd19b064c8fa8819bfe" ] }, "id": "da_9gp8xarMK", "executionInfo": { "status": "ok", "timestamp": 1716313720859, "user_tz": -60, "elapsed": 4213, "user": { "displayName": "Massimo Frasson", "userId": "09747843240273532020" } }, "outputId": "e442ae10-105f-4934-e481-894d81f5b3ff", "gather": { "logged": 1716386525978 } } }, { "cell_type": "code", "source": [ "data" ], "outputs": [ { "output_type": "execute_result", "execution_count": 35, "data": { "text/plain": "Dataset({\n features: ['id', 'tokens', 'pos_tags', 'chunk_tags', 'ner_tags', 'input_ids', 'token_type_ids', 'attention_mask', 'special_tokens_mask', 'new_ner_tags'],\n num_rows: 1764\n})" }, "metadata": {} } ], "execution_count": 35, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "zS_eJdu9P_GD", "executionInfo": { "status": "ok", "timestamp": 1716313720860, "user_tz": -60, "elapsed": 7, "user": { "displayName": "Massimo Frasson", "userId": "09747843240273532020" } }, "outputId": "01833c64-c99f-4356-eb71-c2df9e4ebbd5", "gather": { "logged": 1716386532404 } } }, { "cell_type": "code", "source": [ "# Let's reduce the dataset size to have quicker computation. (Just for today's sake)\n", "data = data.select(range(20))" ], "outputs": [], "execution_count": 36, "metadata": { "id": "jV7utX0NRy6-", "gather": { "logged": 1716386546378 } } }, { "cell_type": "code", "source": [ "pd.set_option('display.max_columns', 100)\n", "test = data[10]\n", "\n", "display(pd.DataFrame([\n", " tokenizer.convert_ids_to_tokens(test['input_ids']),\n", " test['new_ner_tags']\n", "], index=['tokens', 'labels']))\n", "\n", "display(pd.DataFrame([\n", " test['tokens'],\n", " [id_to_tag[id] for id in test['ner_tags']]\n", "], index=['tokens', 'labels']))" ], "outputs": [ { "output_type": "display_data", "data": { "text/plain": " 0 1 2 3 4 5 6 7 8 9 \\\ntokens [CLS] Bit ##ar pulled off fine saves whenever they did \nlabels O B-PER I-PER O O O O O O O \n\n 10 11 12 13 14 15 16 17 18 19 \\\ntokens . [SEP] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] \nlabels O O O O O O O O O O \n\n 20 21 22 23 24 25 26 27 28 29 \\\ntokens [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] \nlabels O O O O O O O O O O \n\n 30 31 32 33 34 35 36 37 38 39 \\\ntokens [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] \nlabels O O O O O O O O O O \n\n 40 41 42 43 44 45 46 47 48 49 \\\ntokens [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] \nlabels O O O O O O O O O O \n\n ... 462 463 464 465 466 467 468 469 470 \\\ntokens ... [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] \nlabels ... O O O O O O O O O \n\n 471 472 473 474 475 476 477 478 479 480 \\\ntokens [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] \nlabels O O O O O O O O O O \n\n 481 482 483 484 485 486 487 488 489 490 \\\ntokens [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] \nlabels O O O O O O O O O O \n\n 491 492 493 494 495 496 497 498 499 500 \\\ntokens [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] \nlabels O O O O O O O O O O \n\n 501 502 503 504 505 506 507 508 509 510 \\\ntokens [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] \nlabels O O O O O O O O O O \n\n 511 \ntokens [PAD] \nlabels O \n\n[2 rows x 512 columns]", "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
012345678910111213141516171819202122232425262728293031323334353637383940414243444546474849...462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511
tokens[CLS]Bit##arpulledofffinesaveswhenevertheydid.[SEP][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD]...[PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD]
labelsOB-PERI-PEROOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOO...OOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOO
\n

2 rows × 512 columns

\n
" }, "metadata": {} }, { "output_type": "display_data", "data": { "text/plain": " 0 1 2 3 4 5 6 7 8\ntokens Bitar pulled off fine saves whenever they did .\nlabels B-PER O O O O O O O O", "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
012345678
tokensBitarpulledofffinesaveswhenevertheydid.
labelsB-PEROOOOOOOO
\n
" }, "metadata": {} } ], "execution_count": 37, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 291 }, "id": "Tt3eD9yP6AWy", "executionInfo": { "status": "ok", "timestamp": 1716313720860, "user_tz": -60, "elapsed": 5, "user": { "displayName": "Massimo Frasson", "userId": "09747843240273532020" } }, "outputId": "c7961d62-c6c6-4d89-87a7-a1aad74871d3", "gather": { "logged": 1716386582963 } } }, { "cell_type": "code", "source": [ "data.set_format('torch', columns=['input_ids', 'token_type_ids', 'attention_mask', 'special_tokens_mask'], output_all_columns=True, device=device)" ], "outputs": [], "execution_count": 38, "metadata": { "id": "wdiCtOrwGoRY", "executionInfo": { "status": "ok", "timestamp": 1716313720860, "user_tz": -60, "elapsed": 4, "user": { "displayName": "Massimo Frasson", "userId": "09747843240273532020" } }, "gather": { "logged": 1716386650834 } } }, { "cell_type": "code", "source": [ "# 2. Second step: predict the whole dataset\n", "def predict_batch(rows):\n", " outputs = model(rows['input_ids'], attention_mask=rows['attention_mask']).logits\n", " predictions = torch.argmax(outputs, dim=-1)\n", " return {'predictions': predictions}\n", "\n", "data = data.map(predict_batch, batched=True, batch_size=8)\n", "#data = data.map(predict_batch, batched=False)" ], "outputs": [ { "output_type": "stream", "name": "stderr", "text": "Map: 100%|██████████| 20/20 [00:54<00:00, 2.73s/ examples]\n" } ], "execution_count": 40, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 49, "referenced_widgets": [ "779b91e29e6b4a989f5942a353eee218", "914d9689292e46f79b21e1ff841e2754", "062308c782d34ad5a0392d6ce8309e1a", "e3f67cf561c34aaf90a4b999ccfa2008", "fb0b628bfb564133abd7162cda02f59b", "fb85a41ca0fe4966942d73232436b7b7", "d3e88177b8ce4acb8a2d4b2a96cd4f79", "e06010ffecfc458584ed6642462d3d8b", "78b0f976c4c94750b7f0965cc9945e60", "b4349b37d9fe4fbba704093fb4243a98", "d64dcc7fbf064bb48016d9810a53f7c9" ] }, "id": "RkjbArHwx1hd", "outputId": "dfde207d-f596-4954-d7fb-f3c71bfd71e6", "executionInfo": { "status": "ok", "timestamp": 1716313955013, "user_tz": -60, "elapsed": 178989, "user": { "displayName": "Massimo Frasson", "userId": "09747843240273532020" } }, "gather": { "logged": 1716386801702 } } }, { "cell_type": "code", "source": [ "data[10]['predictions']" ], "outputs": [ { "output_type": "execute_result", "execution_count": 42, "data": { "text/plain": "tensor([0, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 3, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0])" }, "metadata": {} } ], "execution_count": 42, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "yGDgHenJRkIG", "executionInfo": { "status": "ok", "timestamp": 1716314017808, "user_tz": -60, "elapsed": 249, "user": { "displayName": "Massimo Frasson", "userId": "09747843240273532020" } }, "outputId": "584f96c5-4bfb-42d5-868c-bd46655977ca", "gather": { "logged": 1716386815930 } } }, { "cell_type": "code", "source": [ "def iob2_row(row):\n", " return {'iob2_pred': [config.id2label[p.item()] for p in row]}\n", "\n", "pred = data.map(iob2_row, input_columns='predictions')" ], "outputs": [ { "output_type": "stream", "name": "stderr", "text": "\rMap: 0%| | 0/20 [00:00