biot5x/examples/finetunning_biot5x_blurb.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3950e320-16c4-48ab-bdc7-21d406635524",
   "metadata": {},
   "outputs": [],
   "source": [
    "!rm -r SciFive\n",
    "!git clone /~https://github.com/justinphan3110/SciFive.git\n",
    "!cp -r SciFive/biot5x ."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "270a8d42-7851-433c-9b77-ebf5c23648be",
   "metadata": {},
   "outputs": [],
   "source": [
    "## Install JAX for GPU\n",
    "!pip install jax==0.4.2 jaxlib==0.4.2+cuda11.cudnn86 -f https://storage.googleapis.com/jax-releases/jax_cuda_releases.html\n",
    "## Install T5X and dependencies\n",
    "!cd biot5x && python3 setup.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1fa22a3c-f750-486f-8e17-0106600a7c1d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# download BioT5X base model\n",
    "!mkdir biot5x_base\n",
    "!gsutil -m cp -r gs://scifive/biot5x/base/* biot5x_base/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6d0fd9b7-b40a-46e9-a078-930d84662f5b",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "############################### RE: chemprot ###############################\n",
    "\n",
    "model_size = 'base'\n",
    "task = 'chemprot'\n",
    "train_file = f'biot5x/data/{task}/train_blurb.tsv'\n",
    "test_file = f'biot5x/data/{task}/test_blurb.tsv'\n",
    "dev_file = f'biot5x/data/{task}/dev_blurb.tsv'\n",
    "\n",
    "model_dir = f'out/{task}/biot5x_{model_size}_{task}'\n",
    "pretrained_path=f'biot5x_{model_size}'\n",
    "gin_file = f'biot5x/configs/biot5x/finetune/base/{task}_blurb.gin'\n",
    "\n",
    "metric = 'PRF1'\n",
    "\n",
    "\n",
    "%run 'biot5x/src/finetune_biot5x.py' \\\n",
    "  --gin_file=\"{gin_file}\" \\\n",
    "  --gin.MODEL_DIR=\"'{model_dir}'\" \\\n",
    "  --gin.INITIAL_CHECKPOINT_PATH=\"'{pretrained_path}'\" \\\n",
    "  --task=\"{task}\" \\\n",
    "  --metric=\"{metric}\" \\\n",
    "  --train_file=\"{train_file}\" \\\n",
    "  --predict_file=\"{test_file}\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "252c6504-2dcd-4a77-b941-39b351aa9361",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "############################### RE: ddi ###############################\n",
    "\n",
    "model_size = 'base'\n",
    "task = 'ddi'\n",
    "train_file = f'biot5x/data/{task}/train_blurb.tsv'\n",
    "test_file = f'biot5x/data/{task}/test_blurb.tsv'\n",
    "dev_file = f'biot5x/data/{task}/dev_blurb.tsv'\n",
    "\n",
    "model_dir = f'out/{task}/biot5x_{model_size}_{task}'\n",
    "pretrained_path=f'biot5x_{model_size}'\n",
    "gin_file = f'biot5x/configs/biot5x/finetune/base/{task}_blurb.gin'\n",
    "metric = 'PRF1'\n",
    "\n",
    "%run 'biot5x/src/finetune_biot5x.py' \\\n",
    "  --gin_file=\"{gin_file}\" \\\n",
    "  --gin.MODEL_DIR=\"'{model_dir}'\" \\\n",
    "  --gin.INITIAL_CHECKPOINT_PATH=\"'{pretrained_path}'\" \\\n",
    "  --task=\"{task}\" \\\n",
    "  --metric=\"{metric}\" \\\n",
    "  --train_file=\"{train_file}\" \\\n",
    "  --predict_file=\"{test_file}\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8a6bd1f3-9c3c-497c-8e78-9f10d7c50bad",
   "metadata": {},
   "outputs": [],
   "source": [
    "############################### NER: NCBI_disease ###############################\n",
    "\n",
    "model_size = 'base'\n",
    "task = 'NCBI_disease'\n",
    "train_file = f'biot5x/data/{task}/train_blurb.tsv'\n",
    "test_file = f'biot5x/data/{task}/test_blurb.tsv'\n",
    "dev_file = f'biot5x/data/{task}/dev_blurb.tsv'\n",
    "\n",
    "model_dir = f'out/{task}/{model_size}_{task}'\n",
    "pretrained_path=f'biot5x_{model_size}'\n",
    "gin_file = f'biot5x/configs/biot5x/finetune/base/{task}_blurb.gin'\n",
    "\n",
    "metric = 'ner_metric'\n",
    "batch_size = 64\n",
    "\n",
    "\n",
    "%run 'biot5x/src/finetune_biot5x.py' \\\n",
    "  --gin_file=\"{gin_file}\" \\\n",
    "  --gin.MODEL_DIR=\"'{model_dir}'\" \\\n",
    "  --gin.INITIAL_CHECKPOINT_PATH=\"'{pretrained_path}'\" \\\n",
    "  --gin.BATCH_SIZE='{batch_size}' \\\n",
    "  --task=\"{task}\" \\\n",
    "  --metric=\"{metric}\" \\\n",
    "  --train_file=\"{train_file}\" \\\n",
    "  --predict_file=\"{test_file}\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "818a87a3-c675-4661-b6f0-90d54497c026",
   "metadata": {},
   "outputs": [],
   "source": [
    "############################### NER: BC5CDR_disease ###############################\n",
    "model_size = 'base'\n",
    "task = 'BC5CDR_disease'\n",
    "train_file = f'biot5x/data/{task}/train_blurb.tsv'\n",
    "test_file = f'biot5x/data/{task}/test_blurb.tsv'\n",
    "dev_file = f'biot5x/data/{task}/dev_blurb.tsv'\n",
    "\n",
    "model_dir = f'out/{task}/{model_size}_{task}'\n",
    "pretrained_path=f'biot5x_{model_size}'\n",
    "gin_file = f'biot5x/configs/biot5x/finetune/base/{task}_blurb.gin'\n",
    "\n",
    "metric = 'ner_metric'\n",
    "batch_size = 64\n",
    "\n",
    "\n",
    "%run 'biot5x/src/finetune_biot5x.py' \\\n",
    "  --gin_file=\"{gin_file}\" \\\n",
    "  --gin.MODEL_DIR=\"'{model_dir}'\" \\\n",
    "  --gin.INITIAL_CHECKPOINT_PATH=\"'{pretrained_path}'\" \\\n",
    "  --gin.BATCH_SIZE='{batch_size}' \\\n",
    "  --task=\"{task}\" \\\n",
    "  --metric=\"{metric}\" \\\n",
    "  --train_file=\"{train_file}\" \\\n",
    "  --predict_file=\"{test_file}\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "69dacedc-27bf-4b15-ab28-1e67ebb6c111",
   "metadata": {},
   "outputs": [],
   "source": [
    "############################### NER: BC5CDR_disease ###############################\n",
    "model_size = 'base'\n",
    "task = 'JNLPBA'\n",
    "train_file = f'biot5x/data/{task}/train_blurb.tsv'\n",
    "test_file = f'biot5x/data/{task}/test_blurb.tsv'\n",
    "dev_file = f'biot5x/data/{task}/dev_blurb.tsv'\n",
    "\n",
    "model_dir = f'out/{task}/{model_size}_{task}'\n",
    "pretrained_path=f'biot5x_{model_size}'\n",
    "gin_file = f'biot5x/configs/biot5x/finetune/base/{task}_blurb.gin'\n",
    "\n",
    "metric = 'ner_metric'\n",
    "batch_size = 64\n",
    "\n",
    "\n",
    "%run 'biot5x/src/finetune_biot5x.py' \\\n",
    "  --gin_file=\"{gin_file}\" \\\n",
    "  --gin.MODEL_DIR=\"'{model_dir}'\" \\\n",
    "  --gin.INITIAL_CHECKPOINT_PATH=\"'{pretrained_path}'\" \\\n",
    "  --gin.BATCH_SIZE='{batch_size}' \\\n",
    "  --task=\"{task}\" \\\n",
    "  --metric=\"{metric}\" \\\n",
    "  --train_file=\"{train_file}\" \\\n",
    "  --predict_file=\"{test_file}\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fecf32d1-8c9f-4b9c-a0f0-0b66d67b1413",
   "metadata": {},
   "outputs": [],
   "source": [
    "############################### NER: BC5CDR_chem ###############################\n",
    "model_size = 'base'\n",
    "task = 'BC5CDR_chem'\n",
    "train_file = f'biot5x/data/{task}/train_blurb.tsv'\n",
    "test_file = f'biot5x/data/{task}/test_blurb.tsv'\n",
    "dev_file = f'biot5x/data/{task}/dev_blurb.tsv'\n",
    "\n",
    "model_dir = f'out/{task}/{model_size}_{task}'\n",
    "pretrained_path=f'biot5x_{model_size}'\n",
    "gin_file = f'biot5x/configs/biot5x/finetune/base/{task}_blurb.gin'\n",
    "\n",
    "metric = 'ner_metric'\n",
    "batch_size = 64\n",
    "\n",
    "\n",
    "%run 'biot5x/src/finetune_biot5x.py' \\\n",
    "  --gin_file=\"{gin_file}\" \\\n",
    "  --gin.MODEL_DIR=\"'{model_dir}'\" \\\n",
    "  --gin.INITIAL_CHECKPOINT_PATH=\"'{pretrained_path}'\" \\\n",
    "  --gin.BATCH_SIZE='{batch_size}' \\\n",
    "  --task=\"{task}\" \\\n",
    "  --metric=\"{metric}\" \\\n",
    "  --train_file=\"{train_file}\" \\\n",
    "  --predict_file=\"{test_file}\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cbab2ec7-0d0d-48a0-8644-57d1b48351cd",
   "metadata": {},
   "outputs": [],
   "source": [
    "############################### NER: HoC ###############################\n",
    "model_size = 'base'\n",
    "task = 'HoC'\n",
    "train_file = f'biot5x/data/{task}/train_blurb.tsv'\n",
    "test_file = f'biot5x/data/{task}/test_blurb.tsv'\n",
    "dev_file = f'biot5x/data/{task}/dev_blurb.tsv'\n",
    "\n",
    "model_dir = f'out/{task}/{model_size}_{task}'\n",
    "pretrained_path=f'biot5x_{model_size}'\n",
    "gin_file = f'biot5x/configs/biot5x/finetune/base/{task}_blurb.gin'\n",
    "\n",
    "\n",
    "# micro F1 for HoC metric adapt from /~https://github.com/michiyasunaga/LinkBERT/blob/main/src/seqcls/utils_hoc.py\n",
    "metric = 'hoc'\n",
    "\n",
    "%run 'biot5x/src/finetune_biot5x.py' \\\n",
    "  --gin_file=\"{gin_file}\" \\\n",
    "  --gin.MODEL_DIR=\"'{model_dir}'\" \\\n",
    "  --gin.INITIAL_CHECKPOINT_PATH=\"'{pretrained_path}'\" \\\n",
    "  --task=\"{task}\" \\\n",
    "  --metric=\"{metric}\" \\\n",
    "  --train_file=\"{train_file}\" \\\n",
    "  --predict_file=\"{test_file}\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b1e15634-d5b9-46c4-af87-32a0db36af0f",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  },
  "vscode": {
   "interpreter": {
    "hash": "eedb55a3f3d5a08c90a45b02edd9d5201f64a9996f64fdac14a22b56503f46e8"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}