converter_deprecated-use_convert-starxml-script.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Star2BF - Star to Bibframe"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Import libraries:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "from rdflib import Graph, Literal\n",
    "from rdflib.namespace import RDF, RDFS, XSD, Namespace\n",
    "from rdflib import BNode\n",
    "from rdflib import URIRef\n",
    "import xml.etree.ElementTree as ET\n",
    "import re\n",
    "import html\n",
    "import modules.mappings as mappings\n",
    "# import modules.open_science as open_science\n",
    "import requests_cache\n",
    "from datetime import timedelta\n",
    "\n",
    "# old fuzzy compare for reconciliations: using fuzzywuzzy\n",
    "from fuzzywuzzy import fuzz\n",
    "from fuzzywuzzy import process\n",
    "\n",
    "# new fuzzy compare: using the faster rapidfuzz as a drop-in replacement for fuzzywuzzy:\n",
    "# from rapidfuzz import fuzz\n",
    "# from rapidfuzz import process\n",
    "\n",
    "import csv\n",
    "\n",
    "# ror lookup\n",
    "ROR_API_URL = \"https://api.ror.org/organizations?affiliation=\"  \n",
    "\n",
    "from modules.mappings import funder_names_replacelist\n",
    "\n",
    "# set up friendly session by adding mail in request:\n",
    "CROSSREF_FRIENDLY_MAIL = \"&mailto=ttr@leibniz-psychology.org\"\n",
    "# for getting a list of funders from api ():\n",
    "CROSSREF_API_URL = \"https://api.crossref.org/funders?query=\"\n",
    "\n",
    "urls_expire_after = {\n",
    "    # Custom cache duration per url, 0 means \"don't cache\"\n",
    "    # f'{SKOSMOS_URL}/rest/v1/label?uri=https%3A//w3id.org/zpid/vocabs/terms/09183&lang=de': 0,\n",
    "    # f'{SKOSMOS_URL}/rest/v1/label?uri=https%3A//w3id.org/zpid/vocabs/terms/': 0,\n",
    "}\n",
    "# using cache for ror requests\n",
    "session = requests_cache.CachedSession(\n",
    "    \".cache/requests\",\n",
    "    allowable_codes=[200, 404],\n",
    "    expire_after=timedelta(days=30),\n",
    "    urls_expire_after=urls_expire_after,\n",
    ")\n",
    "# and a cache for the crossref api:\n",
    "session_fundref = requests_cache.CachedSession(\n",
    "    \".cache/requests\",\n",
    "    allowable_codes=[200, 404],\n",
    "    expire_after=timedelta(days=30),\n",
    "    urls_expire_after=urls_expire_after,\n",
    ")\n",
    "\n",
    "# import csv of LUX authority institutes:\n",
    "with open('institute_lux.csv', newline='') as csvfile:\n",
    "    reader = csv.DictReader(csvfile)\n",
    "    # save it in a list:\n",
    "    lux_institutes = list(reader)\n",
    "    # split string \"known_names\" into a list of strings on \"##\":\n",
    "    for institute in lux_institutes:\n",
    "        institute[\"known_names\"] = institute[\"known_names\"].split(\" ## \")\n",
    "# print(\"Und die ganze Tabelle:\")\n",
    "# print(dachlux_institutes)\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create an \"element tree\" from the records in my xml file so we can loop through them and do things with them:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "# root = ET.parse(\"xml-data/records-440.xml\")\n",
    "# root = ET.parse(\"xml-data/records-322.xml\")\n",
    "# root = ET.parse(\"xml-data/records-395.xml\")\n",
    "# root = ET.parse(\"xml-data/records-214.xml\")\n",
    "root = ET.parse(\"/home/tina/Developement/psyndex-workflows/star-to-rdf/data/230424_000956/xml/records-556.xml\")\n",
    "\n",
    "# To see the source xml's structure, uncomment this function:\n",
    "# def print_element(element, depth=0):\n",
    "#     print(\"\\t\"*depth, element.tag, element.attrib, element.text)\n",
    "#     for child in element:\n",
    "#         print_element(child, depth+1)\n",
    "\n",
    "# for child in root.getroot()[:2]:\n",
    "#     print_element(child)\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We first set a few namespace objects for bibframe, schema.org and for our resources (the works and instances) \n",
    "themselves.\n",
    "\n",
    "Then, we create two graphs from the xml source file, one to generate triples for our bibframe profile output, and the other for the simplified schema.org profile. \n",
    "\n",
    "Finally, we bind the prefixes with their appropriate namespaces to the graphs."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "BF = Namespace(\"http://id.loc.gov/ontologies/bibframe/\")\n",
    "BFLC = Namespace(\"http://id.loc.gov/ontologies/bflc/\")\n",
    "MADS = Namespace(\"http://www.loc.gov/mads/rdf/v1#\")\n",
    "SCHEMA = Namespace(\"https://schema.org/\")\n",
    "WORKS = Namespace(\"https://w3id.org/zpid/resources/works/\")\n",
    "INSTANCES = Namespace(\"https://w3id.org/zpid/resources/instances/\")\n",
    "INSTANCEBUNDLES = Namespace(\"https://w3id.org/zpid/resources/instancebundles/\")\n",
    "PXC = Namespace(\"https://w3id.org/zpid/ontology/classes/\")\n",
    "PXP = Namespace(\"https://w3id.org/zpid/ontology/properties/\")\n",
    "LANG = Namespace (\"http://id.loc.gov/vocabulary/iso639-2/\")\n",
    "LOCID = Namespace(\"http://id.loc.gov/vocabulary/identifiers/\")\n",
    "CONTENTTYPES = Namespace(\"http://id.loc.gov/vocabulary/contentTypes/\")\n",
    "ROLES = Namespace(\"https://w3id.org/zpid/vocabs/roles/\")\n",
    "RELATIONS = Namespace(\"https://w3id.org/zpid/vocabs/relations/\")\n",
    "GENRES = Namespace(\"https://w3id.org/zpid/vocabs/genres/\")\n",
    "\n",
    "\n",
    "# graph for bibframe profile:\n",
    "records_bf = Graph()\n",
    "# make the graph named:\n",
    "records_bf = Graph(identifier=URIRef(\"https://w3id.org/zpid/bibframe/records/\"))\n",
    "\n",
    "kerndaten = Graph()\n",
    "kerndaten.parse(\"ttl-data/kerndaten.ttl\", format=\"turtle\")    \n",
    "\n",
    "# import graph for crossref funder registry dump:\n",
    "# crossref_funders = Graph()\n",
    "# crossref_funders.parse(\"crossref_fundref_registry.rdf\", format=\"xml\")\n",
    "# we need a new graph for the schema.org profile, so it won't just reuse the old triples from the other profile\n",
    "# records_schema = Graph()\n",
    "\n",
    "# Bind the namespaces to the prefixes we want to see in the output:\n",
    "records_bf.bind(\"bf\", BF) \n",
    "records_bf.bind(\"bflc\", BFLC) \n",
    "records_bf.bind(\"works\", WORKS)  \n",
    "# records_schema.bind(\"works\", WORKS) \n",
    "records_bf.bind(\"instances\", INSTANCES) \n",
    "records_bf.bind(\"pxc\", PXC) \n",
    "records_bf.bind(\"pxp\", PXP) \n",
    "records_bf.bind(\"lang\", LANG) \n",
    "records_bf.bind(\"schema\", SCHEMA) \n",
    "records_bf.bind(\"locid\", LOCID) \n",
    "records_bf.bind(\"mads\", MADS) \n",
    "records_bf.bind(\"roles\", ROLES) \n",
    "records_bf.bind(\"relations\", RELATIONS)\n",
    "records_bf.bind(\"genres\", GENRES)\n",
    "records_bf.bind(\"contenttypes\", CONTENTTYPES)\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Functions to do all the things\n",
    "\n",
    "We need functions for the different things we will do - to avoid one long monolith of a loop.\n",
    "\n",
    "This is where they will go. Examples: Create blank nodes for Idebtifiers, create nested contribution objects from disparate person entries in AUP, AUK, CS and COU fields, merge PAUP (psychauthor person names and ids) with the person's name in AUP...\n",
    "\n",
    "These functions will later be called at the bottom of this notebook, in a loop over all the xml records."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## TODO: Splitting Instances from single records with MT and MT2\n",
    "\n",
    "A record that has two media types (both a MT and a MT2 field) actually contains two instances.\n",
    "\n",
    "Let's start with books, first: Records with BE=SS or SM. Usually, when there are two media types, MT is \"Print\" and MT2 is \"Online Medium\" or vice versa.\n",
    "\n",
    "So we go through each record with BE=SS (or SM) and check for MT and MT2. If both are present, we create two instances, one for each media type. We will first describe this by giving them additional classes: bf:Electronic for Online Medium and bf:Print for Print.\n",
    "\n",
    "We will also add a property to the instance that links it to its other format, via bf:otherPsysicalFormat.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "# function to set mediaCarrier from mt and mt2:\n",
    "def get_mediacarrier(mediatype):\n",
    "    cases = [\n",
    "        (\"Print\", \"Print\"),\n",
    "        (\"Online Medium\", \"Electronic\"),\n",
    "        (\"eBook\", \"Electronic\"),\n",
    "    ]\n",
    "    for case in cases:\n",
    "        if case[0] == mediatype:\n",
    "            return URIRef(BF[case[1]])\n",
    "    return URIRef(BF[mediatype])\n",
    "\n",
    "def get_publication_info(instance_uri, record, mediatype):\n",
    "    # get the publication info:\n",
    "    pu = None\n",
    "    pu = record.find(\"PU\")\n",
    "    pufield = html.unescape(pu.text.strip())\n",
    "    if pu is not None and pufield != \"\":\n",
    "        # split out the content after |e:\n",
    "        pub_lisher = pufield.split(\"|v\")\n",
    "        pub_place = pufield.split(\"|o\")\n",
    "        p_isbn = pufield.split(\"|i\")\n",
    "        e_isbn = pufield.split(\"|e\")\n",
    "        # add a bf:provisionActivity to the instance:\n",
    "        publication_node = BNode()\n",
    "        records_bf.add((instance_uri, BF.provisionActivity, publication_node))\n",
    "        # add the bf:place to the bf:provisionActivity:\n",
    "        if len(pub_place) > 1:\n",
    "            records_bf.add((publication_node, BFLC.simplePlace, Literal(str(pub_place[1]).strip())))\n",
    "        # add the pub_lisher to the bf:provisionActivity as bflc:simpleAgent:\n",
    "        if len(pub_lisher) > 1:\n",
    "            records_bf.add((publication_node, BFLC.simpleAgent, Literal(str(pub_lisher[1]).strip())))\n",
    "        # add the p_isbn to the instance:\n",
    "        isbn_node = BNode()\n",
    "        records_bf.add((instance_uri, BF.identifiedBy, isbn_node))\n",
    "        records_bf.add((isbn_node, RDF.type, BF.Isbn))\n",
    "        if get_mediacarrier(mediatype) == BF.Electronic:\n",
    "            records_bf.add((instance_uri, BF.identifiedBy, isbn_node))\n",
    "            records_bf.add((isbn_node, RDF.type, BF.Isbn))\n",
    "            records_bf.add((isbn_node, RDF.value, Literal(str(e_isbn[1]).strip())))\n",
    "        else:\n",
    "            records_bf.add((isbn_node, RDF.value, Literal(str(p_isbn[1]).strip())))\n",
    "\n",
    "def split_books(instance_uri, record):\n",
    "    # check the BE field to see if it is \"SS\" or \"SM\":\n",
    "    be = None\n",
    "    be = record.find(\"BE\")\n",
    "    befield = be.text.strip()\n",
    "    if be is not None and befield == \"SS\" or befield == \"SM\":\n",
    "        mt=None\n",
    "        mt2 = None\n",
    "        mtfield = html.unescape(record.find(\"MT\").text.strip())\n",
    "        mt2field = html.unescape(record.find(\"MT2\").text.strip())\n",
    "        # we should check if there is an \"e isbn\" somewhere in PU subfield |e:\n",
    "        \n",
    "        if mt is not None and mtfield != \"\":\n",
    "            # note the content of the MT field and use get_mediacarrier to get the corresponding bibframe instance class:\n",
    "            # add the resulting bf class to the instance:\n",
    "            # print(\"It's a book! Subclass: \" + str(get_mediacarrier(mt.text)))\n",
    "            records_bf.add((instance_uri, RDF.type, get_mediacarrier(mtfield)))\n",
    "            \n",
    "        \n",
    "        if mt2 is not None and mt2field != \"\":\n",
    "            # use get_mediacarrier to get the corresponding bibframe instance class:\n",
    "            # add the resulting bf class to the instance:\n",
    "            # print(\"It's also a subclass: \" + str(get_mediacarrier(mt2.text)))\n",
    "            # we add another instance for the second book:\n",
    "            instance2 = BNode()\n",
    "            # make it class Instance:\n",
    "            records_bf.add((instance2, RDF.type, BF.Instance))\n",
    "            # since it is \"second instance\", also give it the class bflc:SecondaryInstance:\n",
    "            records_bf.add((instance2, RDF.type, BFLC.SecondaryInstance))\n",
    "            # also add instance subclass that describes the media type (Electronic or Print):\n",
    "            records_bf.add((instance2, RDF.type, get_mediacarrier(mt2field)))\n",
    "            # make sure to connect the two instances with an explicit bf:otherPsychicalFormat,\n",
    "            # even though we already say this _implicitly_ with the bflc:SecondaryInstance class:\n",
    "            records_bf.add((instance_uri, BF.otherPhysicalFormat, instance2))\n",
    "            \n",
    "       \n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Semi-generic helper functions"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Getting subfields from a field"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_subfield(subfield_full_string, subfield_name):\n",
    "    \"\"\"Given a string that contains star subfields (|name ) and the name of the subfield,\n",
    "e.g. i for |i, return the content of only that subfield as a string.\"\"\"\n",
    "    # first, make sure that the extracted substring is not None, not empty or completely comprised of spaces:\n",
    "    if subfield_full_string is not None and subfield_full_string != \"\":\n",
    "        # strip out any double spaces and replace with single space, also strip spaces around:\n",
    "        subfield_full_string = re.sub(' {2,}', ' ', subfield_full_string.strip())\n",
    "        # split out the content of the field - from the first |name to either the next | or the end of subfield_full_string:\n",
    "        subfield = None\n",
    "        # check if the subfield is in the string:\n",
    "        if f\"|{subfield_name}\" in subfield_full_string:\n",
    "            # if it is, split the string on the subfield name:\n",
    "            subfield = subfield_full_string.split(f\"|{subfield_name}\")[1].strip()\n",
    "            # end the string at the next | or the end of the string:\n",
    "            subfield = subfield.split(\"|\")[0].strip()\n",
    "        #subfield = subfield_full_string.split(f\"|{subfield_name}\")[1].strip().split(\"|\")[0].strip()\n",
    "        # print(subfield)\n",
    "        if subfield != \"\" and subfield is not None:\n",
    "            return subfield \n",
    "        else:\n",
    "            return None"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Getting URLs and DOIs from a field\n",
    "Converting http-DOIs to pure ones, checking if what looks like url really is one."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "def check_for_url_or_doi(string):\n",
    "    \"\"\"checks if the content of the string is a doi or url or something else.\n",
    "       Returns the a string and a string_type (doi, url, unknown). The given string \n",
    "       is sanitized, eg. missing http protocol is added for urls; dois are stripped\n",
    "       of web protocols and domain/subdomains like dx, doi.org).\"\"\"\n",
    "    # use a regex: if string starts with \"DOI \" or \"DOI:\" or \"DOI: \" (case insensitive), remove that and strip again:\n",
    "    doi_error_pattern = re.compile(r\"^(DOI:|DOI |DOI: )\", re.IGNORECASE)\n",
    "    string = doi_error_pattern.sub(\"\", string).strip()\n",
    "\n",
    "    # find any stray characters at the start of the field with spaces around them and remove them:\n",
    "    # \"^ . \" - this will catch :, but also |u fields marked with \"x\" for empty:\n",
    "    error_pattern2 = re.compile(r\"^(. )\")\n",
    "    string = error_pattern2.sub(\"\", string).strip()\n",
    "    # and also remove the words \"PsychOpen GOLD\" from the start of the field:\n",
    "    string = re.sub(r\"PsychOpen GOLD\", \"\", string)\n",
    "\n",
    "    # replace double spaces with single space\n",
    "    string = re.sub(' {2,}', ' ', string)\n",
    "\n",
    "        # Fix spaces in common urls: \n",
    "# use this regex: \"(.*\\.) ((io)|(org)|(com)|(net)|(de))\\b\"\n",
    "# and replace with: group1, group2:\n",
    "    url_error_pattern = re.compile(r\"(.*\\.) ((io)|(org)|(com)|(net)|(de))\\b\")\n",
    "    string = url_error_pattern.sub(r\"\\1\\2\", string)\n",
    "\n",
    "    # replace spaces after slashes (when followed by a letter or a digit a question mark, eg \"osf.io/ abc\"):\n",
    "    # use this regex: (.*/) ([a-z]) and replace with group1, group2:\n",
    "    # example: \"osf.io/ abc\" -> \"osf.io/abc\", \"https:// osf.io/ abc\" -> \"https://osf.io/abc\"\n",
    "    url_error_pattern3 = re.compile(r\"(.*/) ([a-z]|[0-9]|\\?)\")\n",
    "    string = url_error_pattern3.sub(r\"\\1\\2\", string)\n",
    "\n",
    "    # and now spaces before slashes:\n",
    "    # use this regex: (.*) (/) and replace with group1, group2:\n",
    "    # example: \"https: //\", \"http://domain.org/blabla /\"\n",
    "    url_error_pattern4 = re.compile(r\"(.*) (/)\")\n",
    "    string = url_error_pattern4.sub(r\"\\1\\2\", string)\n",
    "    \n",
    "    # replace single space with underscore, \n",
    "    # fixing a known STAR bug that replaces underscores with spaces, \n",
    "    # which is especially bad for urls.  (In other text, \n",
    "    # we can't really fix it, since usually a space was intended):\n",
    "    string = re.sub(\" \", \"_\", string)\n",
    "    \n",
    "    # now check for doi or url:\n",
    "    doi_pattern = re.compile(r\"^(https?:)?(\\/\\/)?(dx\\.)?doi\\.org\\/?(.*)$\")\n",
    "    if doi_pattern.search(string):\n",
    "        # remove the matching part:\n",
    "        string = doi_pattern.search(string).group(4)\n",
    "        string_type = \"doi\"\n",
    "        # print(\"DOI: \" + doi)\n",
    "    elif string.startswith(\"10.\"):\n",
    "        # if the string starts with \"10.\" the whole thing is a DOI:\n",
    "        string_type = \"doi\"\n",
    "        # print(\"DOI: \" + doi)\n",
    "        # proceed to generate an identifier node for the doi:\n",
    "    else:\n",
    "        # doi = None\n",
    "        # check for validity of url using a regex:\n",
    "        url_pattern = re.compile(r\"[(http(s)?):\\/\\/(www\\.)?a-zA-Z0-9@:%._\\+~#=]{2,256}\\.[a-z]{2,6}\\b([-a-zA-Z0-9@:%_\\+.~#?&//=]*)\", re.IGNORECASE)\n",
    "        if url_pattern.search(string):\n",
    "            # if it's a nonstandard url starting with \"//\", add a \"http:\" protocol to the start:\n",
    "            if string.startswith(\"//\"):\n",
    "                string = \"http:\" + string\n",
    "                # or if it starts with a letter (like osf.io/), add \"http://\" to the start:\n",
    "            elif string[0].isalpha() and not string.startswith(\"http\"):\n",
    "                string = \"http://\" + string\n",
    "            string_type = \"url\"\n",
    "            # print(\"URL: \" + datac_url)\n",
    "        else:\n",
    "            # url = None\n",
    "            string_type = \"unknown\"\n",
    "            # print(\"Das ist weder eine DOI noch eine URL: \" + string)\n",
    "    return string, string_type"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Building identifier nodes for DOIs\n",
    "\n",
    "Should probably refactor to be more general, so we can use it for other identifiers as well. Needs a parameter for the identifier type."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_doi_identifier_node(instance, doi):\n",
    "    # print(f\"bf:identifiedBy > bf:Doi > rdf:value: {doi}.\")\n",
    "    # make bnode for the identifier:\n",
    "    identifier_node = BNode()\n",
    "    # give it class bf:Doi:\n",
    "    records_bf.add((identifier_node, RDF.type, BF.Doi))\n",
    "    # give it the doi as a literal value:\n",
    "    records_bf.add((identifier_node, RDF.value, Literal(doi)))\n",
    "    # attach it to the instance with bf:identifiedBy:\n",
    "    records_bf.add((instance, BF.identifiedBy, identifier_node))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Building \"Links\" as electronic locator nodes for an instance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_electronic_locator_node(instance, url):\n",
    "    locator_node = BNode()\n",
    "    # add it to the instance_node of relationship_node via bf:electronicLocator:\n",
    "    # no specific class!\n",
    "    # give it the url as a literal value:\n",
    "    records_bf.set((locator_node, RDF.value, Literal(url, datatype=XSD.anyURI)))\n",
    "    # attach it to the instance with bf:electronicLocator:\n",
    "    records_bf.set((instance, BF.electronicLocator, locator_node))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Building generic bf:Note nodes\n",
    "\n",
    "Will probably also need this later for other kinds of notes, such as the ones in field BN."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_note_node(instance, note):\n",
    "    note_node = BNode()\n",
    "    records_bf.set((note_node, RDF.type, BF.Note))\n",
    "    records_bf.set((note_node, RDFS.label, Literal(note)))\n",
    "    records_bf.set((instance, BF.note, note_node))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Function: Replace weird characters with unicode\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from modules.mappings import dd_codes\n",
    "\n",
    "# def replace_encodings(text):\n",
    "#     # text = html.escape(text)\n",
    "#     for case in dd_codes:\n",
    "#         text = text.replace(case[0], case[1]) \n",
    "#     return text\n",
    "\n",
    "# moved to modules.mappings!"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Function: Guess language of a given string\n",
    "Used for missing language fields or if there are discrepancies between the language field and the language of the title etc."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "import langid\n",
    "langid.set_languages([\"de\", \"en\"])\n",
    "\n",
    "def guess_language(string_in_language):\n",
    "    return (langid.classify(string_in_language)[0])"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Function: Adding DFK as an Identifier"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### DFK as id for Bibframe\n",
    "\n",
    "We want to add the DFK as a local bf:Identifier to the work (or instance?). \n",
    "We also want to say where the Identifier originates (to say it is from PSYNDEX/ZPID). \n",
    "\n",
    "The format for that is:\n",
    "```turtle\n",
    "<Work/Instance> bf:identifiedBy [\n",
    "    a bf:Local, pxc:DFK; \n",
    "    rdf:value \"1234456\"; \n",
    "    bf:source [\n",
    "        a bf:Source; bf:code \"ZPID.PSYNDEX.DFK\"\n",
    "    ]\n",
    "];\n",
    "```\n",
    "\n",
    "So, we need a blank node for the Identifier and inside, another nested bnode for the bf:Source. This is a function that will return such an identifier bnode to add to the work_uri. We are calling it way up down below in the loop:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "#  a function to be called in a for-loop while going through all records of the source xml, \n",
    "# which returns a new triple to add to the graph that has a bnode for the dfk identifier.\n",
    "# The predicate is \"bf:identifiedBy\" and the object is a blank node of rdf:Type \"bf:Identifier\" and \"bf:Local\":\n",
    "# The actual identifier is a literal with the text from the \"DFK\" element of the record.\n",
    "def get_bf_identifier_dfk(resource_uri, dfk):\n",
    "    # make a node of the Identifier class from the BF namespace:\n",
    "    identifier = URIRef(resource_uri + \"#dfk\")\n",
    "    #identifier = URIRef(instance_uri + \"/identifier/dfk\")\n",
    "    #identifier_source = BNode()\n",
    "    # records_bf.add ((identifier, RDF.type, BF.Identifier))\n",
    "    #records_bf.add ((identifier, RDF.type, BF.Local))\n",
    "    records_bf.add ((identifier, RDF.type, PXC.DFK))\n",
    "    # build the source node:\n",
    "   # records_bf.add((identifier_source, RDF.type, BF.Source))\n",
    "    #records_bf.add((identifier_source, BF.code, Literal(\"ZPID.PSYNDEX.DFK\")))\n",
    "\n",
    "    # hang the id source node into the id node:\n",
    "    #records_bf.add((identifier, BF.source, identifier_source))\n",
    "    records_bf.add((identifier, RDF.value, Literal(dfk)))\n",
    "    return (identifier)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generic Function: Replace languages with their language tag\n",
    "\n",
    "Can be used for different fields that are converted to langstrings or language uris. Use within other functions that work with the languages in different fields.\n",
    "\n",
    "Returns an array with two values: a two-letter langstring tag at [0] and a three-letter uri code for the library of congress language vocab at [1]."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_langtag_from_field(langfield):\n",
    "    # when passed a string from any language field in star, returns an array with two items. \n",
    "    # Index 0: two-letter langstring tag, e.g. \"de\"\n",
    "    # Index 1: two-letter iso langtag, e.g. \"ger\"\n",
    "    # can be used on these fields (it contains the different spellings found in them):\n",
    "    # \"LA\", \"LA2\", \"TIL\", \"TIUL\", \"ABLH\", \"ABLN\", \"TIUE |s\"\n",
    "    match langfield:\n",
    "        case \"german\" | \"de\" | \"GERM\" | \"Deutsch\" | \"GERMAN\" | \"GERMaN\" | \"German\" | \"Fi\":\n",
    "            return [\"de\", \"ger\"]\n",
    "        case \"en\" | \"ENGL\" | \"ENGLISH\" | \"Englisch\" | \"English\" | \"English; English\" | \"english\" :\n",
    "            return [\"en\", \"eng\"]\n",
    "        case \"BULG\" | \"Bulgarian\":\n",
    "            return [\"bg\", \"bul\"]\n",
    "        case \"SPAN\"| \"Spanish\":\n",
    "            return [\"es\", \"spa\"]\n",
    "        case \"Dutch\":\n",
    "            return [\"nl\", \"dut\"]\n",
    "        case \"CZEC\":\n",
    "            return [\"cs\", \"ces\"]\n",
    "        case \"FREN\" | \"French\":\n",
    "            return [\"fr\", \"fra\"]\n",
    "        case \"ITAL\" | \"Italian\":\n",
    "            return [\"it\", \"ita\"]\n",
    "        case \"PORT\" | \"Portuguese\":\n",
    "            return [\"pt\", \"por\"]\n",
    "        case \"JAPN\" | \"Japanese\":\n",
    "            return [\"jp\", \"jpn\"]\n",
    "        case \"HUNG\":\n",
    "            return [\"hu\", \"hun\"]\n",
    "        case \"RUSS\" | \"Russian\":\n",
    "            return [\"ru\", \"rus\"]\n",
    "        case \"NONE\" | \"Silent\":\n",
    "            return [\"zxx\", \"zxx\"]\n",
    "        case _:\n",
    "            return [\"und\", \"und\"] # for \"undetermined!\"\n",
    "\n",
    "\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Function: Get work language from LA\n",
    "\n",
    "Example\n",
    "\n",
    "```turtle\n",
    "@prefix lang: <http://id.loc.gov/vocabulary/iso639-2/> .\n",
    "<W> bf:language lang:ger .\n",
    "```\n",
    "\n",
    "Calls the generic language code lookup function above, get_langtag_from_field, passing the LA field content, returning a uri from the library of congress language vocabulary (built from namespace + 3-letter iso code). \n",
    "\n",
    "TODO:\n",
    "- But what if field LA is missing? (doesn't occur in test set, but not impossible)\n",
    "- or if there is another language in LA2? (in my test set, 2 out of 700 records have LA2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "# function \n",
    "def get_work_language(record):\n",
    "    work_language = get_langtag_from_field(record.find(\"LA\").text.strip())[1]\n",
    "    work_lang_uri = LANG[work_language]\n",
    "    return (work_lang_uri)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Function: Build a Relationship Node for different types of related works\n",
    "\n",
    "Should take parameters - a dict per type (research data closed access, rd open access, ...) that has values for all the needed fields"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "from uri_template import URITemplate\n",
    "\n",
    "\n",
    "def build_work_relationship_node(work_uri, relation_type):\n",
    "    # check the relation_type against the relation_types dict:\n",
    "    if relation_type in relation_types:\n",
    "        # if it is, get the values for the relation_type:\n",
    "        relation = relation_types[relation_type][\"relation\"]\n",
    "        relatedTo_subprop = relation_types[relation_type][\"relatedTo_subprop\"]\n",
    "        work_subclass = relation_types[relation_type][\"work_subclass\"]\n",
    "        content_type = relation_types[relation_type][\"content_type\"]\n",
    "        genre = relation_types[relation_type][\"genre\"]\n",
    "        access_policy_label = relation_types[relation_type][\"access_policy_label\"]\n",
    "        access_policy_value = relation_types[relation_type][\"access_policy_value\"]\n",
    "    # make a bnode for this relationship:\n",
    "    relationship_bnode = BNode()\n",
    "    # make it class bflc:Relationship:\n",
    "    records_bf.set((relationship_bnode, RDF.type, BFLC.Relationship))\n",
    "    # add a bflc:Relation (with a label and value) via bflc:relation to the relationship bnode \n",
    "    # (label and value could be given as a parameter):\n",
    "    # print(\"\\tbflc:relation [a bflc:Relation ; rdfs:label 'has research data', rdf:value 'relation:hasResearchData'^^xsd:anyURI] ;\")\n",
    "    # relation_bnode = BNode()\n",
    "    # records_bf.set((relation_bnode, RDF.type, BFLC.Relation))\n",
    "    # records_bf.add((relation_bnode, RDFS.label, Literal(\"has research data\", lang=\"en\")))\n",
    "    # records_bf.add((relation_bnode, RDF.value, Literal(RELATIONS.hasResearchData)))\n",
    "    records_bf.set((relationship_bnode, BFLC.relation, URIRef(RELATIONS[relation])))\n",
    "    # make a bnode for the work:\n",
    "    related_work_bnode = BNode()\n",
    "    records_bf.add((related_work_bnode, RDF.type, BF.Work))\n",
    "    records_bf.add((related_work_bnode, RDF.type, URIRef(BF[work_subclass])))\n",
    "    # give work a content type:\n",
    "    records_bf.add((related_work_bnode, BF.content, URIRef(CONTENTTYPES[content_type])))\n",
    "    # make the content type a bf:Content:\n",
    "    records_bf.add((URIRef(CONTENTTYPES[content_type]), RDF.type, BF.Content))\n",
    "    # and a genre:\n",
    "    records_bf.add((related_work_bnode, BF.genreForm, URIRef(GENRES[genre])))\n",
    "    # make the genreform a bf:GenreForm:\n",
    "    records_bf.add((URIRef(GENRES[genre]), RDF.type, BF.GenreForm))\n",
    "    # attach the work bnode to the relationship bnode with bf:relatedTo \n",
    "    # (or a subproperty as given as a parameter)):\n",
    "    # print(\"\\tbf:relatedTo [a bf:Work ;\")\n",
    "    records_bf.add((relationship_bnode, BF[relatedTo_subprop], related_work_bnode))\n",
    "    # make a bnode for the instance:\n",
    "    related_instance_bnode = BNode()\n",
    "    records_bf.set((related_instance_bnode, RDF.type, BF.Instance))\n",
    "    records_bf.add((related_instance_bnode, RDF.type, BF.Electronic))\n",
    "    records_bf.add((related_work_bnode, BF.hasInstance, related_instance_bnode))\n",
    "    # add accesspolicy to instance:\n",
    "    if access_policy_label is not None and access_policy_value is not None:\n",
    "        access_policy_node = BNode()\n",
    "        records_bf.add((access_policy_node, RDF.type, BF.AccessPolicy))\n",
    "        records_bf.add((access_policy_node, RDFS.label, Literal(access_policy_label, lang=\"en\")))\n",
    "        records_bf.add((access_policy_node, RDF.value, Literal(access_policy_value, datatype=XSD.anyURI)))\n",
    "        records_bf.add((related_instance_bnode, BF.usageAndAccessPolicy, access_policy_node))\n",
    "    # in the end, return the relationship bnode so it can be attached to the work\n",
    "    # records_bf.add((work_uri, BFLC.relationship, relationship_bnode))\n",
    "    return relationship_bnode, related_instance_bnode\n",
    "\n",
    "relation_types = {\n",
    "    \"rd_open_access\": {\n",
    "        \"relation\": \"hasResearchData\",\n",
    "        \"relatedTo_subprop\": \"supplement\",\n",
    "        \"work_subclass\": \"Dataset\",\n",
    "        \"content_type\": \"cod\",\n",
    "        \"genre\": \"researchData\",\n",
    "        \"access_policy_label\": \"open access\",\n",
    "        \"access_policy_value\": \"http://purl.org/coar/access_right/c_abf2\"\n",
    "    },\n",
    "    \"rd_restricted_access\": {\n",
    "        \"relation\": \"hasResearchData\",\n",
    "        \"relatedTo_subprop\": \"supplement\",\n",
    "        \"work_subclass\": \"Dataset\",\n",
    "        \"content_type\": \"cod\",\n",
    "        \"genre\": \"researchData\",\n",
    "        \"access_policy_label\": \"restricted access\",\n",
    "        \"access_policy_value\": \"http://purl.org/coar/access_right/c_16ec\"\n",
    "    },\n",
    "    \"preregistration\": {\n",
    "        \"relation\": \"hasPreregistration\",\n",
    "        \"relatedTo_subprop\": \"supplement\",\n",
    "        \"work_subclass\": \"Text\",\n",
    "        \"content_type\": \"txt\",\n",
    "        \"genre\": \"preregistration\",\n",
    "        \"access_policy_label\": None,\n",
    "        \"access_policy_value\": None,\n",
    "    },\n",
    "}"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Function: Create Instance Title nodes from fields TI, TIU, TIL, TIUE...\n",
    "\n",
    "Titles and Translated titles are attached to Instances. Translated titles also have a source, which can be DeepL, ZPID, or Original.\n",
    "\n",
    "Example:\n",
    "\n",
    "```turtle\n",
    "<Instance> bf:title \n",
    "        [a bf:Title; \n",
    "            bf:mainTitle \"Disentangling the process of epistemic change\"@en;\n",
    "            bf:subtitle \"The role of epistemic volition\"@en;\n",
    "        ],\n",
    "        [a pxc:TranslatedTitle;\n",
    "            rdfs:label \"Den Prozess des epistemischen Wandels entwirren: Die Rolle des epistemischen Willens.\"@de;\n",
    "            bf:mainTitle \"Den Prozess des epistemischen Wandels entwirren: Die Rolle des epistemischen Willens.\"@de;\n",
    "            bf:adminMetadata  [ \n",
    "                a bf:AdminMetadata ;\n",
    "                bflc:metadataLicensor  \"DeepL\";\n",
    "        ]\n",
    "        ].\n",
    "```\n",
    "\n",
    "- [x] add TI as bf:Title via bf:mainTitle\n",
    "- [x] add subtitle from TIU\n",
    "- [x] create a concatenated rdfs:label from TI and TIU\n",
    "- [x] add languages for maintitle and subtitle (from TIL and TIUL)\n",
    "\n",
    "- [x] add translated title from TIUE as pxc:TranslatedTitle with bf:mainTitle and rdfs:label \n",
    "- [x] add languages for translated title (from subfield TIU |s, or if unavailable, guess language from the subtitle string itself (contents of TIU)\n",
    "- [x] create a source/origin for the translated title (from \"(DeepL)\" at the end)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "#  a function to be called in a for-loop while going through all records of the source xml, \n",
    "# which returns a new triple to add to the graph that has a bnode for the dfk identifier.\n",
    "# The predicate is \"bf:identifiedBy\" and the object is a blank node of rdf:Type \"bf:Identifier\" and \"bf:Local\":\n",
    "# The actual identifier is a literal with the text from the \"DFK\" element of the record.\n",
    "def get_bf_title(resource_uri, record):\n",
    "    # make a  BNODE for the title:\n",
    "    title = URIRef(resource_uri + \"#title\")\n",
    "    # make it bf:Title class:\n",
    "    records_bf.add ((title, RDF.type, BF.Title))\n",
    "\n",
    "    # get the content of th TI field as the main title:\n",
    "    maintitle = html.unescape(mappings.replace_encodings(record.find(\"TI\").text).strip())\n",
    "    # write a full title for the rdfs:label \n",
    "    # (update later if subtitle exists to add that)\n",
    "    fulltitle = maintitle\n",
    "    # set fallback language for main title:\n",
    "    maintitle_language = \"en\"\n",
    "    subtitle_language = \"en\"\n",
    "    # get language of main title - if exists!:\n",
    "    if record.find(\"TIL\") is not None:\n",
    "        maintitle_language = get_langtag_from_field(record.find(\"TIL\").text.strip())[0]\n",
    "        # if maintitle_language that is returned the get_langtag_from_field is \"und\" \n",
    "        # (because it was a malformed language name), guess the language from the string itself!\n",
    "        if maintitle_language == \"und\":\n",
    "            maintitle_language = guess_language(maintitle)\n",
    "    else: # if there is no TIL field, guess the language from the string itself!\n",
    "        maintitle_language = guess_language(maintitle)\n",
    "\n",
    "   \n",
    "\n",
    "    # add the content of TI etc via bf:mainTitle:\n",
    "    records_bf.add((title, BF.mainTitle, Literal(maintitle, lang=maintitle_language)))\n",
    "    # get content of the TIU field as the subtitle, \n",
    "    # _if_ it exists and has text in it:\n",
    "    if record.find(\"TIU\") is not None and record.find(\"TIU\").text != \"\":\n",
    "        subtitle = html.unescape(mappings.replace_encodings(record.find(\"TIU\").text).strip()) # sanitize encoding and remove extraneous spaces\n",
    "        # concatenate a full title from main- and subtitle, \n",
    "        # separated with a : and overwrite fulltitle with that\n",
    "        fulltitle = fulltitle + \": \" + subtitle\n",
    "        # get language of subtitle - it is in field TIUL, but sometimes that is missing...:\n",
    "        #  # get language of subtitle:\n",
    "        if record.find(\"TIUL\") is not None:\n",
    "            subtitle_language = get_langtag_from_field(record.find(\"TIUL\").text.strip())[0]\n",
    "            if subtitle_language == \"und\":\n",
    "                subtitle_language = guess_language(subtitle)\n",
    "        else: # if there is no TIUL field, guess the language from the string itself!\n",
    "            subtitle_language = guess_language(subtitle)\n",
    "\n",
    "        # add the content of TIU to the bf:Title via bf:subtitle:\n",
    "        records_bf.add((title, BF.subtitle, Literal(subtitle, lang=subtitle_language)))\n",
    "\n",
    "    \n",
    "\n",
    "    # add the concatenated full title to the bf:Title via rdfs:label:\n",
    "    # (we don't care if the main title's and subtitle's languages don't match - we just set the language of the main title as the full title's language)\n",
    "    records_bf.add((title, RDFS.label, Literal(fulltitle, lang=maintitle_language)))\n",
    "\n",
    "    # # hang the id source node into the id node:\n",
    "    # records_bf.add((identifier, BF.source, identifier_source))\n",
    "    return (title)\n",
    "\n",
    "# function for the translated title:\n",
    "def get_bf_translated_title(resource_uri, record):\n",
    "    translated_title = URIRef(resource_uri + \"#translatedtitle\")\n",
    "    #translated_title = URIRef(instance_uri + \"/title/translated\")\n",
    "    records_bf.add ((translated_title, RDF.type, PXC.TranslatedTitle))\n",
    "    fulltitle = html.unescape(mappings.replace_encodings(record.find(\"TIUE\").text).strip())\n",
    "    fulltitle_language = \"de\"\n",
    "    # read subfield |s to get the actual language (it doesn't always exist, though). \n",
    "    # if fulltitle string ends with \"|s \" followed by some text (use a regex):\n",
    "    match = re.search(r'^(.*)\\s\\|s\\s(.*)', fulltitle)\n",
    "    if match:\n",
    "        fulltitle = match.group(1).strip()\n",
    "        fulltitle_language = get_langtag_from_field(match.group(2).strip())[0]\n",
    "    else:\n",
    "        # get the language in TIUE, if that field exists, and invert it to get the language of the translation:\n",
    "        # if record.find(\"TIL\") is not None:\n",
    "        #     original_title_language_til = get_langtag_from_field(record.find(\"TIL\").text.strip())[0]\n",
    "            \n",
    "        #     # if it is German -> use inverse: \"en\"\n",
    "        #     if original_title_language_til == \"de\":\n",
    "        #         fulltitle_language = \"en\"\n",
    "        #     # else -> keep \"de\"\n",
    "        # if the language of the translated title (in |s) is missing, guess the language from the string itself!\n",
    "        fulltitle_language = guess_language(fulltitle)\n",
    "\n",
    "    # check if the title contains a \"(DeepL)\" and cut it into a variable for the source:\n",
    "    titlesource = \"ZPID\" # translation source is \"ZPID\" by default\n",
    "    # note: we might be able to add source \"Original\" by finding out \n",
    "    # if the source of the secondary abstract is something other than ZPID!\n",
    "    match_source = re.search(r'^(.*)\\((DeepL)\\)$', fulltitle)\n",
    "    if match_source:\n",
    "        fulltitle = match_source.group(1).strip()\n",
    "        titlesource = match_source.group(2)\n",
    "\n",
    "    # build a source node for the translation:\n",
    "    titlesource_node = URIRef(translated_title + \"_source\")\n",
    "    records_bf.add ((titlesource_node, RDF.type, BF.AdminMetadata))\n",
    "    records_bf.add ((titlesource_node, BFLC.metadataLicensor, Literal(titlesource)))\n",
    "\n",
    "    # add the title string to the bnode:\n",
    "    records_bf.add((translated_title, BF.mainTitle, Literal(fulltitle, lang=fulltitle_language)))\n",
    "    records_bf.add((translated_title, RDFS.label, Literal(fulltitle, lang=fulltitle_language)))\n",
    "    records_bf.add((translated_title, BF.adminMetadata, titlesource_node))\n",
    "\n",
    "    return (translated_title)\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Function: Add Abstracts - original abstract (from fields ABH, ABLH, ABSH1, ABSH2) and translated/secondary abstract (from ABN, ABLN, ASN1, ASN2)\n",
    "\n",
    "- Main Abstract: \n",
    "    - abstract text is in field ABH.\n",
    "    - abstract language is in ABLH (\"German\" or \"English\") but can be missing in rare cases! In that case, we guess it using the langid module.\n",
    "    - abstract original source is in ASH1 (\"Original\" or \"ZPID\")\n",
    "    - agent who edited the original, if that happened, is in ASH2 ()\n",
    "- Secondary Abstract \n",
    "    - abstract text is in field ABN.\n",
    "    - abstract language is in ABLN (\"German\" or \"English\")\n",
    "    - abstract original source is in ASN1 (\"Original\" or \"ZPID\")\n",
    "    - agent who edited the original, if that happened, is in ASN2 ()\n",
    "\n",
    "Scheme:\n",
    "\n",
    "```turtle\n",
    "<W> bf:summary \n",
    "    [ a pxc:Abstract , bf:Summary ;\n",
    "        rdfs:label  \"Background: Loneliness is ...\"@en ;\n",
    "        bf:adminMetadata  [ \n",
    "            a bf:AdminMetadata ;\n",
    "            bflc:metadataLicensor  \"Original\";\n",
    "            bf:descriptionModifier \"ZPID\"\n",
    "        ]\n",
    "] .\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "from modules.mappings import abstract_origin_original, abstract_origin_zpid, abstract_origin_deepl, abstract_origin_gesis, abstract_origin_fis_bildung, abstract_origin_krimz\n",
    "\n",
    "def replace_abstract_origin_string(origin_string):\n",
    "    # if the passed string is in \"abstract_origin_original\", thenreplace it with \"Original\":\n",
    "    if origin_string in abstract_origin_original:\n",
    "        return \"Original\"\n",
    "    elif origin_string in abstract_origin_zpid:\n",
    "        return \"ZPID\"\n",
    "    # elif origin_string in abstract_origin_iwf:\n",
    "    #     return \"IWF\"\n",
    "    elif origin_string in abstract_origin_deepl:\n",
    "        return \"DeepL\"\n",
    "    elif origin_string in abstract_origin_gesis:\n",
    "        return \"GESIS\"\n",
    "    elif origin_string in abstract_origin_fis_bildung:\n",
    "        return \"FIS Bildung\"\n",
    "    elif origin_string in abstract_origin_krimz:\n",
    "        return \"KrimZ\"\n",
    "    else:\n",
    "        return origin_string\n",
    "\n",
    "\n",
    "# function to get the original abstract:\n",
    "def get_bf_abstract(work_uri, record):\n",
    "    \"\"\"Extracts the abstract from field ABH and adds a bf:Summary bnode with the abstract and its metadata. Also extracts the Table of Content from the same field.\"\"\"\n",
    "    abstract = BNode()\n",
    "    # abstract = URIRef(work_uri + \"/abstract\")\n",
    "    records_bf.add ((abstract, RDF.type, PXC.Abstract))\n",
    "    # get abstract text from ABH\n",
    "    abstracttext = html.unescape(mappings.replace_encodings(record.find(\"ABH\").text).strip())\n",
    "    # check if the abstracttext ends with \" (translated by DeepL)\" and if so, remove that part:\n",
    "    match1 = re.search(r'^(.*)\\s\\(translated by DeepL\\)$', abstracttext)\n",
    "    if match1:\n",
    "        abstracttext = match1.group(1).strip()\n",
    "    # check via regex if there is a \" - Inhalt: \" or \" - Contents: \" in it.\n",
    "    # if so, split out what comes after. Drop the contents/inhalt part itself.\n",
    "    match2 = re.search(r'^(.*)[-–]\\s*(?:Contents|Inhalt)\\s*:\\s*(.*)$', abstracttext)\n",
    "    if match2:\n",
    "        abstracttext = match2.group(1).strip()\n",
    "        contents = match2.group(2).strip()\n",
    "        # make a node for bf:TableOfContents:\n",
    "        toc = BNode()\n",
    "        records_bf.add((toc, RDF.type, BF.TableOfContents))\n",
    "        # add the bnode to the work via bf:tableOfContents:\n",
    "        records_bf.add((work_uri, BF.tableOfContents, toc))\n",
    "        # add the contents to the abstract node as a bf:tableOfContents:\n",
    "        # if the contents start with http, extract as url into rdf:value:\n",
    "        if contents.startswith(\"http\"):\n",
    "            records_bf.add((toc, RDF.value, Literal(contents, datatype=XSD.anyURI)))\n",
    "            # otherwise it's a text toc and needs to go into the label\n",
    "        else:\n",
    "            records_bf.add((toc, RDFS.label, Literal(contents, lang=\"und\")))\n",
    "    # get abstract language from ABLH (\"German\" or \"English\")\n",
    "    abstract_language = \"en\" # set default\n",
    "    # that's a bad idea, actually. Better: if field is missing, use a language recog function!\n",
    "    if record.find(\"ABLH\") is not None:\n",
    "        abstract_language = get_langtag_from_field(record.find(\"ABLH\").text.strip())[0]\n",
    "        if abstract_language == \"und\":\n",
    "            # guess language from the text:\n",
    "            abstract_language = guess_language(abstracttext)\n",
    "    else: # if the ABLH field is missing, try to recognize the language of the abstract from its text:\n",
    "        abstract_language = guess_language(abstracttext)\n",
    "\n",
    "    # add the text to the bnode:\n",
    "    records_bf.add ((abstract, RDFS.label, Literal(abstracttext, lang=abstract_language)))\n",
    "\n",
    "    # get abstract original source from ASH1 (\"Original\" or \"ZPID\")\n",
    "    abstract_source = \"Original\" # default\n",
    "    # create a blank node for admin metadata:\n",
    "    abstract_source_node = BNode()\n",
    "    records_bf.add((abstract_source_node, RDF.type, BF.AdminMetadata))\n",
    "\n",
    "    if record.find(\"ASH1\") is not None:\n",
    "        # overwrite default (\"Original\") with what we find in ASH1:\n",
    "        # and while we're at it, replace some known strings with their respective values \n",
    "        # (e.g. employee tags with \"ZPID\"):\n",
    "        abstract_source = replace_abstract_origin_string(record.find(\"ASH1\").text.strip())\n",
    "    \n",
    "    # write final source text into source node:\n",
    "    records_bf.add((abstract_source_node, BFLC.metadataLicensor, Literal(abstract_source)))\n",
    "\n",
    "    # here is a list of known zpid employee tags, we will use them later to replace these with \"ZPID\" if found in ASH2:\n",
    "\n",
    "    # and this is a list of things we want to replace with \"Original\":\n",
    "    \n",
    "\n",
    "    # get optional agent who edited the original abstract from ASH2\n",
    "    if record.find(\"ASH2\") is not None:\n",
    "        # note what we find in ABSH2:\n",
    "        abstract_editor = replace_abstract_origin_string(record.find(\"ASH2\").text.strip())\n",
    "        \n",
    "        records_bf.add((abstract_source_node, BF.descriptionModifier, Literal(abstract_editor)))\n",
    "\n",
    "\n",
    "    #add the source node to the abstract node:\n",
    "    records_bf.add((abstract, BF.adminMetadata, abstract_source_node))\n",
    "    # and return the completed node:\n",
    "    #return (abstract)\n",
    "# or better, attach it right away:\n",
    "    records_bf.add((work_uri, BF.summary, abstract))\n",
    "\n",
    "def get_bf_secondary_abstract(work_uri, record):\n",
    "    abstract = BNode()\n",
    "    # abstract = URIRef(work_uri + \"/abstract/secondary\")\n",
    "    records_bf.add ((abstract, RDF.type, PXC.Abstract))\n",
    "    records_bf.add ((abstract, RDF.type, PXC.SecondaryAbstract))\n",
    "    abstracttext = html.unescape(mappings.replace_encodings(record.find(\"ABN\").text).strip())\n",
    "    # check if the abstracttext ends with \" (translated by DeepL)\" and if so, remove that part:\n",
    "    match = re.search(r'^(.*)\\s\\(translated by DeepL\\)$', abstracttext)\n",
    "    if match:\n",
    "        abstracttext = match.group(1).strip()\n",
    "\n",
    "    abstract_language = \"de\" # fallback default\n",
    "    \n",
    "    if record.find(\"ABLN\") is not None and record.find(\"ABLN\").text != \"\":\n",
    "        abstract_language = get_langtag_from_field(record.find(\"ABLN\").text.strip())[0]\n",
    "        if abstract_language == \"und\":\n",
    "            # guess language from the text:\n",
    "            abstract_language = guess_language(abstracttext)\n",
    "    else: # if no language field, guess language from the text:\n",
    "        abstract_language = guess_language(abstracttext)\n",
    "    \n",
    "    records_bf.add ((abstract, RDFS.label, Literal(abstracttext, lang=abstract_language)))\n",
    "    \n",
    "    abstract_source_node = BNode()\n",
    "    records_bf.add((abstract_source_node, RDF.type, BF.AdminMetadata))\n",
    "    abstract_source = \"Original\" # fallback default\n",
    "    if record.find(\"ASN1\") is not None:\n",
    "        # overwrite default (\"Original\") with what we find in ASH1:\n",
    "        abstract_source = replace_abstract_origin_string(record.find(\"ASN1\").text.strip())\n",
    "    \n",
    "    records_bf.add((abstract_source_node, BFLC.metadataLicensor, Literal(abstract_source)))\n",
    "\n",
    "    # get optional agent who edited the original abstract from ASH2\n",
    "    if record.find(\"ASN2\") is not None:\n",
    "        # note what we find in ABSN2:\n",
    "        abstract_editor = replace_abstract_origin_string(record.find(\"ASN2\").text.strip())\n",
    "        # and add it via decription modifier:\n",
    "        records_bf.add((abstract_source_node, BF.descriptionModifier, Literal(abstract_editor)))\n",
    "\n",
    "    #add the source node to the abstract node:\n",
    "    records_bf.add((abstract, BF.adminMetadata, abstract_source_node))\n",
    "    # and return the completed node:\n",
    "    return (abstract)\n",
    "\n",
    "\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Function to split Table of Content from the Abstract field (ABH)\n",
    "\n",
    "This usually starts with \" - Inhalt: \" (for German Abstracts) or \" - Contents: \" (in English abstracts) and ends at the end of the field.\n",
    "It can contain a numbered list of chapters or sections as a long string. It can also contain a uri from dnb namespace instead or in addition!\n",
    "\n",
    "Examples:\n",
    "- \" - Contents: (1) ...\"\n",
    "- \" - Inhalt: https://d-nb.info/1256712809/04</ABH>\" (URI pattern: \"https://d-nb.info/\" + \"1256712809\" 10 digits + \"/04\")\n",
    "\n",
    "Example:\n",
    "\n",
    "```turtle\n",
    "<W> bf:tableOfContents [\n",
    "    a bf:TableOfContents;\n",
    "    rdfs:label \"(1) Wünsche, J., Weidmann, R. &amp; Grob, A. (n. d.). Happy in the same way? The link between domain satisfaction and overall life satisfaction in romantic couples. Manuscript submitted for publication. (2) Wünsche, J., Weidmann,...\";\n",
    "] .\n",
    "```\n",
    "\n",
    "Or\n",
    "\n",
    "```turtle\n",
    "<W> bf:tableOfContents [\n",
    "    a bf:TableOfContents;\n",
    "    rdf:value \"https://d-nb.info/1002790794/04\"^^xsd:anyURI ;\n",
    "] .\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_bf_toc(work_uri, record):\n",
    "    # read the abstract in ABH\n",
    "    contents = \"\"\n",
    "    if record.find(\"ABH\") is not None:\n",
    "        abstracttext = html.unescape(mappings.replace_encodings(record.find(\"ABH\").text).strip())\n",
    "        # check via regex if there is a \" - Inhalt: \" or \" - Contents: \" in it.\n",
    "        # if so, split out what comes after. Drop the contents/inhalt part itself.\n",
    "        match = re.search(r'^(.*)[-–]\\s*(?:Contents|Inhalt)\\s*:\\s*(.*)$', abstracttext)\n",
    "        if match:\n",
    "            abstracttext = match.group(1).strip()\n",
    "            contents = match.group(2).strip()\n",
    "\n",
    "    # also check if what comes is either a string or a uri following thegiven pattern\n",
    "    # and export one as a rdfs_label and the other as rdf:value \"...\"^^xsd:anyUrl (remember to add XSD namespace!)\n",
    "    # also remember that we should only create a node and attach it to the work\n",
    "    # if a) ABH exists at all and\n",
    "    # b) the regex is satisfied.\n",
    "    # So I guess we must do the whole checking and adding procedure in this function!\n",
    "\n",
    "    # only return an added triple if the toc exisits, otherwise return nothing:\n",
    "    if contents:\n",
    "        return records_bf.add((work_uri, BF.tableOfContents, Literal(contents)))\n",
    "    else: \n",
    "        return None\n",
    "    # return records_bf.add((work_uri, BF.tableOfContents, Literal(\"test\")))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Function: Create Person Contribution nodes from Fields AUP, EMID, EMAIL, AUK, PAUP, CS and COU\n",
    "\n",
    "Use this scheme:\n",
    "\n",
    "```turtle\n",
    "<Work> a bf:Work;\n",
    "    bf:contribution \n",
    "    [\n",
    "        # the Bibframe Contribution includes, as usual, an agent and their role,\n",
    "        # but is supplemented with an Affiliation (in the context of that work/while it was written),\n",
    "        # and a position in the author sequence.\n",
    "        a bf:Contribution, bflc:PrimaryContribution; \n",
    "        bf:agent \n",
    "        [\n",
    "            a bf:Person, schema:Person; \n",
    "            rdfs:label \"Trillitzsch, Tina\"; # name when creating work\n",
    "            schema:givenName \"Tina\"; schema:familyName \"Trillitzsch\";\n",
    "            # owl:sameAs <https://w3id.org/zpid/person/tt_0000001>, <https://orcid.org/0000-0001-7239-4844>; # authority uris of person (local, orcid)\n",
    "            bf:identifiedBy [a pxc:PsychAuthorsID; rdf:value \"p01979TTR\"; #legacy authority ID\n",
    "            ];\n",
    "            bf:identifiedBy [a locid:orcid; rdf:value \"0000-0001-7239-4844\"; # ORCID \n",
    "            ];\n",
    "        ]\n",
    "        # we use a model inspired by Option C in Osma Suominen'a suggestion for /~https://github.com/dcmi/dc-srap/issues/3\n",
    "        # adding the Affiliation into the Contribution, separate from the agent itself, since the affiliation\n",
    "        # is described in the context of this work, not not as a statement about the person's\n",
    "        # current affiliation:\n",
    "        mads:hasAffiliation [\n",
    "            a mads:Affiliation;\n",
    "            # Affiliation blank node has info about the affiliation org (including persistent identifiers),\n",
    "            # the address (country with geonames identifier),\n",
    "            # and the person's email while affiliated there.\n",
    "            mads:organization [\n",
    "                a bf:Organization; \n",
    "                rdfs:label \"Leibniz Institute of Psychology (ZPID); Digital Research Development Services\"; # org name when work was created\n",
    "                # owl:sameAs <https://w3id.org/zpid/org/zpid_0000001>, <https://ror.org/0165gz615>; # authority uris of org (local, ror)\n",
    "                # internal id and ror id as literal identifiers:\n",
    "                bf:identifiedBy [a pxc:ZpidCorporateBodyId; rdf:value \"0000001\"; ];\n",
    "                bf:identifiedBy [a locid:ror; rdf:value \"0165gz615\"; ];\n",
    "            ];\n",
    "            mads:hasAffiliationAddress [a mads:Address;\n",
    "                mads:country [\n",
    "                    a mads:Country;\n",
    "                    rdfs:label \"Germany\";\n",
    "                    bf:identifiedBy [a locid:geonames; rdf:value \"2921044\"; ];\n",
    "                    # owl:sameAs <https://w3id.org/zpid/place/country/ger>;\n",
    "                ]\n",
    "            ];\n",
    "            mads:email <mailto:ttr@leibniz-psychology.org>; # correspondence author email\n",
    "        ];\n",
    "        bf:role <http://id.loc.gov/vocabulary/relators/aut>;\n",
    "        pxp:contributionPosition 1; bf:qualifier \"first\"; # first author in sequence: our own subproperty of bf:qualifier & schema:position (also: middle, last)\n",
    "    ].\n",
    "```\n",
    "\n",
    "Todos:\n",
    "- [x] create blank node for contribution and add agent of type bf:Person\n",
    "- [x] add author position (first, middle, last plus order number) to the contribution\n",
    "- [x] make first author a bflc:PrimaryContribution\n",
    "- [x] match AUP with PAUP to get person names and ids (normalize first)\n",
    "- [x] extend AUP-PAUP match with lookup in kerndaten table/ttl to compare schema:alternatename of person with name in AUP (but first before normalization)\n",
    "- [x] add ORCID to the person's blank node (doesn't add 4 ORCIDs for unknown reason - maybe duplicates?)\n",
    "- [x] add EMAIL to person's blank node (either to person in EMID or to first author)\n",
    "- [x] add affiliation from CS field and COU field to first author\n",
    "- [x] add Affiliation blank node with org name, country to each author that has these subfields in their AUP (|i and |c)\n",
    "- [x] add role from AUP subfield |f\n",
    "- [x] add country geonames id using lookup table\n",
    "- [ ] move mads:email Literal from bf:Contribution to mads:Affiliation\n",
    "- [ ] later: reconcile affiliations to add org id, org ror id (once we actually have institution authority files)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "from modules.mappings import geonames_countries\n",
    "\n",
    "def country_geonames_lookup(country):\n",
    "    for case in geonames_countries:\n",
    "        if case[0].casefold() == str(country).casefold():\n",
    "            return case[0], case[1]\n",
    "    return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sanitize_country_names(country_name):\n",
    "    if country_name == \"COSTA\":\n",
    "        country_name = \"Costa Rica\"\n",
    "    elif country_name == \"CZECH\":\n",
    "        country_name = \"Czech Republic\"\n",
    "    elif country_name == \"NEW\":\n",
    "        country_name = \"New Zealand\"\n",
    "    elif country_name == \"SAUDI\":\n",
    "        country_name = \"Saudi Arabia\"\n",
    "    elif country_name == \"PEOPLES\":\n",
    "        country_name = \"People's Republic of China\"\n",
    "    return country_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "\n",
    "\n",
    "def add_bf_contributor_person_role(role):\n",
    "    # # return role_uri,\n",
    "    # # generate a node of type bf:Role:\n",
    "    # records_bf.add((role, RDF.type, BF.Role))\n",
    "    # # construct the uri for the bf:Role object:\n",
    "    # role_uri = URIRef(ROLES + role)\n",
    "    # # add the uri to the role node:\n",
    "    # records_bf.add((role, RDF.value, role_uri))\n",
    "    # # return the new node:\n",
    "    # return role\n",
    "    return URIRef(ROLES + role)\n",
    "\n",
    "\n",
    "\n",
    "def normalize_names(familyname,givenname):\n",
    "    familyname_normalized = familyname.replace(\"ä\", \"ae\").replace(\"ö\", \"oe\").replace(\"ü\", \"ue\").replace(\"Ä\", \"Ae\").replace(\"Ö\", \"Oe\").replace(\"Ü\", \"Ue\").replace(\"ß\", \"ss\")\n",
    "    # generate an abbreviated version of givenname (only the first letter), but \n",
    "    if givenname:\n",
    "        givenname_abbreviated = givenname[0] + \".\"\n",
    "        # generate a normalized version of the name by concatenating the two with a comma as the separator:\n",
    "        fullname_normalized = familyname_normalized + \", \" + givenname_abbreviated\n",
    "    return fullname_normalized\n",
    "\n",
    "def match_paup(record, person_node, personname_normalized):\n",
    "    # loop through all PAUPs and check if the name matches the normalized personname\n",
    "    for paup in record.findall(\"PAUP\"):\n",
    "        # given a string such as \"Forkmann, Thomas |n p06946TF |u https://www.psychauthors.de/psychauthors/index.php?wahl=forschung&amp;#38;uwahl=psychauthors&amp;#38;uuwahl=p06946TF\"\n",
    "        # split into family name, given name and paId where \"Forkmann\" is the family name, \"Thomas\" is the given name and \"p06946TF\" is the paId:\n",
    "        paup_split = paup.text.strip().split(\"|n\")[0].strip().split(\",\")\n",
    "        if len(paup_split) > 1:\n",
    "            paup_familyname = paup_split[0].strip()\n",
    "            paup_givenname = paup_split[1].strip()\n",
    "            paId = paup.text.strip().split(\"|n\")[1].strip().split(\"|\")[0].strip()\n",
    "            \n",
    "            # generate a normalized version of paup_familyname:\n",
    "            paup_name_normalized = normalize_names(paup_familyname,paup_givenname)\n",
    "            # generate a uri for the person from the paId that can match the one in kerndaten.ttl generated from psychauthors database:\n",
    "            person_uri = URIRef(\"https://w3id.org/zpid/person/\" + paId)\n",
    "            \n",
    "            # now check if the normalized name from PAUP matches the normalized person name from AUP:\n",
    "            # if they match and there is a matching person in the kerndaten.ttl graph, add the person uri as schema:sameAs and the current preferred name from psychauthors as schema:preferredName, then return the paId:\n",
    "            if paup_name_normalized == personname_normalized and (person_uri, RDF.type, SCHEMA.Person) in kerndaten:\n",
    "                # for debugging, print the actual name in the matching PAUP:\n",
    "                #records_bf.add((person_node, PXP.paupName, Literal(paup_familyname + \", \" + paup_givenname)))\n",
    "                # records_bf.add((person_node, SCHEMA.sameAs, person_uri))\n",
    "                # add the preferred name from kerndaten as schema:preferredName:\n",
    "                #records_bf.add((person_node, SCHEMA.preferredName, kerndaten.value(person_uri, SCHEMA.name)))\n",
    "                # return the psychauthors ID:\n",
    "                return paId\n",
    "            # but if PAUP and AUP names are no match, even normalized,\n",
    "            # go through all the alternate names in kerndaten for that Psychauthors ID and check if they match the normalized person name from AUP (this will even find completely changed names, from maiden name to married name etc.):\n",
    "            elif paup_name_normalized != personname_normalized and (person_uri, RDF.type, SCHEMA.Person) in kerndaten:\n",
    "                for alternatename in kerndaten.objects(person_uri, SCHEMA.alternateName):\n",
    "                    # split alternatename into first and last name:\n",
    "                    alternatename_split = alternatename.split(\",\")\n",
    "                    if len(alternatename_split) > 1:\n",
    "                        alternatename_familyname = alternatename_split[0].strip()\n",
    "                        alternatename_givenname = alternatename_split[1].strip()\n",
    "                        # generate a normalized version of alternatename_familyname to compare with PAUP name later:\n",
    "                        alternatename_normalized = normalize_names(alternatename_familyname,alternatename_givenname)\n",
    "                        if personname_normalized == alternatename_normalized:\n",
    "                            # we have found another match!\n",
    "                            # add the uri as schema:sameAs and put the current preferred name from psychauthors here, too (for debugging purposes):\n",
    "                            # records_bf.add((person_node, SCHEMA.sameAs, person_uri))\n",
    "                            #records_bf.add((person_node, SCHEMA.preferredName, kerndaten.value(person_uri, SCHEMA.name)))\n",
    "                            return paId\n",
    "            else:\n",
    "                return None\n",
    "\n",
    "def get_orcid(record, person_node, personname):\n",
    "    # loop through all ORCIDs and check if the name matches the personname\n",
    "    for orcid in record.findall(\"ORCID\"):\n",
    "        # go through all ORCID fields and check for matches of personname with the text before \"|u\": \n",
    "        # split the orcid string into the orcid id and the name:\n",
    "        orcid_split = orcid.text.strip().split(\"|u\")\n",
    "        \n",
    "        # if there is a name part, compare it to the personname:\n",
    "        if len(orcid_split) > 1:\n",
    "            orcid_name = mappings.replace_encodings(orcid_split[0]).strip()\n",
    "            orcidId = orcid_split[1].strip()\n",
    "            # clean up the orcid_id by removing spaces that sometimes sneak in when entering them in the database:\n",
    "            orcidId = orcidId.replace(\" \", \"\")\n",
    "            \n",
    "            # by the way, here is a regex pattern for valid orcids:\n",
    "            orcid_pattern = re.compile(r'^\\d{4}-\\d{4}-\\d{4}-\\d{3}[0-9X]$')\n",
    "            # fixing any orcid ids that have known prefixes, but are recognizable, e.g. \"orcid.org/0000-0002-1694-233X\", \"/0000...\":\n",
    "            orcid_pattern = re.compile(r\"^(https?:\\/\\/(orcid\\.)?org\\/)?(orcid\\.org\\/)?(\\/)?([0-9]{4}-[0-9]{4}-[0-9]{4}-[0-9]{3}[0-9X])$\")\n",
    "            if orcid_pattern.match(orcidId):\n",
    "                # remove the prefixes and slashes from the orcid id:\n",
    "                orcidId = orcid_pattern.match(orcidId).group(5)\n",
    "            else:\n",
    "                # don't use it if it doesn't match the pattern:\n",
    "                print(f\"invalid orcid: {orcidId}, dropping.\")    \n",
    "                orcidId = None    \n",
    "            # if the name matches, return the orcid id for adding it to the person node:\n",
    "            if orcid_name == personname:\n",
    "                return orcidId\n",
    "            else:\n",
    "                #print(\"dangling orcid (no match): \" + orcidId)\n",
    "                return None\n",
    "            \n",
    "\n",
    "def get_local_authority_institute(affiliation_string, country):\n",
    "    \"\"\"Uses ~~fuzzywuzzy~~ RapidFuzz to look up the affilaition string in a local authority table loaded from a csv file.\"\"\"\n",
    "    if country == \"LUXEMBOURG\":\n",
    "        best_match = process.extractOne(affiliation_string, lux_institutes, scorer=fuzz.token_set_ratio)\n",
    "        return best_match[0].get(\"uuid\")\n",
    "    else: \n",
    "        return None\n",
    "\n",
    "def get_ror_id_from_api(affiliation_string):\n",
    "    # this function takes a string with an affiliation name and returns the ror id for that affiliation from the ror api\n",
    "    # clean the string to make sure things like \"^DDS\" are replaced:\n",
    "    #affiliation_string = my_xml_escape(affiliation_string)\n",
    "    #replace_encodings(affiliation_string)\n",
    "    ror_api_url = ROR_API_URL + affiliation_string\n",
    "    # make a request to the ror api:\n",
    "    # ror_api_request = requests.get(ror_api_url)\n",
    "    # make request to api with caching:\n",
    "    ror_api_request = session.get(\n",
    "            ror_api_url, timeout=20\n",
    "    )\n",
    "    # if the request was successful, get the json response:\n",
    "    if ror_api_request.status_code == 200:\n",
    "        ror_api_response = ror_api_request.json()\n",
    "        # check if the response has any hits:\n",
    "        if len(ror_api_response[\"items\"]) > 0:\n",
    "            # if so, get the item with a key value pair of \"chosen\" and \"true\" and return its id:\n",
    "            for item in ror_api_response[\"items\"]:\n",
    "                if item[\"chosen\"] == True:\n",
    "                    return item[\"organization\"][\"id\"]\n",
    "        else:\n",
    "            return None\n",
    "    else:\n",
    "        return None\n",
    "\n",
    "def build_affiliation_nodes(person_affiliation, person_affiliation_country):\n",
    "    # person_affiliation = replace_encodings(person_affiliation)\n",
    "    # is passed two string: the affiliation name and the affiliation country name\n",
    "    # make a blank node for the affiliation and make it class mads:Affiliation:\n",
    "    person_affiliation_node = BNode()\n",
    "    records_bf.add((person_affiliation_node, RDF.type, MADS.Affiliation))\n",
    "    # make a blank node for the affiliation organization and make it class bf:Organization:\n",
    "    person_affiliation_org_node = BNode()\n",
    "    records_bf.add((person_affiliation_org_node, RDF.type, BF.Organization))\n",
    "    # add the affiliation organization node to the affiliation node:\n",
    "    records_bf.add((person_affiliation_node, MADS.organization, person_affiliation_org_node))\n",
    "    # add the affiliation string to the affiliation org node:\n",
    "    records_bf.add((person_affiliation_org_node, RDFS.label, Literal(person_affiliation, lang=\"und\")))\n",
    "\n",
    "    # do a ror lookup for the affiliation string\n",
    "    # and if there is a ror id, add the ror id as an identifier:\n",
    "    affiliation_ror_id = None\n",
    "    affiliation_ror_id = get_ror_id_from_api(person_affiliation)\n",
    "    \n",
    " \n",
    "    if affiliation_ror_id is not None:\n",
    "        # add a blank node fore the identifier:\n",
    "        affiliation_ror_id_node = BNode()\n",
    "        # make it a locid:ror:\n",
    "        records_bf.add((affiliation_ror_id_node, RDF.type, LOCID.ror))\n",
    "        records_bf.add((person_affiliation_org_node, BF.identifiedBy, affiliation_ror_id_node))\n",
    "        # add the ror id as a literal to the identifier node:\n",
    "        records_bf.add((affiliation_ror_id_node, RDF.value, Literal(affiliation_ror_id)))\n",
    "\n",
    "    affiliation_local_id = None\n",
    "    affiliation_local_id = get_local_authority_institute(person_affiliation, person_affiliation_country)\n",
    "    if affiliation_local_id is not None:\n",
    "        # add a blank node fore the identifier:\n",
    "        affiliation_local_id_node = BNode()\n",
    "        # make it a pxc:OrgID:\n",
    "        records_bf.add((affiliation_local_id_node, RDF.type, PXC.OrgID))\n",
    "        records_bf.add((person_affiliation_org_node, BF.identifiedBy, affiliation_local_id_node))\n",
    "        # add the local uuid as a literal to the identifier node:\n",
    "        records_bf.add((affiliation_local_id_node, RDF.value, Literal(affiliation_local_id)))\n",
    "\n",
    "    # make a blank node for the affiliation address and make it class mads:Address:\n",
    "    person_affiliation_address_node = BNode()\n",
    "    records_bf.add((person_affiliation_address_node, RDF.type, MADS.Address))\n",
    "    # add a country node to the affiliation address node:\n",
    "    person_affiliation_country_node = BNode()\n",
    "    records_bf.add((person_affiliation_country_node, RDF.type, MADS.Country))\n",
    "    # add the country node to the affiliation address node:\n",
    "    records_bf.add((person_affiliation_address_node, MADS.country, person_affiliation_country_node))\n",
    "    # add the affiliation address string to the affiliation address node:\n",
    "    records_bf.add((person_affiliation_country_node, RDFS.label, Literal(person_affiliation_country, lang=\"en\")))\n",
    "\n",
    "    # if the country is in the geonames lookup table, add the geonames uri as sameAs and the geonames id as an identifier:\n",
    "    if country_geonames_lookup(person_affiliation_country):\n",
    "        improved_country_name, geonamesId = country_geonames_lookup(person_affiliation_country)\n",
    "        # create a url to click and add it with sameas:\n",
    "        # geonames_uri = URIRef(\"http://geonames.org/\" + geonamesId + \"/\")\n",
    "        # records_bf.add((person_affiliation_country_node, SCHEMA.sameAs, geonames_uri))\n",
    "        # replace the country name in the affiliation address node with the improved country name:\n",
    "        records_bf.add((person_affiliation_country_node, RDFS.label, Literal(improved_country_name, lang=\"en\")))\n",
    "        # and remove the old label:\n",
    "        records_bf.remove((person_affiliation_country_node, RDFS.label, Literal(person_affiliation_country, lang=\"en\")))\n",
    "        # add the geonames identifier:\n",
    "        person_affiliation_country_identifier_node = BNode()\n",
    "        # records_bf.add((person_affiliation_country_identifier_node, RDF.type, BF.Identifier))\n",
    "        records_bf.add((person_affiliation_country_identifier_node, RDF.type, LOCID.geonames))\n",
    "        records_bf.add((person_affiliation_country_identifier_node, RDF.value, Literal(geonamesId)))\n",
    "        records_bf.add((person_affiliation_country_node, BF.identifiedBy, person_affiliation_country_identifier_node))\n",
    "    # add the affiliation address node to the affiliation node:\n",
    "    records_bf.add((person_affiliation_node, MADS.hasAffiliationAddress, person_affiliation_address_node))\n",
    "\n",
    "    # return the finished affiliation node with all its children and attached strings:\n",
    "    return person_affiliation_node\n",
    "\n",
    "# the full function that creates a contribution node for each person in AUP:\n",
    "# first, get all AUPs in a record and create a blank node for each of them\n",
    "def add_bf_contributor_person(work_uri, record):\n",
    "    # initialize a counter for the contribution position and a variable for the contribution qualifier:\n",
    "    contribution_counter = 0\n",
    "    contribution_qualifier = None\n",
    "    \n",
    "    for person in record.findall(\"AUP\"):\n",
    "        # count how often we've gone through the loop to see the author position:\n",
    "        contribution_counter += 1\n",
    "        # make a fragement uri node for the bf:Contribution (previously a bnode!):\n",
    "        contribution_node = URIRef(work_uri + \"#contribution\" + str(contribution_counter))\n",
    "        # contribution_node = URIRef(work_uri + \"/contribution/\" + str(contribution_counter))\n",
    "        records_bf.add((contribution_node, RDF.type, BF.Contribution))\n",
    "        \n",
    "        # make a fragment uri node for the person:\n",
    "        person_node = BNode()\n",
    "        records_bf.add((person_node, RDF.type, BF.Person))\n",
    "\n",
    "        # add the counter as an author position to the contribution node:\n",
    "        records_bf.add((contribution_node, PXP.contributionPosition, Literal(contribution_counter)))\n",
    "\n",
    "        # if we are in the first loop, set \"contrution_qualifier\" to \"first\":\n",
    "        if contribution_counter == 1:\n",
    "            contribution_qualifier = \"first\"\n",
    "            records_bf.add((contribution_node, RDF.type, BFLC.PrimaryContribution))\n",
    "        # if we are in the last loop, set \"contribution_qualifier\" to \"last\":\n",
    "        elif contribution_counter == len(record.findall(\"AUP\")):\n",
    "            contribution_qualifier = \"last\"\n",
    "        # if we are in any other loop but the first or last, set \"contribution_qualifier\" to \"middle\":\n",
    "        else:\n",
    "            contribution_qualifier = \"middle\"\n",
    "\n",
    "        # add the contribution qualifier to the contribution node:\n",
    "        records_bf.add((contribution_node, BF.qualifier, Literal(contribution_qualifier)))\n",
    "\n",
    "        # add the name from AUP to the person node, but only use the text before the first |: (and clean up the encoding):\n",
    "        personname = mappings.replace_encodings(person.text.split(\"|\")[0]).strip()\n",
    "\n",
    "        records_bf.add((person_node, RDFS.label, Literal(personname, lang=\"und\")))        \n",
    "\n",
    "        # initialize variables for later use:\n",
    "        personname_normalized = None\n",
    "        orcidId = None\n",
    "\n",
    "        # split personname into first and last name:\n",
    "        personname_split = personname.split(\",\")\n",
    "        if len(personname_split) > 1:\n",
    "            familyname = personname_split[0].strip()\n",
    "            givenname = personname_split[1].strip()\n",
    "            records_bf.add((person_node, SCHEMA.familyName, Literal(familyname)))\n",
    "            records_bf.add((person_node, SCHEMA.givenName, Literal(givenname)))\n",
    "            # generate a normalized version of familyname to compare with PAUP name later:\n",
    "            personname_normalized = normalize_names(familyname,givenname)\n",
    "            # for debugging, print the normalized name:\n",
    "            # records_bf.add((person_node, PXP.normalizedName, Literal(personname_normalized)))\n",
    "\n",
    "        # call the function match_paup to match the personname from AUP with the PAUPs:\n",
    "        paId = match_paup(record, person_node, personname_normalized)\n",
    "        if paId is not None:\n",
    "            # create a blank node for the identifier:\n",
    "            # we coulkd do this into the function, but then I will have to return something else\n",
    "            psychauthors_identifier_node = BNode()\n",
    "            #records_bf.add((psychauthors_identifier_node, RDF.type, BF.Identifier))\n",
    "            #records_bf.add((psychauthors_identifier_node, RDF.type, BF.Local))\n",
    "            records_bf.add((psychauthors_identifier_node, RDF.type, PXC.PsychAuthorsID))\n",
    "            records_bf.add((psychauthors_identifier_node, RDF.value, Literal(paId)))\n",
    "            # add the identifier node to the person node:\n",
    "            records_bf.add((person_node, BF.identifiedBy, psychauthors_identifier_node))\n",
    "            # create a urL from the paid and add it as a \"webpage describing this entity\" to the person node:\n",
    "            #psychauthors_url = \"https://www.psychauthors.de/psychauthors/index.php?wahl=forschung&uwahl=psychauthors&uuwahl=\" + paId\n",
    "            #records_bf.add((person_node, SCHEMA.mainEntityOfPage, URIRef(psychauthors_url)))\n",
    "        \n",
    "        # call the function get_orcid to match the personname with the ORCIDs in the record:\n",
    "        orcidId = get_orcid(record, person_node, personname)\n",
    "        if orcidId is not None:\n",
    "            # create a blank node for the identifier:\n",
    "            orcid_identifier_node = BNode()\n",
    "            # records_bf.add((orcid_identifier_node, RDF.type, BF.Identifier))\n",
    "            records_bf.add((orcid_identifier_node, RDF.type, LOCID.orcid))\n",
    "            records_bf.add((orcid_identifier_node, RDF.value, Literal(orcidId)))\n",
    "            # add the identifier node to the person node:\n",
    "            records_bf.add((person_node, BF.identifiedBy, orcid_identifier_node))\n",
    "            # add the orcid id as a sameAs link to the person node:\n",
    "            #orcid_uri = \"https://orcid.org/\" + orcidId\n",
    "            #records_bf.add((person_node, SCHEMA.sameAs, URIRef(orcid_uri)))\n",
    "\n",
    "        ## ----- \n",
    "        # Getting Affiliations and their countries from first, CS and COU (only for first author), and then from subfields |i and |c in AUP (for newer records)\n",
    "        ## -----\n",
    "        \n",
    "        # initialize variables we'll need for adding affiliations and country names from AUP |i and CS/COU/ADR:\n",
    "        affiliation_string = None\n",
    "        affiliation_country = None\n",
    "      \n",
    "        # match affiliations in CS and COU to first contribution/author:\n",
    "        # dont add ADR here yet (even if this is the place for it - we may drop that info anyway.\n",
    "        # look for the field CS:\n",
    "        # if the contribution_counter is 1 (i.e. if this is the first loop/first author), add the affiliation to the person node:\n",
    "        if contribution_counter == 1:\n",
    "            if record.find(\"CS\") is not None:\n",
    "                # get the content of the CS field:\n",
    "                affiliation_string = html.unescape(mappings.replace_encodings(record.find(\"CS\").text.strip()))\n",
    "\n",
    "            if record.find(\"COU\") is not None:\n",
    "                # get the country from the COU field:\n",
    "                affiliation_country = mappings.replace_encodings(sanitize_country_names(record.find(\"COU\").text.strip()))\n",
    "\n",
    "                \n",
    "        ## Get affiliation from AUP |i, country from |c:\n",
    "        # no looping necessary here, just check if a string |i exists in AUP and if so, add it to the person node:\n",
    "        # if AUP contains \"|i \", use anything after it and before the end of the string or before another \"|\" as the affiliation string:\n",
    "        if person.text.find(\"|i \") > -1:\n",
    "            # save that text in a variable:\n",
    "            affiliation_string = html.unescape(mappings.replace_encodings(person.text.split(\"|i\")[1].split(\"|\")[0]).strip())\n",
    "            # affiliation_string = replace_encodings(affiliation_string)\n",
    "            \n",
    "        # now check if there is a country in |c:\n",
    "        if person.text.strip().find(\"|c \") > -1:\n",
    "            # save that text in a variable:\n",
    "            affiliation_country = mappings.replace_encodings(sanitize_country_names(person.text.strip().split(\"|c\")[1].strip().split(\"|\")[0].strip()))\n",
    "\n",
    "        # pass this to function build_affiliation_nodes to get a finished affiliation node:\n",
    "        if affiliation_string != \"\" and affiliation_string is not None:\n",
    "            affiliation_node = build_affiliation_nodes(affiliation_string, affiliation_country)\n",
    "            # add the affiliation node to the contribution node:\n",
    "            records_bf.add((contribution_node, MADS.hasAffiliation, affiliation_node))\n",
    "\n",
    "        # look for the field EMAIL:\n",
    "        email = None\n",
    "        # TODO: the email address actually belongs into the affiliation section, but we'll leave it directly in the contribution node for now:\n",
    "        if record.find(\"EMAIL\") is not None:\n",
    "            # get the email address from the EMAIL field, replacing spaces with underscores (common problem in urls in star) and adding a \"mailto:\" prefix:\n",
    "            email = html.unescape(mappings.replace_encodings(record.find(\"EMAIL\").text.strip().replace(\" \", \"_\")))\n",
    "            # check if this is a valid email address:\n",
    "            email_pattern = re.compile(r'^([^\\x00-\\x20\\x22\\x28\\x29\\x2c\\x2e\\x3a-\\x3c\\x3e\\x40\\x5b-\\x5d\\x7f-\\xff]+|\\x22([^\\x0d\\x22\\x5c\\x80-\\xff]|\\x5c[\\x00-\\x7f])*\\x22)(\\x2e([^\\x00-\\x20\\x22\\x28\\x29\\x2c\\x2e\\x3a-\\x3c\\x3e\\x40\\x5b-\\x5d\\x7f-\\xff]+|\\x22([^\\x0d\\x22\\x5c\\x80-\\xff]|\\x5c[\\x00-\\x7f])*\\x22))*\\x40([^\\x00-\\x20\\x22\\x28\\x29\\x2c\\x2e\\x3a-\\x3c\\x3e\\x40\\x5b-\\x5d\\x7f-\\xff]+|\\x5b([^\\x0d\\x5b-\\x5d\\x80-\\xff]|\\x5c[\\x00-\\x7f])*\\x5d)(\\x2e([^\\x00-\\x20\\x22\\x28\\x29\\x2c\\x2e\\x3a-\\x3c\\x3e\\x40\\x5b-\\x5d\\x7f-\\xff]+|\\x5b([^\\x0d\\x5b-\\x5d\\x80-\\xff]|\\x5c[\\x00-\\x7f])*\\x5d))*$')\n",
    "            # check if email matches the regex in email_pattern:\n",
    "            if not email_pattern.match(email):\n",
    "                print(\"invalid email address: \" + email)\n",
    "                # email = None\n",
    "            email = \"mailto:\" + email\n",
    "            # email = \"mailto:\" + record.find(\"EMAIL\").text.strip()\n",
    "            # if there is no EMID and the contribution_counter is 1 (i.e. if this is the first loop), add the email to the person node:\n",
    "            if record.find(\"EMID\") is None and contribution_counter == 1:\n",
    "                records_bf.add((contribution_node, MADS.email, URIRef(email)))\n",
    "            # else match the existing EMID field to the personname:\n",
    "            elif record.find(\"EMID\") is not None and mappings.replace_encodings(record.find(\"EMID\").text.strip()) == personname:\n",
    "                records_bf.add((contribution_node, MADS.email, URIRef(email))) \n",
    "                \n",
    "            \n",
    "        role = None\n",
    "        # check if there is a role in the AUP field:\n",
    "        if person.text.strip().find(\"|f \") > -1:\n",
    "            # save that text in a variable:\n",
    "            role = person.text.strip().split(\"|f\")[1].strip().split(\"|\")[0].strip()\n",
    "            # add it to the contribution node:\n",
    "            records_bf.add((contribution_node, BF.role, add_bf_contributor_person_role(role)))\n",
    "        # if there isn't, the role is \"AU\" by default:\n",
    "        else:\n",
    "            records_bf.add((contribution_node, BF.role, add_bf_contributor_person_role(\"AU\")))\n",
    "\n",
    "        ## --- Add the contribution node to the work node:\n",
    "        records_bf.add((work_uri, BF.contribution, contribution_node))\n",
    "        # add the person node to the contribution node as a contributor:\n",
    "        records_bf.add((contribution_node, BF.agent, person_node))    "
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Function: Create Topics, Weighted Topics and Classifications from CT, SH\n",
    "\n",
    "CTs - controlled terms from our own thesaurus - are added like this. Some are always weighted (=more important) and have an added class pxc:WeightedTopic (a subclass of bf:Topic), some are not. \n",
    "\n",
    "Currently, we don't export the concept uri, since the export files we use have no code info (which we use to construct concept uris). But at least we have a source uri for the concept scheme (which is the same for all concepts in the file).\n",
    "\n",
    "\n",
    "```turtle\n",
    "<Work> a bf:Work;\n",
    "    bf:subject [a bf:Topic, pxc:WeightedTopic;  # topic, weighted\n",
    "        # a skos:Concept;\n",
    "        # owl:sameAs <https://w3id.org/zpid/vocabs/terms/35365>;\n",
    "        rdfs:label \"Ontologies\"@en, \"Ontologien\"@de;\n",
    "        bf:source <https://w3id.org/zpid/vocabs/terms>;\n",
    "    ];\n",
    "    bf:subject [a bf:Topic;  # a non-weighted topic\n",
    "        # a skos:Concept; \n",
    "        # owl:sameAs <https://w3id.org/zpid/vocabs/terms/60135>;\n",
    "        rdfs:label \"Semantic Networks\"@en, \"Semantische Netzwerke\"@de;\n",
    "        bf:source <https://w3id.org/zpid/vocabs/terms>;\n",
    "    ];\n",
    "    # PSYNDEX subject heading classification - not done yet\n",
    "    bf:classification [ a bf:Classification, pxc:SubjectHeading, skos:Concept;\n",
    "        rdfs:label \"Professional Psychological & Health Personnel Issues\"@en;\n",
    "        bf:code \"3400\";\n",
    "        owl:sameAs <https://w3id.org/zpid/vocabs/class/3400>;\n",
    "        bf:source <https://w3id.org/zpid/vocabs/class>;\n",
    "    ].\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_controlled_terms(work_uri, record):\n",
    "    # get the controlled terms from the field CT:\n",
    "    for data in record.findall(\"CT\"):\n",
    "        # get the content of the CT field:\n",
    "        controlled_term_string = mappings.replace_encodings(data.text.strip())\n",
    "        # we need to split the string into these possible parts: \n",
    "        # \"|e English Label\" for the English label, \n",
    "        # \"|d Deutsches Label\" for the German label, and\n",
    "        # \"|g x\", which, if it exists, indicates that this Topic is also weighted and should get an additional class (beside bf:Topic) of either \"PXC.WeightedTopic\"\n",
    "        # we will use the get_Subfield function, which takes the subfield name as the parameter and returns the content of that subfield if it exists, or None if it doesn't.        \n",
    "        # initialize variables for the controlled term string parts:\n",
    "        controlled_term_string_english = get_subfield(controlled_term_string, \"e\")\n",
    "        controlled_term_string_german = get_subfield(controlled_term_string, \"d\")\n",
    "        term_weighting = get_subfield(controlled_term_string, \"g\")\n",
    "        if term_weighting is not None and term_weighting == \"x\":\n",
    "            controlled_term_weighted = True\n",
    "        else:\n",
    "            controlled_term_weighted = False\n",
    "        # create a blank node for the controlled term and make it a class bf:Topic:\n",
    "        controlled_term_node = BNode()\n",
    "        # a ct node is a bf:Topic (and a skos:Concept - albeit without a uri identifier, because the CT \"code\" is not exported, only the labels); sometimes it is also a pxc:WeightedTopic:\n",
    "        records_bf.add((controlled_term_node, RDF.type, BF.Topic))\n",
    "        # records_bf.add((controlled_term_node, RDF.type, SKOS.Concept))\n",
    "        # add source to the controlled term node (bf:source <https://w3id.org/zpid/vocabs/terms>):\n",
    "        records_bf.add((controlled_term_node, BF.source, URIRef(\"https://w3id.org/zpid/vocabs/terms\")))\n",
    "        if controlled_term_weighted:\n",
    "            records_bf.add((controlled_term_node, RDF.type, PXC.WeightedTopic))\n",
    "        # add the controlled term string to the controlled term node:\n",
    "        records_bf.add((controlled_term_node, RDFS.label, Literal(controlled_term_string_english, lang=\"en\")))\n",
    "        records_bf.add((controlled_term_node, RDFS.label, Literal(controlled_term_string_german, lang=\"de\")))\n",
    "        # attach the controlled term node to the work node:\n",
    "        records_bf.add((work_uri, BF.subject, controlled_term_node))\n",
    "        \n",
    "    "
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## TODO: Function: Create nodes for Population Age Group (AGE) and Population Location (PLOC)\n",
    "\n",
    "Use this scheme:\n",
    "\n",
    "```turtle\n",
    "<Work> \n",
    "# age group study is about/sample was from:\n",
    "    bflc:demographicGroup [a bflc:DemographicGroup, pxc:AgeGroup, skos:Concept;\n",
    "        rdfs:label \"Adulthood\"@en, \"Erwachsenenalter\"@de;\n",
    "        owl:sameAs <https://w3id.org/zpid/vocabs/age/adulthood>;\n",
    "        bf:source <https://w3id.org/zpid/vocabs/age/AgeGroups>; \n",
    "    ];\n",
    "    # population location: \n",
    "    bf:geographicCoverage [a bf:GeographicCoverage, pxc:PopulationLocation, skos:Concept;\n",
    "        rdfs:label \"Germany\"@en;\n",
    "        owl:sameAs <countries/ger>;\n",
    "    ].\n",
    "```"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Function: Create nodes for PRREG (linked Preregistration Works)\n",
    "\n",
    "Field PRREG can occur multiple times per record (0..n). \n",
    "It contains a link and/or DOI to a preregistration document. \n",
    "\n",
    "Possible subfields:\n",
    "- |u URL linking to the document\n",
    "- |d DOI for the document\n",
    "- |i additional Info text\n",
    "\n",
    "There are many errors we could catch here. \n",
    "- [x] Most importantly, we can replace any \" \" with \"_\" in the |u.\n",
    "- [x] Also, |d should contain pure DOIs with prefixes, so they should start with \"10.\" If they don't, remove any prefixes to make a \"pure\" DOI.\n",
    "- [x] remove or ignore any empty subfields that may exist (|u, |d, |i)\n",
    "\n",
    "Example:\n",
    "\n",
    "```turtle\n",
    "<https://w3id.org/zpid/pub/work/0003> a bf:Work; \n",
    "    bflc:relationship \n",
    "    [\n",
    "        a bflc:Relationship;\n",
    "        bflc:relation relations:hasPreregistration;\n",
    "        bf:note [a bf:Note; rdfs:label \"Australian Sample\"];\n",
    "        bf:supplement # may change, not sure?\n",
    "        [\n",
    "            a bf:Work, bf:Text; \n",
    "            bf:genreForm genres:preregistration; \n",
    "            bf:content content:text;\n",
    "            bf:hasInstance \n",
    "            [\n",
    "                a bf:Instance;\n",
    "                bf:electronicLocator <https://osf.io/prereg1>;\n",
    "                bf:identifier [a bf:Identifier, bf:Doi; rdf:value \"10.123code003\"];\n",
    "                # add bf:media \"computer\" from rda media types\n",
    "                bf:media <http://rdvocab.info/termList/RDAMediaType/1003>;\n",
    "                # bf:carrier \"online resource\" from rda vocabulary\n",
    "                bf:carrier <http://rdvocab.info/termList/RDACarrierType/1018>;\n",
    "            ]\n",
    "        ] \n",
    "    ]\n",
    ".\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "# function to build the nodes for preregistration links\n",
    "from distutils.command import build\n",
    "\n",
    "\n",
    "def get_bf_preregistrations(work_uri, record):\n",
    "    # get the preregistration link from the field PREREG:\n",
    "    preregistration_note = None\n",
    "    unknown_field_content = None\n",
    "    for prreg in record.findall(\"PRREG\"):\n",
    "        # get the full content of the field, sanitize it:\n",
    "        prregfield = html.unescape(mappings.replace_encodings(prreg.text.strip()))\n",
    "        # use our node-building function to build the node:\n",
    "        relationship_node, instance = build_work_relationship_node(work_uri,relation_type=\"preregistration\")\n",
    "        doi_set = set()\n",
    "        url_set = set()\n",
    "        for subfield_name in (\"u\", \"d\"):\n",
    "            try: \n",
    "                subfield = get_subfield(prregfield, subfield_name)\n",
    "            except:\n",
    "                subfield = None\n",
    "            else:\n",
    "                # print(subfield)\n",
    "                # if the string_type returned [1] is doi or url, treat them accordingly, using the returned string [0]\n",
    "                # as a doi or url:\n",
    "                # if it is a doi, run a function to generate a doi identifier node\n",
    "                if subfield is not None and check_for_url_or_doi(subfield)[1] == \"doi\":\n",
    "                    # add the doi to a list:\n",
    "                    doi_set.add(check_for_url_or_doi(subfield)[0])\n",
    "                    #build_doi_identifier_node(instance, check_for_url_or_doi(subfield)[0])\n",
    "                elif subfield is not None and check_for_url_or_doi(subfield)[1] == \"url\":\n",
    "                    url_set.add(check_for_url_or_doi(subfield)[0])\n",
    "                    #build_electronic_locator_node(instance, check_for_url_or_doi(subfield)[0])\n",
    "                    # if the returned typ is something else - \"unknown\", do nothing with it:\n",
    "                else:\n",
    "                    # print(\"bf:note > bf:Note > rdfs:label: \" + subfield)\n",
    "                    # build_note_node(instance, check_for_url_or_doi(subfield)[0])\n",
    "                    if subfield is not None and check_for_url_or_doi(subfield)[0] is not None and check_for_url_or_doi(subfield)[0] != \"\":\n",
    "                        # add a variable \n",
    "                        unknown_field_content = check_for_url_or_doi(subfield)[0].strip()\n",
    "                        print(f\"unknown type: {unknown_field_content}. Adding as a note.\")\n",
    "                        # add the string as a note to the instance:\n",
    "                        # build_note_node(instance, check_for_url_or_doi(subfield)[0])\n",
    "\n",
    "        # compare all the dois in the doi_set to each item in the url set. If the url contains the doi, remove it from the url set:\n",
    "        for doi in doi_set:\n",
    "            for url in url_set.copy():\n",
    "                if doi in url:\n",
    "                    url_set.remove(url)\n",
    "                    # special case: if url is osf and contains the same short code as the osf doi, also remove the url. Example: https://osf.io/kcb4d/ and 10.17605/OSF.IO/KCB4D\n",
    "                elif \"osf.io\" in url and \"OSF.IO/\" in doi and doi.split(\"/\")[2].lower() in url:\n",
    "                    print(f\"duplicate doi in url {url}: {doi.split('/')[2]} from {doi}. Removing url in favor of doi.\")\n",
    "                    url_set.remove(url)\n",
    "        # now build the doi identifier nodes for any DOIs in the set we collected:\n",
    "        for doi in doi_set:\n",
    "            build_doi_identifier_node(instance, doi)\n",
    "        for url in url_set:\n",
    "            build_electronic_locator_node(instance, url)\n",
    "        # for the text in the |i subfield, build a note without further processing:\n",
    "        try:\n",
    "            preregistration_note = get_subfield(prregfield, \"i\")\n",
    "        except:\n",
    "            preregistration_note = None\n",
    "        else:\n",
    "            # add anything in the |i subfield as a note to the instance:\n",
    "            # but if we found something unrecognizable in |u or |i, also add it to the note:\n",
    "            if unknown_field_content is not None:\n",
    "                build_note_node(instance, preregistration_note+ \". \" + unknown_field_content )\n",
    "            else:\n",
    "                build_note_node(instance, preregistration_note)\n",
    "        # now attach the finished node for the relationship to the work:\n",
    "        records_bf.add((work_uri, BFLC.relationship, relationship_node))\n",
    "      \n",
    "\n",
    "\n",
    "        # add preregistration_node to work:\n",
    "        records_bf.add((work_uri, BFLC.relationship, relationship_node))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Function: Create nodes for Grants (GRANT)\n",
    "\n",
    "Includes several helper functions that \n",
    "- extract grant numbers if several were listed in the |n subfield\n",
    "- replace funder names that fundref usually doesn't match correctly or at all\n",
    "- look up funder names in crossref's fundref api to get their fundref id (a doi)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_grant_numbers(subfield_n_string):\n",
    "    # this function takes a string and returns a list of award numbers\n",
    "    # first, split the string on \",\" or \";\" or \"and\": (first replacing all semicolons and \"ands\" with commas)\")\n",
    "    subfield_n_string = subfield_n_string.replace(\" and \", \", \")\n",
    "    subfield_n_string = subfield_n_string.replace(\" und \", \", \")\n",
    "    subfield_n_string = subfield_n_string.replace(\" & \", \", \")\n",
    "    subfield_n_string = subfield_n_string.replace(\";\", \",\")\n",
    "    subfield_n_string = subfield_n_string.split(\", \")\n",
    "    # in each of the returned list elements, remove any substrings that are shorter \n",
    "    # than 5 characters (to get rid of things like \" for\" or \"KDL: \" YG: \" etc.)\n",
    "    # for element in subfield_n_string:\n",
    "    #     if len(element) < 5:\n",
    "    #         subfield_n_string.remove(element)\n",
    "    # go through all the list elements and replace each with a dict,\n",
    "    # which has a key \"grant_number\" and a key \"grant_name\" (which is None for now):\n",
    "    # for i, element in enumerate(subfield_n_string):\n",
    "    #     subfield_n_string[i] = {\"grant_number\": element, \"grant_name\": None}\n",
    "    # # return the list of dicts:\n",
    "    return subfield_n_string\n",
    "\n",
    "def replace_common_fundernames(funder_name):\n",
    "    \"\"\"This will accept a funder name that crossref api may not recognize, at least not as the first hit,\n",
    "    and replace it with a string that will supply the right funder as the first hit\"\"\"\n",
    "    # if the funder_name is in the list of funder names to replace (in index 0), then replace it with what is in index 1:\n",
    "    for funder in mappings.funder_names_replacelist:\n",
    "        if funder_name == funder[0]:\n",
    "            funder_name = funder[1]\n",
    "            # print(\"replacing \" + funder[0] + \" with \" + funder[1])\n",
    "    return funder_name\n",
    "\n",
    "def get_crossref_funder_id(funder_name):\n",
    "    # this function takes a funder name and returns the crossref funder id for that funder name\n",
    "    # to do this, use the crossref api.\n",
    "    funder_name = replace_common_fundernames(funder_name)\n",
    "    # construct the api url:\n",
    "    crossref_api_url = CROSSREF_API_URL + funder_name + CROSSREF_FRIENDLY_MAIL\n",
    "    # make a request to the crossref api:\n",
    "    # crossref_api_request = requests.get(crossref_api_url)\n",
    "    # make request to api:\n",
    "    crossref_api_request = session_fundref.get(\n",
    "            crossref_api_url, timeout=20\n",
    "    )\n",
    "    crossref_api_response = crossref_api_request.json()\n",
    "    # result_count = int(crossref_api_response[\"message\"][\"total-results\"])\n",
    "    # if the request was successful, get the json response:\n",
    "    \n",
    "    if crossref_api_request.status_code == 200 and \\\n",
    "    crossref_api_response[\"message\"][\"total-results\"] >0: \n",
    "        # return the number of results:\n",
    "        #print(\"Treffer: \" + str(crossref_api_response[\"message\"][\"total-results\"]))\n",
    "        # return the first hit:\n",
    "        # print(\"Erster Treffer: \" + crossref_api_response[\"message\"][\"items\"][0][\"name\"])\n",
    "        # print(\"DOI: \" + \"10.13039\" + crossref_api_response[\"message\"][\"items\"][0][\"id\"])\n",
    "       return \"10.13039/\" + crossref_api_response[\"message\"][\"items\"][0][\"id\"]\n",
    "    else:\n",
    "        # retry the funder_name, but remove any words after the first comma:\n",
    "        if funder_name.find(\",\") > -1:\n",
    "            funder_name = funder_name.split(\",\")[0]\n",
    "            return get_crossref_funder_id(funder_name)\n",
    "        else:\n",
    "            return None\n",
    "    \n",
    "\n",
    "# function to build the nodes for preregistration links\n",
    "def get_bf_grants(work_uri, record):\n",
    "    \"\"\"this function takes a string and returns a funder (name and fundref doi), a list of grant number, a note with grant holder and info\"\"\"\n",
    "    for grant in record.findall(\"GRANT\"):\n",
    "        # point zero: remove html entities from the field:\n",
    "        grantfield = html.unescape(grant.text)\n",
    "        # if the field contains these, skip it - don't even create a fundinfregerence node:\n",
    "        if \"projekt deal\" in grantfield.lower() or \"open access\" in grantfield.lower():\n",
    "            continue\n",
    "    # point one: pipe all text in the field through the DD-Code replacer function:\n",
    "        grantfield = mappings.replace_encodings(grantfield)\n",
    "        # add a blank node for a new Contribution:\n",
    "        funding_contribution_node = BNode()\n",
    "        # records_bf.add((funding_contribution_node, RDF.type, BF.Contribution))\n",
    "        records_bf.add((funding_contribution_node, RDF.type, PXC.FundingReference))\n",
    "        # add a blank node for the funder agent:\n",
    "        funder_node = BNode()\n",
    "        records_bf.add((funder_node, RDF.type, BF.Agent))\n",
    "        # add the funder agent node to the funding contribution node:\n",
    "        records_bf.add((funding_contribution_node, BF.agent, funder_node))\n",
    "        # add a role to the funding contribution node:\n",
    "        records_bf.add((funding_contribution_node, BF.role, URIRef(\"http://id.loc.gov/vocabulary/relators/spn\")))\n",
    "        \n",
    "    # first, use anything before the first \"|\" as the funder:\n",
    "        # but because the database is still messy, use a default funder name in case there\n",
    "        # is no name in the field:\n",
    "        funder_name = \"FUNDERNAME NOT FOUND\"\n",
    "        # funder = {\"funder_name\": grantfield.split(\"|\")[0].strip(), \"funder_id\": None}\n",
    "        funder_name = grantfield.split(\"|\")[0].strip()\n",
    "        # add the funder name to the funder node:\n",
    "        records_bf.add((funder_node, RDFS.label, Literal(funder_name)))\n",
    "        # try to look up this funder name in the crossref funder registry:\n",
    "        # if there is a match, add the crossref funder id as an identifier:\n",
    "        crossref_funder_id = None\n",
    "        crossref_funder_id = get_crossref_funder_id(funder_name)\n",
    "        if crossref_funder_id is not None:\n",
    "            # add a blank node for the identifier:\n",
    "            crossref_funder_id_node = BNode()\n",
    "            # use our custim identifier class FundRefDoi (subclass of bf:Doi):\n",
    "            records_bf.add((crossref_funder_id_node, RDF.type, PXC.FundRefDoi))\n",
    "            records_bf.add((funder_node, BF.identifiedBy, crossref_funder_id_node))\n",
    "            # add the crossref funder id as a literal to the identifier node:\n",
    "            records_bf.add((crossref_funder_id_node, RDF.value, Literal(crossref_funder_id)))\n",
    "\n",
    "        # then check the rest for a grant number:\n",
    "        try:\n",
    "        # if \"|n \" in grantfield:\n",
    "            grants = grantfield.split(\"|n \")[1].split(\" |\")[0]\n",
    "        except:\n",
    "            grants = None\n",
    "        else:\n",
    "            grants = extract_grant_numbers(grants)\n",
    "            # add the grant number to the funding contribution node:\n",
    "            for grant_id in grants:\n",
    "                # add a blank node for the grant (class pxc:Grant via pxp:grant)\n",
    "                grant_node = BNode()\n",
    "                records_bf.add((grant_node, RDF.type, PXC.Grant))\n",
    "                # add the grant node to the funding contribution node:\n",
    "                records_bf.add((funding_contribution_node, PXP.grant, grant_node))\n",
    "\n",
    "                # add a blank node for the identifier:\n",
    "                grant_identifier_node = BNode()\n",
    "                # records_bf.add((grant_identifier_node, RDF.type, BF.Identifier))\n",
    "                records_bf.add((grant_identifier_node, RDF.type, PXC.GrantId))\n",
    "                records_bf.add((grant_identifier_node, RDF.value, Literal(grant_id.strip())))\n",
    "                # add the identifier node to the grant node:\n",
    "                records_bf.add((grant_node, BF.identifiedBy, grant_identifier_node))\n",
    "        # then check the rest for a grant name or other info/note:\n",
    "        try:\n",
    "        # if \"|i \" in grantfield:\n",
    "            funding_info = grantfield.split(\"|i \")[1].split(\" |\")[0]\n",
    "        except:\n",
    "            funding_info = None\n",
    "\n",
    "        try:\n",
    "        # if \"|e \" in grantfield:\n",
    "            funding_recipient = grantfield.split(\"|e \")[1].split(\" |\")[0]\n",
    "        except:\n",
    "            funding_recipient = None\n",
    "        else:\n",
    "            # add an explanatory prefix text:\n",
    "            funding_recipient = \"Recipient(s): \" + funding_recipient\n",
    "            # add the funding_recipient to the funding_info (with a \". \" separator), if that already contains some text, otherwise just use the funding_recipient as the funding_info:\n",
    "            if funding_info is not None:\n",
    "                funding_info = funding_info + \". \" + funding_recipient\n",
    "            else:\n",
    "                funding_info = funding_recipient\n",
    "        if funding_info is not None:\n",
    "        # add the funding_info (with data from |i and |e to the funding contribution node as a bf:note:\n",
    "            funding_info_node = BNode()\n",
    "            records_bf.add((funding_info_node, RDF.type, BF.Note))\n",
    "            records_bf.set((funding_info_node, RDFS.label, Literal(funding_info)))\n",
    "            records_bf.add((funding_contribution_node, BF.note, funding_info_node))\n",
    "        # add the funding contribution node to the work node:\n",
    "        records_bf.add((work_uri, BF.contribution, funding_contribution_node))\n",
    "        # return funding_contribution_node"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Function: Add Conference info from field CF\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_bf_conferences(work_uri, record):\n",
    "    # only use conferences from actual books (BE=SS or SM)\n",
    "    # ignore those in other publication types like journal article\n",
    "    if record.find(\"BE\").text == \"SS\" or record.find(\"BE\").text == \"SM\":\n",
    "        for conference in record.findall(\"CF\"):\n",
    "            # get the text content of the CF field,\n",
    "            # sanitize it by unescaping html entities and \n",
    "            # replacing STAR's ^DD encodings:\n",
    "            conference_field = html.unescape(mappings.replace_encodings(conference.text.strip()))\n",
    "            # try to get the conference name from the CF field:\n",
    "            try: \n",
    "                # get conference_name from main CF field, using the first part before any |:\n",
    "                conference_name = conference_field.split(\"|\")[0].strip()\n",
    "            except:\n",
    "                conference_name = \"MISSING CONFERENCE NAME\"\n",
    "            # then check the field for a date in apotential subfield |d:\n",
    "            try:\n",
    "                conference_date = conference_field.split(\"|d \")[1].split(\" |\")[0]\n",
    "            except:\n",
    "                conference_date = None\n",
    "            else:\n",
    "                # if there is a |d, add the full date to conference_note:\n",
    "                conference_note = \"Date(s): \" + conference_date\n",
    "                # extract the year from the date to use it as conference_year:\n",
    "                # Anything with 4 consecutive digits anywhere in the date string is a year.\n",
    "                # here is a regex for finding YYYY pattern in any string:\n",
    "                year_pattern = re.compile(r'\\d{4}')\n",
    "                # if there is a year in the date string, use that as the date:\n",
    "                if year_pattern.search(conference_date):\n",
    "                    conference_year = year_pattern.search(conference_date).group()\n",
    "                else:\n",
    "                    conference_year = None\n",
    "            # then check the field for a place in a potential subfield |o:\n",
    "            try:\n",
    "                conference_place = conference_field.split(\"|o \")[1].split(\" |\")[0]\n",
    "            except:\n",
    "                conference_place = None\n",
    "            # then check for a note in a potential subfield |b, but \n",
    "            # remebering to keep what is already in conference_note:\n",
    "            try:\n",
    "                conference_note = conference_note + \". \" + conference_field.split(\"|b \")[1].split(\" |\")[0]\n",
    "            except:\n",
    "                conference_note = conference_note\n",
    "        \n",
    "            # construct the node for the conference:\n",
    "            # a bnode for the contribution/conferencereference:\n",
    "            conference_reference_node = BNode()\n",
    "            records_bf.add((conference_reference_node, RDF.type, PXC.ConferenceReference))\n",
    "            # a blank node for the conference/meeting/agent:\n",
    "            conference_node = BNode()\n",
    "            records_bf.add((conference_node, RDF.type, BF.Meeting))\n",
    "            # attach the agent to the contribution/conferencereference:\n",
    "            records_bf.add((conference_reference_node, BF.agent, conference_node))\n",
    "            # add the conference name as a label to the agent/meeting node:\n",
    "            records_bf.add((conference_node, RDFS.label, Literal(conference_name)))\n",
    "            # add the year as a bflc:simpleDate to the agent/meeting node:\n",
    "            records_bf.add((conference_node, BFLC.simpleDate, Literal(conference_year)))\n",
    "            # add the place as a bflc:simplePlace to the agent/meeting node:\n",
    "            records_bf.add((conference_node, BFLC.simplePlace, Literal(conference_place)))\n",
    "            # add the note as a bf:note to the agent/meeting node, first adding a bnode for the bf:Note:\n",
    "            conference_note_node = BNode()\n",
    "            # make it a bf:Note:\n",
    "            records_bf.add((conference_note_node, RDF.type, BF.Note))\n",
    "            # add the note to the note node as a literal via rdfs:label:\n",
    "            records_bf.add((conference_note_node, RDFS.label, Literal(conference_note)))\n",
    "            # add a bf:role <http://id.loc.gov/vocabulary/relators/ctb> to the ConferenceReference (\"contributor\" - which is the default for conferences in DNB and LoC):\n",
    "            records_bf.add((conference_reference_node, BF.role, URIRef(\"http://id.loc.gov/vocabulary/relators/ctb\")))\n",
    "            # add the note node to the agent/meeting node via bf:note:\n",
    "            records_bf.add((conference_reference_node, BF.note, conference_note_node))\n",
    "            # add the conference node to the work node:\n",
    "            records_bf.add((work_uri, BF.contribution, conference_reference_node))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Functions: Add Research Data Link from DATAC and URLAI\n",
    "\n",
    "Field URLAI should only hold a doi for a psychdata dataset (these are usually, or rather, always, restricted access). This field always has a full doi link, in various prefix formats. We remove the prefix and only keep the pure doi. \n",
    "\n",
    "Field DATAC has either a subfield |u with a regular url link or a subfield |d with a doi (or both).\n",
    "The doi in DATAC is usually a pure doi, without any prefixes. But sometimes it's not! \n",
    "\n",
    "Since the data is so dirty, we make our own classficiation: we run all subfields, no matter declared Doi or URL (so from |u or |d) through our own recognition tree:\n",
    "- anything that is a pure doi (starts with \"10.\") will be saved as a DOI (bf:identifiedBy > bf:Doi)\n",
    "- so will anything with a pseudo url that is in reality just a DOI with a prefix (like \"https://doi.org/10.1234/5678\")\n",
    "- anything that seems a regular URL (with a DOI inside) will be declared a URL (bf:electronicLocator)\n",
    "- anything that is neither of the above will be ignored (or copied into a note)\n",
    "\n",
    "So we just check all subfields, see if they contain a DOI in any form, and keep that. And then check for other urls and keep those as electroniclocators (but only if they don't contain the DOI again?)\n",
    "\n",
    "In Bibframe, Research Data is modeled as a bnode bf:Work with a bf:Instance that has a bf:electronicLocator for a URL and a bf:identifiedBy for the Doi. This Work is in a bflc:Relationship to the study'S work, and the general relatedTo-subproperty should be bf:supplement. We also define a skos bflc:Relation \"hasResearchData\" to use as the bflc:Relationship's bflc:relation.\n",
    "\n",
    "Research Data can be (or rather, contain) either Code or a DataSet, or both. We can use the bf:genreForm to distinguish between the two, and also the Work subclass (bf:Dataset, bf:Multimedia).\n",
    "\n",
    "We also want to add the information whether the data is restricted or open access. We can do this in Bibframe with [bf_usageAndAccessPolicy](http://id.loc.gov/ontologies/bibframe/usageAndAccessPolicy) and [bf:AccessPolicy](http://id.loc.gov/ontologies/bibframe/AccessPolicy) on the Data's Instance (it is what the LoC instance-bf2marc excel table does. This info is based on MARC21 field 506).\n",
    "\n",
    "According to the github repo of the conversion script, it should look like this:\n",
    "\n",
    "```turtle\n",
    "<Instance> bf:usageAndAccessPolicy [\n",
    "    a bf:AccessPolicy;\n",
    "    rdfs:label \"open access\"@en, \"offener Zugang\"@de;\n",
    "    # or:\n",
    "    # rdfs:label \"restricted access\"@en, \"eingeschränkter Zugang\"@de;\n",
    "    rdf:value \"http://purl.org/coar/access_right/c_abf2\"^^xsd:anyURI; # a link to the license or uri of the skos term: here: open access\n",
    "    # or:\n",
    "    # rdf:value \"http://purl.org/coar/access_right/c_16ec\"^^xsd:anyURI; # restricted \n",
    "].\n",
    "```\n",
    "\n",
    "To be able to use a controlled vocabulary for this, we will make use of the COAR \"access rights\" skos vocabulary!\n",
    "https://vocabularies.coar-repositories.org/access_rights/ - its four concepts: open access, restriced access, embargoed access, metadata only access. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_urlai(work_uri, record):\n",
    "    \"\"\"Gets research data from field URLAI. This is always in PsychData, so it will be restricted access by default.\n",
    "    We will also assume it to always be just research data, not code.\n",
    "    \"\"\"\n",
    "    for data in record.findall(\"URLAI\"):\n",
    "        urlai_field = mappings.replace_encodings(data.text.strip())\n",
    "        unknown_field_content = None\n",
    "        #build the relationship node:\n",
    "        relationship_node, instance = build_work_relationship_node(work_uri, relation_type=\"rd_restricted_access\") \n",
    "        # there are no subfields in urlai, so let's just grab the whole thing and pass it on to the url or doi checker:\n",
    "        # if the string_type returned [1] is doi or url, treat them accordingly, using the returned string [0]\n",
    "        # as a doi or url:\n",
    "        url_set = set()\n",
    "        doi_set = set()\n",
    "        # if it is a doi, run a function to generate a doi identifier node\n",
    "        if check_for_url_or_doi(urlai_field)[1] == \"doi\":\n",
    "            # build_doi_identifier_node(instance,check_for_url_or_doi(urlai_field)[0])\n",
    "            doi_set.add(check_for_url_or_doi(urlai_field)[0])\n",
    "        elif check_for_url_or_doi(urlai_field)[1] == \"url\":\n",
    "            # build_electronic_locator_node(instance, check_for_url_or_doi(urlai_field)[0])\n",
    "            url_set.add(check_for_url_or_doi(urlai_field)[0])\n",
    "        # if the returned typ is something else \"unknown\", do nothing with it:\n",
    "        else:\n",
    "            # print(\"bf:note > bf:Note > rdfs:label: \" + urlai_field)\n",
    "            if urlai_field is not None and check_for_url_or_doi(urlai_field)[0] is not None and check_for_url_or_doi(urlai_field)[0] != \"\":\n",
    "                # add a variable \n",
    "                unknown_field_content = check_for_url_or_doi(urlai_field)[0].strip()\n",
    "                print(f\"unknown type: {unknown_field_content}. Adding as a note.\")\n",
    "                build_note_node(instance, check_for_url_or_doi(urlai_field)[0])\n",
    "\n",
    "        # compare all the dois in the doi_set to each item in the url set. If the url contains the doi, remove it from the url set:\n",
    "        for doi in doi_set:\n",
    "            for url in url_set.copy():\n",
    "                if doi in url:\n",
    "                    url_set.remove(url)\n",
    "                    # special case: if url is osf and contains the same short code as the osf doi, also remove the url. Example: https://osf.io/kcb4d/ and 10.17605/OSF.IO/KCB4D\n",
    "                elif \"osf.io\" in url and \"OSF.IO/\" in doi and doi.split(\"/\")[2].lower() in url:\n",
    "                    print(f\"duplicate doi in url {url}: {doi.split('/')[2]} from {doi}. Removing url in favor of doi.\")\n",
    "                    url_set.remove(url)\n",
    "\n",
    "        # loop through the set to build doi nodes, so we won't have duplicates:\n",
    "        for doi in doi_set:\n",
    "            build_doi_identifier_node(instance, doi)\n",
    "        for url in url_set:\n",
    "            build_electronic_locator_node(instance, url)\n",
    "            \n",
    "        # now attach the finished node for the relationship to the work:\n",
    "        records_bf.add((work_uri, BFLC.relationship, relationship_node))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_datac(work_uri, record):\n",
    "    \"\"\"Gets research data from field DATAC, adds a Relationship node to the work.\n",
    "Note: We define all data from this field as type \"research data only, no code\", and \"open/unrestricted access\"\n",
    "Newer data from PSYNDEXER may be something else, but for first migration, we assume all data is research data only.\n",
    "\"\"\"\n",
    "    # go through the list of datac fields and get the doi, if there is one:\n",
    "    for data in record.findall(\"DATAC\"):\n",
    "        datac_field = mappings.replace_encodings(data.text.strip())\n",
    "        # print(datac_field)\n",
    "        # add an item \"hello\" to the set:\n",
    "        #build the relationship node:\n",
    "        relationship_node, instance = build_work_relationship_node(work_uri, relation_type=\"rd_open_access\") \n",
    "        # we want to drop any duplicate dois that can occur if datac has a doi and doi url (same doi, but protocol etc prefixed) \n",
    "        # for the same data that,\n",
    "        # after conversion, ends up being identical. So we make a set of dois,\n",
    "        # which we will add dois to, and then later loop through the set (sets are by defintion list with only unique items!):\n",
    "        doi_set = set()\n",
    "        url_set = set()\n",
    "        # grab subfields u and d as strings and check if they are a url or a doi:\n",
    "        for subfield_name in (\"u\", \"d\"):\n",
    "            try: \n",
    "                subfield = get_subfield(datac_field, subfield_name)\n",
    "            except:\n",
    "                subfield = None\n",
    "            else:\n",
    "                # print(subfield)\n",
    "                # if the string_type returned [1] is doi or url, treat them accordingly, using the returned string [0]\n",
    "                # as a doi or url:\n",
    "                # if it is a doi, run a function to generate a doi identifier node\n",
    "                if subfield is not None and check_for_url_or_doi(subfield)[1] == \"doi\":\n",
    "                    # add the doi to a list:\n",
    "                    doi_set.add(check_for_url_or_doi(subfield)[0])\n",
    "                    #build_doi_identifier_node(instance, check_for_url_or_doi(subfield)[0])\n",
    "                elif subfield is not None and check_for_url_or_doi(subfield)[1] == \"url\":\n",
    "                    # add the url to a list:\n",
    "                    url_set.add(check_for_url_or_doi(subfield)[0])\n",
    "                    #build_electronic_locator_node(instance, check_for_url_or_doi(subfield)[0])\n",
    "                    # if the returned typ is something else \"unknown\", do nothing with it:\n",
    "                else:\n",
    "                    # print(\"bf:note > bf:Note > rdfs:label: \" + subfield)\n",
    "                    if subfield is not None and check_for_url_or_doi(subfield)[0] is not None and check_for_url_or_doi(subfield)[0] != \"\":\n",
    "                        # add a variable \n",
    "                        unknown_field_content = check_for_url_or_doi(subfield)[0].strip()\n",
    "                        print(f\"unknown type: {unknown_field_content}. Adding as a note.\")\n",
    "                        build_note_node(instance, check_for_url_or_doi(subfield)[0])\n",
    "        # first, compare all the dois in the doi_set to each item in the url set. If the url contains the doi, remove it from the url set:\n",
    "        for doi in doi_set:\n",
    "            for url in url_set.copy():\n",
    "                if doi in url:\n",
    "                    url_set.remove(url)\n",
    "                    # special case: if url is osf and contains the same short code as the osf doi, also remove the url. Example: https://osf.io/kcb4d/ and 10.17605/OSF.IO/KCB4D\n",
    "                elif \"osf.io\" in url and \"OSF.IO/\" in doi and doi.split(\"/\")[2].lower() in url:\n",
    "                    print(f\"duplicate doi in url {url}: {doi.split('/')[2]} from {doi}. Removing url in favor of doi.\")\n",
    "                    url_set.remove(url)\n",
    "        for doi in doi_set:\n",
    "            build_doi_identifier_node(instance, doi)\n",
    "        for url in url_set:\n",
    "            build_electronic_locator_node(instance, url)\n",
    "        # now attach the finished node for the relationship to the work:\n",
    "        records_bf.add((work_uri, BFLC.relationship, relationship_node))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Function: Get Series info for Books (field SE)\n",
    "\n",
    "Books (their Instances) can pe part of a monographic series - or even part of more than one series (usually that is a subseries of a bigger series). We currently store the title of that series, along with the volume number of the current book within that series, in field SE. Numbering is usually separated with a comma (but not always, sigh.)\n",
    "Also, there are cases where there is no numbering at all, or a strange one that is actually the issue and volume like from a journal?\n",
    "\n",
    "What we want is something like this - a regular case where there is a series title and a volume number that follows the title after a comma:\n",
    "\n",
    "```r\n",
    "Works:0396715_work a bf:Work;\n",
    "    # omitted: info about the work of the book itself (title, abstract, topics etc.)\n",
    "bf:hasInstance \n",
    "    [\n",
    "        # omitted: info about the instance of the book itself (publisher, date, etc.)\n",
    "        bflc:relationship [a bflc:Relationship;\n",
    "        bf:hasSeries [\n",
    "                a bf:Instance;\n",
    "                bf:title [ a bf:Title; bf:mainTitle \"Psychologie im Schulalltag\"];\n",
    "                # we don't index the series itself - so no local identifiers and no work except to say that \n",
    "                # the work is of subclass series and it is uncontrolled/not indexed.\n",
    "                bf:instanceOf [a bf:Series, bflc:Uncontrolled];\n",
    "            ];\n",
    "            # volume in series belongs into the relationship, not the series instance:\n",
    "            # based on /~https://github.com/lcnetdev/bibframe-ontology/issues/100\n",
    "            bf:seriesEnumeration \"Band 5\";\n",
    "        ];\n",
    "    ] .\n",
    "```\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# The Loop!\n",
    "## Creating the Work and Instance uris and adding other triples via functions"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Uris and types for Bibframe profile\n",
    "\n",
    "We want two URIs, since we split the Records into (at first) one work and one instance, which will be linked together.\n",
    "We also say one will be a (rdf:type) bf:Work and the other bf:Instance.\n",
    "Then we print all these triples into a file for the bibframe profile."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20 records\n",
      "2296 triples\n"
     ]
    }
   ],
   "source": [
    "# print(len(root.findall(\"Record\")))\n",
    "\n",
    "\n",
    "import json\n",
    "\n",
    "\n",
    "record_count = 0\n",
    "for record in root.findall(\"Record\")[0:20]:\n",
    "    # get the count of this record:\n",
    "    record_count += 1\n",
    "    # create a named graph dataset for each \"record\":\n",
    "    # Create an empty Dataset\n",
    "    # d = Dataset()\n",
    "    # Add a namespace prefix to it, just like for Graph\n",
    "    # d.bind(\"ex\", Namespace(\"http://example.com/\"))\n",
    "    # Declare a Graph URI to be used to identify a Graph\n",
    "    # graph_1 = URIRef(\"http://example.com/graph/\" + dfk + \"/\")\n",
    "    # Add an empty Graph, identified by graph_1, to the Dataset\n",
    "    #d.graph(identifier=graph_1)\n",
    "    \n",
    "\n",
    "\n",
    "\n",
    "    # get the DFK identifier from the record:\n",
    "    dfk = record.find(\"DFK\").text\n",
    "\n",
    "    # create a URI for the work and the instance and give them their correct bf classes:\n",
    "    # make sure a work_uri will look like this: works:dfk_work, eg works:123456_work\n",
    "    work_uri = URIRef(WORKS + dfk + \"_work\")\n",
    "    records_bf.add((work_uri, RDF.type, BF.Work))\n",
    "    #d.add((work_uri, RDF.type, BF.Work, graph_1))\n",
    "\n",
    "\n",
    "    # for each work, create one pxc:InstanceBundle with an uri like this instancebundles:0123456 (where 0123456 is the dfk of the record):\n",
    "    instance_bundle_uri = URIRef(INSTANCEBUNDLES + dfk)\n",
    "    records_bf.add((instance_bundle_uri, RDF.type, PXC.InstanceBundle))\n",
    "    # create one instance:\n",
    "    instance_uri =  URIRef(INSTANCES + dfk + \"#1\")\n",
    "    records_bf.add((instance_uri, RDF.type, BF.Instance))\n",
    "    #d.add((instance_uri, RDF.type, BF.Instance, graph_1))\n",
    "\n",
    "    # connect the instancebundle to the work:\n",
    "    records_bf.add((work_uri, PXP.hasInstanceBundle, instance_bundle_uri))\n",
    "    # connect the instance_bundle to the instance:\n",
    "    records_bf.add((instance_bundle_uri, BF.hasPart, instance_uri))\n",
    "    # connect work and instance via bf:instanceOf and bf:hasInstance:\n",
    "    records_bf.add((instance_uri, BF.instanceOf, work_uri))\n",
    "    records_bf.add((work_uri, BF.hasInstance, instance_uri))\n",
    "    #d.add((instance_uri, BF.instanceOf, work_uri, graph_1))\n",
    "    #d.add((work_uri, BF.hasInstance, instance_uri, graph_1))\n",
    "\n",
    "    # for the DFK, add an identifier node to the instancebundle using a function:\n",
    "    records_bf.add((instance_bundle_uri, BF.identifiedBy, get_bf_identifier_dfk(instance_bundle_uri, dfk)))\n",
    "    #d.add((instance_uri, BF.identifiedBy, get_bf_identifier_dfk(instance_uri, dfk), graph_1))\n",
    "\n",
    "    # get field TI and add as title node to the instance bundle:\n",
    "    records_bf.add((instance_bundle_uri, BF.title, get_bf_title(instance_bundle_uri, record)))\n",
    "\n",
    "    # get TIUE field and add as translated title node:\n",
    "    # but only if the field exists!\n",
    "    if record.find(\"TIUE\") is not None and record.find(\"TIUE\").text != \"\":\n",
    "        records_bf.add((instance_bundle_uri, BF.title, get_bf_translated_title(instance_bundle_uri, record)))\n",
    "        #d.add((instance_uri, BF.title, get_bf_translated_title(instance_uri, record), graph_1))\n",
    "\n",
    "    # get work language from LA\n",
    "    records_bf.add((work_uri, BF.language, get_work_language(record)))\n",
    "    #d.add((work_uri, BF.language, get_work_language(record), graph_1))\n",
    "\n",
    "    # get and add contributors:\n",
    "    # records_bf.add((work_uri, BF.contribution, add_bf_contributor_person(record)))\n",
    "    add_bf_contributor_person(work_uri, record)\n",
    "    # get toc, if it exists:\n",
    "    # get_bf_toc(work_uri, record)\n",
    "    \n",
    "    # get and add main/original abstract:\n",
    "    # note: somehow not all records have one!\n",
    "    if record.find(\"ABH\") is not None:\n",
    "        get_bf_abstract(work_uri, record)\n",
    "        # records_bf.add((work_uri, BF.summary, get_bf_abstract(work_uri, record)))\n",
    "        #d.add((work_uri, BF.summary, get_bf_abstract(work_uri, record), graph_1))\n",
    "\n",
    "    # get and add main/original abstract:\n",
    "    # note: somehow not all records have one!\n",
    "    if record.find(\"ABN\") is not None:\n",
    "        records_bf.add((work_uri, BF.summary, get_bf_secondary_abstract(work_uri, record)))\n",
    "       # d.add((work_uri, BF.summary, get_bf_secondary_abstract(work_uri, record), graph_1))\n",
    "\n",
    "    # get and add any CTs:\n",
    "    #add_controlled_terms(work_uri, record)\n",
    "\n",
    "    # get and add preregistration links:\n",
    "    #get_bf_preregistrations(work_uri, record)\n",
    "        # switched off for performance reasons\n",
    "\n",
    "    # get and add grants by using the returned set of nodes and adding it to the work:\n",
    "    # get_bf_grants(work_uri, record)\n",
    "    \n",
    "    #get and add conferences:\n",
    "    #get_bf_conferences(work_uri, record) # adds the generated bfls:Relationship node to the work\n",
    "\n",
    "    #get_datac(work_uri, record) # adds the generated bfls:Relationship node to the work\n",
    "        # switched off for performance reasons\n",
    "       \n",
    "    \n",
    "    # get book mediacarrier and add as secondary instance class:\n",
    "    #split_books(instance_uri, record)\n",
    "\n",
    "    # Serialize the Dataset to a file.\n",
    "    #d.serialize(destination=\"ttl-data/\" + dfk + \".jsonld\", format=\"json-ld\", auto_compact=True)\n",
    "\n",
    "\n",
    "# add a Literal for the count of records:\n",
    "# records_bf.add((URIRef(\"https://w3id.org/zpid/bibframe/records/\"), BF.count, Literal(record_count)))\n",
    "# and add it to the graph:\n",
    "# first, add a bnode of class bf:AdminMetadata to the graph:\n",
    "records_bf_admin_metadata_root = BNode()\n",
    "records_bf.add((records_bf_admin_metadata_root, RDF.type, BF.AdminMetadata))\n",
    "# add this bnode to the graph:\n",
    "records_bf.add((URIRef(\"https://w3id.org/zpid/bibframe/records/\"), BF.adminMetadata, records_bf_admin_metadata_root))\n",
    "# # add a bf:generationProcess to the admin metadata node:\n",
    "# records_bf.add((records_bf_admin_metadata_root, BF.generationProcess, Literal(\"Converted from PsychAuthors XML to BIBFRAME 2.2 using Python scripts\")))\n",
    "# # add a bf:generationDate to the admin metadata node:\n",
    "# #records_bf.add((records_bf_admin_metadata_root, BF.generationDate, Literal(datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\"))))\n",
    "# # add the count as BF.count:\n",
    "records_bf.add((records_bf_admin_metadata_root, PXP.recordCount, Literal(record_count)))\n",
    "\n",
    "\n",
    "print(record_count, \"records\")\n",
    "\n",
    "# print all the resulting triples:\n",
    "records_bf.serialize(\"ttl-data/bibframe_records.ttl\", format=\"turtle\")\n",
    "#records_bf.serialize(\"ttl-data/bibframe_records.jsonld\", format=\"json-ld\", auto_compact=True, sort_keys=True, index=True)\n",
    "# serialize as xml\n",
    "records_bf.serialize(\"ttl-data/bibframe_records.xml\", format=\"pretty-xml\")\n",
    "\n",
    "\n",
    "# # import a context file as a dict: \n",
    "# with open(\"bibframe-context.json\", \"r\") as context_file:\n",
    "#     context = json.load(context_file)\n",
    "# # add the context to the graph:\n",
    "# context_dict = context\n",
    "# # serialize as json-ld and pass the context dict:\n",
    "# records_bf.serialize(\"ttl-data/bibframe_records.jsonld\", format=\"json-ld\", auto_compact=True, sort_keys=True, index=True, context=context_dict)\n",
    "\n",
    "#testwork = \"https://w3id.org/resources/works/0401567\"\n",
    "#records_bf[testwork].serialize(\"ttl-data/bibframe_sample.jsonld\", format=\"json-ld\")\n",
    "\n",
    "print(len(records_bf), \"triples\")\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Uris and types for simplified profile (schema-org)\n",
    "\n",
    "For the simplified profile, we only need one entity per record (for now) and we give it the class schema:CreativeWork.\n",
    "Then we print the resulting triples into a separate file for the simplified profile that mostly uses schema.org properties and classes."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "# print(len(root.findall(\"Record\")))\n",
    "\n",
    "# for record in root.findall(\"Record\"):\n",
    "#     # get the DFK identifier from the record:\n",
    "#     dfk = record.find(\"DFK\").text\n",
    "\n",
    "#     # create a URI for the work by attaching the dfk to the works namespace and \n",
    "#     # then give it the correct schema.org class:\n",
    "#     work_uri = WORKS[dfk]\n",
    "#     records_schema.add((work_uri, RDF.type, SCHEMA.CreativeWork))\n",
    "\n",
    "#     # get work language from LA\n",
    "#     records_schema.add((work_uri, SCHEMA.inLanguage, get_work_language(record)))\n",
    "\n",
    "\n",
    "# records_schema.serialize(\"ttl-data/schema_records.jsonld\", format=\"json-ld\")\n",
    "# # records_schema.serialize(\"ttl-data/schema_records.ttl\", format=\"turtle\")\n",
    "# print(len(records_schema), \"triples\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "python-3.10.10",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.10"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}