;;; doi-utils.el --- DOI utilities for making bibtex entries -*- lexical-binding: t; -*-

;; Copyright (C) 2015-2021  John Kitchin

;; Author: John Kitchin <jkitchin@andrew.cmu.edu>
;; Keywords: convenience
;; Package-Requires: ((org-ref))

;; This program is free software; you can redistribute it and/or modify
;; it under the terms of the GNU General Public License as published by
;; the Free Software Foundation, either version 3 of the License, or
;; (at your option) any later version.

;; This program is distributed in the hope that it will be useful,
;; but WITHOUT ANY WARRANTY; without even the implied warranty of
;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
;; GNU General Public License for more details.

;; You should have received a copy of the GNU General Public License
;; along with this program.  If not, see <http://www.gnu.org/licenses/>.

;;; Commentary:

;; This package provides functionality to download PDFs and bibtex entries from
;; a DOI, as well as to update a bibtex entry from a DOI.  It depends slightly
;; on org-ref, to determine where to save pdf files too, and where to insert
;; bibtex entries in the default bibliography.

;; The principle commands you will use from here are:

;; - doi-utils-get-bibtex-entry-pdf with the cursor in a bibtex entry.
;; - doi-utils-insert-bibtex-entry-from-doi to insert a bibtex entry at your cursor, clean it and try to get a pdf.
;; - doi-utils-add-bibtex-entry-from-doi to add an entry to your default bibliography (cleaned with pdf if possible).
;; - doi-utils-update-bibtex-entry-from-doi with cursor in an entry to update its fields.

;;; Code:

(defvar url-http-end-of-headers)
(declare-function org-ref-find-bibliography "org-ref-core")
(declare-function org-ref-clean-bibtex-entry "org-ref-core")
(declare-function bibtex-completion-edit-notes "bibtex-completion")
(declare-function org-bibtex-yank "org-bibtex")
(declare-function org-ref-possible-bibfiles "org-ref-core")
(declare-function org-ref-normalize-bibtex-completion-bibliography "org-ref-utils")

(declare-function org-ref--file-ext-p "org-ref-utils")
(declare-function org-ref--directory-files "org-ref-utils")

(eval-when-compile
  (require 'cl-lib))
(require 'bibtex)
(require 'json)
(require 'org)                          ; org-add-link-type

(or (require 'ol-bibtex nil t)
    (require 'org-bibtex)) ; org-bibtex-yank

(require 'url-http)
(require 'url-handlers)
(require 'org-ref-utils)
(require 'transient)

;;* Customization
(defgroup doi-utils nil
  "Customization group for doi-utils."
  :tag "DOI utils"
  :group 'doi-utils)


(defcustom doi-utils-download-pdf
  t
  "Try to download PDFs when adding bibtex entries when non-nil."
  :type 'boolean
  :group 'doi-utils)

(defcustom doi-utils-open-pdf-after-download
  nil
  "Open PDF after adding bibtex entries."
  :type 'boolean
  :group 'doi-utils)


(defcustom doi-utils-timestamp-field
  "DATE_ADDED"
  "The bibtex field to store the date when an entry has been added."
  :type 'string
  :group 'doi-utils)

(defcustom doi-utils-timestamp-format-function
  'current-time-string
  "The function to format the timestamp for a bibtex entry.
Set to a function that returns nil to avoid setting timestamps in the entries.
e.g. (lambda () nil)"
  :type 'function
  :group 'doi-utils)


(defcustom doi-utils-dx-doi-org-url
  "https://doi.org/"
  "Base url to retrieve doi metadata from. A trailing / is required."
  :type 'string
  :group 'doi-utils)


(defcustom doi-utils-metadata-function 'doi-utils-get-json-metadata
  "Function for retrieving json metadata from `doi-utils-dx-doi-org-url'.
The default is `doi-utils-get-json-metadata', but it sometimes
fails with a proxy. An alternative is
`doi-utils-get-json-metadata-curl' which requires an external
program to use curl."
  :type 'function
  :group 'doi-utils)


(defcustom doi-utils-async-download t
  "Use `doi-utils-async-download-pdf' to get pdfs asynchrounously.
If nil use `doi-utils-get-bibtex-entry-pdf' synchronously."
  :type 'boolean
  :group 'doi-utils)


(defun doi-utils-pdf-filename-from-key ()
  "Generate PDF filename from bibtex entry key.
This is the default function for `doi-utils-pdf-filename-function'.
Returns the bibtex entry key as the filename (without extension or path)."
  (cdr (assoc "=key=" (bibtex-parse-entry))))


(defcustom doi-utils-pdf-filename-function
  'doi-utils-pdf-filename-from-key
  "Function to generate PDF filename from bibtex entry.
The function is called with no arguments while point is in the bibtex
entry, and should return a string to use as the PDF filename (without
the .pdf extension or directory path).

The directory path is determined separately by `bibtex-completion-library-path'.

The default function uses the bibtex entry key as the filename.

Example: To use the title field as the filename:
  (setq doi-utils-pdf-filename-function
        (lambda () (bibtex-autokey-get-field \"title\")))

Note: The function is responsible for ensuring the returned filename
is valid for the filesystem. Special characters in fields like title
may cause issues on some systems."
  :type 'function
  :group 'doi-utils)


;;* Getting pdf files from a DOI

;; The idea here is simple. When you visit http://dx.doi.org/doi or
;; https://doi.org/doi, you get redirected to the journal site. Once you have
;; the url for the article, you can usually compute the url to the pdf, or find
;; it in the page. Then you simply download it.

;; There are some subtleties in doing this that are described here. To get the
;; redirect, we have to use url-retrieve, and a callback function. The callback
;; does not return anything, so we communicate through global variables.
;; url-retrieve is asynchronous, so we have to make sure to wait for it to
;; finish.

(defvar *doi-utils-waiting* t
  "Stores waiting state for url retrieval.")

(defvar *doi-utils-redirect* nil
  "Stores redirect url from a callback function.")

(defun doi-utils-redirect-callback (&optional status)
  "Callback for `url-retrieve' to set the redirect.
Optional argument STATUS Unknown why this is optional."
  (when (plist-get status :error)
    (signal (car (plist-get status :error)) (cdr(plist-get status :error))))
  (when (plist-get status :redirect) ;  is nil if there none
    (setq *doi-utils-redirect* (plist-get status :redirect)))
  ;; we have done our job, so we are not waiting any more.
  (setq *doi-utils-waiting* nil))

;; To actually get the redirect we use url-retrieve like this.

(defun doi-utils-get-redirect (doi)
  "Get redirect url from `doi-utils-dx-doi-org-url'/doi."
  ;; we are going to wait until the url-retrieve is done
  (setq *doi-utils-waiting* t)
  ;; start with no redirect. it will be set in the callback.
  (setq *doi-utils-redirect* nil)
  (url-retrieve
   (format "%s%s" doi-utils-dx-doi-org-url doi)
   'doi-utils-redirect-callback)
  ;; I suspect we need to wait here for the asynchronous process to
  ;; finish. we loop and sleep until the callback says it is done via
  ;; `*doi-utils-waiting*'. this works as far as i can tell. Before I
  ;; had to run this a few times to get it to work, which i suspect
  ;; just gave the first one enough time to finish.
  (while *doi-utils-waiting* (sleep-for 0.1)))

;; Once we have a redirect for a particular doi, we need to compute the url to
;; the pdf. We do this with a series of functions. Each function takes a single
;; argument, the redirect url. If it knows how to compute the pdf url it does,
;; and returns it. We store the functions in a variable:

(defvar doi-utils-pdf-url-functions nil
  "Functions that return a url to a pdf from a redirect url.
Each function takes one argument, the redirect url.  The function
must return a pdf-url, or nil.")


;;** APS journals

(defun aps-pdf-url (*doi-utils-redirect*)
  "Get url to the pdf from *DOI-UTILS-REDIRECT*."
  (when (string-match "^http\\(s*\\)://journals.aps.org" *doi-utils-redirect*)
    (replace-regexp-in-string "/abstract/" "/pdf/" *doi-utils-redirect*)))


;;** Science

(defun science-pdf-url (*doi-utils-redirect*)
  "Get url to the pdf from *DOI-UTILS-REDIRECT*."
  (when (string-match "^http\\(s?\\)://www.sciencemag.org" *doi-utils-redirect*)
    (concat *doi-utils-redirect* ".full.pdf")))


;;** Nature

(defun nature-pdf-url (*doi-utils-redirect*)
  "Get url to the pdf from *DOI-UTILS-REDIRECT*."
  (when (string-match "^http\\(s?\\)://www.nature.com" *doi-utils-redirect*)
    (concat *doi-utils-redirect* ".pdf")))


;;** Elsevier/ScienceDirect
;; You cannot compute these pdf links; they are embedded in the redirected pages.

(defvar *doi-utils-pdf-url* nil
  "Stores url to pdf download from a callback function.")

;;** Wiley
;; Wiley have changed the url structure from
;; http://onlinelibrary.wiley.com/doi/10.1002/anie.201402680/abstract
;; http://onlinelibrary.wiley.com/doi/10.1002/anie.201402680/pdf
;; to
;; http://onlinelibrary.wiley.com/doi/abs/10.1002/anie.201402680
;; http://onlinelibrary.wiley.com/doi/pdf/10.1002/anie.201402680
;; Hence fewer steps are now required.

;; https://onlinelibrary.wiley.com/doi/10.1002/adts.202200926
;; https://onlinelibrary.wiley.com/doi/epdf/10.1002/adts.202200926

;; (defun wiley-pdf-url (*doi-utils-redirect*)
;;   "Get url to the pdf from *DOI-UTILS-REDIRECT*."
;;   (when (string-match "^http\\(s?\\)://onlinelibrary.wiley.com" *doi-utils-redirect*)
;;     (replace-regexp-in-string "doi/abs" "doi/pdf" *doi-utils-redirect*)))


(defun wiley-pdf-url-2 (*doi-utils-redirect*)
  "Get url to the pdf from *DOI-UTILS-REDIRECT*.
[2023-04-10 Mon] updated a new rule.
https://onlinelibrary.wiley.com/doi/pdfdirect/10.1002/anie.201310461?download=true"
  (when (string-match "^http\\(s?\\)://onlinelibrary.wiley.com" *doi-utils-redirect*)
    (concat
     (replace-regexp-in-string "doi/" "doi/pdfdirect/" *doi-utils-redirect*)
     "?download=true")))


(defun agu-pdf-url (*doi-utils-redirect*)
  "Get url to the pdf from *DOI-UTILS-REDIRECT*."
  (when (string-match "https://agupubs.onlinelibrary.wiley.com"
		      *doi-utils-redirect*)
    (replace-regexp-in-string "/full/" "/pdfdirect/" *doi-utils-redirect*)))


;;** Springer

(defun springer-chapter-pdf-url (*doi-utils-redirect*)
  (when (string-match "^http\\(s?\\)://link.springer.com/chapter/" *doi-utils-redirect*)
    (replace-regexp-in-string "/chapter" "/content/pdf"
			      (concat *doi-utils-redirect* ".pdf"))))


(defun springer-pdf-url (*doi-utils-redirect*)
  "Get url to the pdf from *DOI-UTILS-REDIRECT*."
  (when (string-match "^http\\(s?\\)://link.springer.com" *doi-utils-redirect*)
    (replace-regexp-in-string "/article/" "/content/pdf/"
			      (concat *doi-utils-redirect* ".pdf"))))


;;** ACS
;; here is a typical url http://pubs.acs.org/doi/abs/10.1021/nl500037x
;; the pdf is found at http://pubs.acs.org/doi/pdf/10.1021/nl500037x

;; we just change /abs/ to /pdf/.

(defun acs-pdf-url-1 (*doi-utils-redirect*)
  "Get url to the pdf from *DOI-UTILS-REDIRECT*."
  (when (string-match "^http\\(s?\\)://pubs.acs.org/doi/abs/" *doi-utils-redirect*)
    (replace-regexp-in-string "/abs/" "/pdf/" *doi-utils-redirect*)))

;; 1/20/2016 I noticed this new pattern in pdf urls, where there is no abs in
;; the url
(defun acs-pdf-url-2 (*doi-utils-redirect*)
  "Get url to the pdf from *DOI-UTILS-REDIRECT*."
  (when (string-match "^http\\(s?\\)://pubs.acs.org/doi/" *doi-utils-redirect*)
    (replace-regexp-in-string "/doi/" "/doi/pdf/" *doi-utils-redirect*)))

;; 1/18/2019: It looks like they are using https now
(defun acs-pdf-url-3 (*doi-utils-redirect*)
  "Get url to the pdf from *DOI-UTILS-REDIRECT*."
  (when (string-match "^https://pubs.acs.org/doi/" *doi-utils-redirect*)
    (replace-regexp-in-string "/doi/" "/doi/pdf/" *doi-utils-redirect*)))


;;** IOP

(defun iop-pdf-url (*doi-utils-redirect*)
  "Get url to the pdf from *DOI-UTILS-REDIRECT*."
  (when (string-match "^http\\(s?\\)://iopscience.iop.org" *doi-utils-redirect*)
    (concat *doi-utils-redirect* "/pdf")))

;;** JSTOR

(defun jstor-pdf-url (*doi-utils-redirect*)
  "Get url to the pdf from *DOI-UTILS-REDIRECT*."
  (when (string-match "^http\\(s?\\)://www.jstor.org" *doi-utils-redirect*)
    (concat (replace-regexp-in-string "/stable/" "/stable/pdfplus/" *doi-utils-redirect*) ".pdf")))


;;** AIP

(defun aip-pdf-url (*doi-utils-redirect*)
  "Get url to the pdf from *DOI-UTILS-REDIRECT*."
  (when (string-match "^http\\(s?\\)://scitation.aip.org" *doi-utils-redirect*)
    ;; get stuff after content
    (let (p1 p2 s p3)
      (setq p2 (replace-regexp-in-string
                "^http\\(s?\\)://scitation.aip.org/" "" *doi-utils-redirect*))
      (setq s (split-string p2 "/"))
      (setq p1 (mapconcat 'identity (org-ref--remove-at-indices '(0 6) s) "/"))
      (setq p3 (concat "/" (nth 0 s) (nth 1 s) "/" (nth 2 s) "/" (nth 3 s)))
      (format "http://scitation.aip.org/deliver/fulltext/%s.pdf?itemId=/%s&mimeType=pdf&containerItemId=%s"
              p1 p2 p3))))

(defun aip-pdf-url-2 (*doi-utils-redirect*)
  "Get url to the pdf from *DOI-UTILS-REDIRECT*."
  ;; [2021-08-28 Sat] Seems like they changed the link a little?
  ;; https://aip.scitation.org/doi/10.1063/1.5019667
  ;; to
  ;; https://aip.scitation.org/doi/pdf/10.1063/1.5019667
  (when (string-match "^http\\(s?\\)://aip.scitation.org" *doi-utils-redirect*)
    (concat "https://aip.scitation.org/doi/pdf" (cl-second (split-string *doi-utils-redirect* "doi")))))

;;** Taylor and Francis

(defun tandfonline-pdf-url (*doi-utils-redirect*)
  "Get url to the pdf from *DOI-UTILS-REDIRECT*."
  (when (string-match "^http\\(s?\\)://www.tandfonline.com" *doi-utils-redirect*)
    (replace-regexp-in-string "/abs/\\|/full/" "/pdf/" *doi-utils-redirect*)))

;;** ECS

(defun ecs-pdf-url (*doi-utils-redirect*)
  "Get url to the pdf from *DOI-UTILS-REDIRECT*."
  (when (string-match "^http\\(s?\\)://jes.ecsdl.org" *doi-utils-redirect*)
    (replace-regexp-in-string "\\.abstract$" ".full.pdf" *doi-utils-redirect*)))

;; http://ecst.ecsdl.org/content/25/2/2769
;; http://ecst.ecsdl.org/content/25/2/2769.full.pdf


(defun ecst-pdf-url (*doi-utils-redirect*)
  "Get url to the pdf from *DOI-UTILS-REDIRECT*."
  (when (string-match "^http\\(s?\\)://ecst.ecsdl.org" *doi-utils-redirect*)
    (concat *doi-utils-redirect* ".full.pdf")))



;;** RSC

(defun rsc-pdf-url (*doi-utils-redirect*)
  "Get url to the pdf from *DOI-UTILS-REDIRECT*."
  (when (string-match "^http\\(s?\\)://pubs.rsc.org" *doi-utils-redirect*)
    (let ((url (downcase *doi-utils-redirect*)))
      (setq url (replace-regexp-in-string "articlelanding" "articlepdf" url))
      url)))

;;** Science Direct

;; https://www.sciencedirect.com/science/article/pii/S001085452200577X?via%3Dihub
;; https://www.sciencedirect.com/science/article/pii/S001085452200577X/pdfft?isDTMRedir=true&download=true

(defun science-direct-pdf-url (*doi-utils-redirect*)
  "Get url to the pdf from *DOI-UTILS-REDIRECT*."
  (when (string-match "^http\\(s?\\)://www.sciencedirect.com" *doi-utils-redirect*)
    (replace-string "?via%3Dihub" "/pdfft?isDTMRedir=true&download=true" *doi-utils-redirect*)))

;; (defun doi-utils-get-science-direct-pdf-url (redirect-url)
;;   "Science direct hides the pdf url in html.  We get it out here.
;; REDIRECT-URL is where the pdf url will be in."
;;   (let ((first-url
;;          (with-current-buffer (url-retrieve-synchronously redirect-url)
;;            (goto-char (point-min))
;;            (when (re-search-forward "pdf_url\" content=\"\\([^\"]*\\)\"" nil t)
;;              (match-string-no-properties 1)))))
;;     (and first-url
;;          (with-current-buffer (url-retrieve-synchronously first-url)
;;            (goto-char (point-min))
;;            (when (re-search-forward "or click <a href=\"\\([^\"]*\\)\">" nil t)
;;              (match-string-no-properties 1))))))

;; (defun science-direct-pdf-url (*doi-utils-redirect*)
;;   "Get url to the pdf from *DOI-UTILS-REDIRECT*."
;;   (when (string-match "^http\\(s?\\)://www.sciencedirect.com" *doi-utils-redirect*)
;;     (doi-utils-get-science-direct-pdf-url *doi-utils-redirect*)))

;; sometimes I get
;; http://linkinghub.elsevier.com/retrieve/pii/S0927025609004558
;; which actually redirect to
;; http://www.sciencedirect.com/science/article/pii/S0927025609004558

;; https://www.sciencedirect.com/science/article/pii/S001085452200577X?via%3Dihub
;; https://www.sciencedirect.com/science/article/pii/S001085452200577X/pdfft?isDTMRedir=true&download=true

;; (defun linkinghub-elsevier-pdf-url (*doi-utils-redirect*)
;;   "Get url to the pdf from *DOI-UTILS-REDIRECT*."
;;   (when (string-match
;; 	 "^https://linkinghub.elsevier.com/retrieve" *doi-utils-redirect*)
;;     (science-direct-pdf-url
;;      (replace-regexp-in-string
;;       ;; change URL to science direct and use function to get pdf URL
;;       "https://linkinghub.elsevier.com/retrieve"
;;       "https://www.sciencedirect.com/science/article"
;;       *doi-utils-redirect*))))

;; https://www.sciencedirect.com/science/article/pii/S1385894723014973/pdfft?isDTMRedir=true&download=true

(defun linkinghub-elsevier-pdf-url (*doi-utils-redirect*)
  "Get url to the pdf from *DOI-UTILS-REDIRECT*."
  (when (string-match
	 "^https://linkinghub.elsevier.com/retrieve" *doi-utils-redirect*)
    (concat
     (replace-regexp-in-string
      ;; change URL to science direct and use function to get pdf URL
      "https://linkinghub.elsevier.com/retrieve"
      "https://www.sciencedirect.com/science/article"
      *doi-utils-redirect*)
     "/pdfft?isDTMRedir=true")))

;;** PNAS
;; http://www.pnas.org/content/early/2014/05/08/1319030111
;; http://www.pnas.org/content/early/2014/05/08/1319030111.full.pdf

;; with supporting info
;; http://www.pnas.org/content/early/2014/05/08/1319030111.full.pdf+html?with-ds=yes

(defun pnas-pdf-url (*doi-utils-redirect*)
  "Get url to the pdf from *DOI-UTILS-REDIRECT*."
  (when (string-match "^http\\(s?\\)://www.pnas.org" *doi-utils-redirect*)
    (concat *doi-utils-redirect* ".full.pdf?with-ds=yes")))


;;** Copernicus Publications
(defvar copernicus-journal-urls '(
                                  "^https://www.adv-geosci.net/"
                                  "^https://www.adv-radio-sci.net/"
                                  "^https://www.adv-sci-res.net/"
                                  "^https://www.adv-stat-clim-meteorol-oceanogr.net/"
                                  "^https://www.ann-geophys.net/"
                                  "^https://www.arch-anim-breed.net/"
                                  "^https://www.astra-proc.net/"
                                  "^https://www.atmos-chem-phys.net/"
                                  "^https://www.atmos-chem-phys-discuss.net/"
                                  "^https://www.atmos-meas-tech.net/"
                                  "^https://www.atmos-meas-tech-discuss.net/"
                                  "^https://www.biogeosciences.net/"
                                  "^https://www.biogeosciences-discuss.net/"
                                  "^https://www.clim-past.net/recent_papers.html"
                                  "^https://www.clim-past-discuss.net/"
                                  "^https://www.drink-water-eng-sci.net/"
                                  "^https://www.drink-water-eng-sci-discuss.net/"
                                  "^https://www.eg-quaternary-sci-j.net/"
                                  "^https://www.earth-surf-dynam.net/"
                                  "^https://www.earth-surf-dynam-discuss.net/"
                                  "^https://www.earth-syst-dynam.net/"
                                  "^https://www.earth-syst-dynam-discuss.net/"
                                  "^https://www.earth-syst-sci-data.net/"
                                  "^https://www.earth-syst-sci-data-discuss.net/"
                                  "^https://www.foss-rec.net/"
                                  "^https://www.geogr-helv.net/"
                                  "^https://www.geosci-instrum-method-data-syst.net/"
                                  "^https://www.geosci-instrum-method-data-syst-discuss.net/"
                                  "^https://www.geosci-model-dev.net/"
                                  "^https://www.geosci-model-dev-discuss.net/"
                                  "^https://www.hist-geo-space-sci.net/"
                                  "^https://www.hydrol-earth-syst-sci.net/"
                                  "^https://www.hydrol-earth-syst-sci-discuss.net/"
                                  "^https://www.j-sens-sens-syst.net/"
                                  "^https://www.mech-sci.net/"
                                  "^https://www.nat-hazards-earth-syst-sci.net/"
                                  "^https://www.nonlin-processes-geophys-discuss.net/"
                                  "^https://www.ocean-sci.net/"
                                  "^https://www.ocean-sci-discuss.net/"
                                  "^https://www.primate-biol.net/"
                                  "^https://www.proc-iahs.net/"
                                  "^https://www.sci-dril.net/"
                                  "^https://www.soil-journal.net/"
                                  "^https://www.soil-discuss.net/"
                                  "^https://www.solid-earth.net/"
                                  "^https://www.solid-earth-discuss.net/"
                                  "^https://www.stephan-mueller-spec-publ-ser.net/"
                                  "^https://www.the-cryosphere.net/"
                                  "^https://www.the-cryosphere-discuss.net/"
                                  "^https://www.web-ecol.net/"
                                  "^https://www.wind-energ-sci.net/"
                                  "^https://www.wind-energ-sci-discuss.net/"
                                  )
  "List of Copernicus URLs.")

(defun doi-utils-get-copernicus-pdf-url (redirect-url)
  "Copernicus hides the pdf url in html.  We get it out here.
REDIRECT-URL is where the pdf url will be in."
  (setq *doi-utils-waiting* t)
  (url-retrieve
   redirect-url
   (lambda (status)
     (goto-char (point-min))
     (re-search-forward "citation_pdf_url\" content=\"\\([^\"]*\\)\"" nil t)

     (setq *doi-utils-pdf-url* (match-string 1)
	   *doi-utils-waiting* nil)))
  (while *doi-utils-waiting* (sleep-for 0.1))
  *doi-utils-pdf-url*)

(defun copernicus-pdf-url (*doi-utils-redirect*)
  "Get url to the pdf from *DOI-UTILS-REDIRECT*."

  (car (cl-loop for copurl in copernicus-journal-urls
	        when (string-match copurl *doi-utils-redirect*)
	        collect
	        (progn (doi-utils-get-copernicus-pdf-url *doi-utils-redirect*)
	               *doi-utils-pdf-url*))))


;;** Sage
(defun sage-pdf-url (*doi-utils-redirect*)
  "Get url to the pdf from *DOI-UTILS-REDIRECT*."
  (when (string-match "^http\\(s?\\)://pss.sagepub.com" *doi-utils-redirect*)
    (concat *doi-utils-redirect* ".full.pdf")))


;;** Journal of Neuroscience
(defun jneurosci-pdf-url (*doi-utils-redirect*)
  "Get url to the pdf from *DOI-UTILS-REDIRECT*."
  (when (string-match "^http\\(s?\\)://www.jneurosci.org" *doi-utils-redirect*)
    (concat *doi-utils-redirect* ".full.pdf")))

;;** Generic .full.pdf
(defun generic-full-pdf-url (*doi-utils-redirect*)
  (let ((pdf (concat *doi-utils-redirect* ".full.pdf")))
    (when (url-http-file-exists-p pdf)
      pdf)))

;;** IEEE
;; 10.1109/re.2014.6912247
;; http(s)://ieeexplore.ieee.org/xpl/articleDetails.jsp?arnumber=6912247
;; http(s)://ieeexplore.ieee.org/ielx7/6903646/6912234/06912247.pdf
;; http(s)://ieeexplore.ieee.org/iel7/6903646/6912234/06912247.pdf?arnumber=6912247
;; <meta name="citation_pdf_url" content="http(s)://ieeexplore.ieee.org/iel7/6903646/6912234/06912247.pdf?arnumber=6912247">
;; <frame src="http(s)://ieeexplore.ieee.org/ielx7/6903646/6912234/06912247.pdf?tp=&arnumber=6912247&isnumber=6912234" frameborder=0 />
(defun ieee-pdf-url (*doi-utils-redirect*)
  "Get a url to the pdf from *DOI-UTILS-REDIRECT* for IEEE urls."
  (when (string-match "^https?://ieeexplore.ieee.org" *doi-utils-redirect*)
    (with-current-buffer (url-retrieve-synchronously *doi-utils-redirect*)
      (goto-char (point-min))
      (when (re-search-forward "<meta name=\"citation_pdf_url\" content=\"\\([[:ascii:]]*?\\)\">" nil t)
	(let ((framed-url (match-string 1)))
          (with-current-buffer (url-retrieve-synchronously framed-url)
            (goto-char (point-min))
            (when (re-search-forward "<frame src=\"\\(http[[:ascii:]]*?\\)\"" nil t)
              (match-string 1))))))))

;; At least some IEEE papers need the following new pdf-link parsing
;; Example: 10.1109/35.667413
(defun ieee2-pdf-url (*doi-utils-redirect*)
  "Get a url to the pdf from *DOI-UTILS-REDIRECT* for IEEE urls."
  (when (string-match "^https?://ieeexplore.ieee.org" *doi-utils-redirect*)
    (with-current-buffer (url-retrieve-synchronously *doi-utils-redirect*)
      (goto-char (point-min))
      (when (re-search-forward "\"pdfUrl\":\"\\([[:ascii:]]*?\\)\"" nil t)
	(let ((framed-url (match-string 1)))
          (with-current-buffer (url-retrieve-synchronously (concat "http://ieeexplore.ieee.org" framed-url))
            (goto-char (point-min))
            (when (re-search-forward "<frame src=\"\\(http[[:ascii:]]*?\\)\"" nil t)
              (match-string 1))))))))

;; Another try to get the ieee pdf
;; <iframe src="http(s)://ieeexplore.ieee.org/ielx5/8/4538127/04538164.pdf?tp=&arnumber=4538164&isnumber=4538127" frameborder=0>
(defun ieee3-pdf-url (*doi-utils-redirect*)
  "Get a url to the pdf from *DOI-UTILS-REDIRECT* for IEEE urls."
  (when (string-match "^https?://ieeexplore.ieee.org" *doi-utils-redirect*)
    (with-current-buffer (url-retrieve-synchronously *doi-utils-redirect*)
      (goto-char (point-min))
      (when (re-search-forward "\"pdfUrl\":\"\\([[:ascii:]]*?\\)\"" nil t)
	(let ((framed-url (match-string 1)))
          (with-current-buffer (url-retrieve-synchronously (concat "http://ieeexplore.ieee.org" framed-url))
            (goto-char (point-min))
            (when (re-search-forward "<iframe src=\"\\(http[[:ascii:]]*?\\)\"" nil t)
              (match-string 1))))))))

;; ACM Digital Library
;; https://dl.acm.org/doi/10.1145/1368088.1368132
(defun acm-pdf-url (*doi-utils-redirect*)
  "Get a url to the pdf from *DOI-UTILS-REDIRECT* for ACM urls."
  (when (string-match "^https?://dl.acm.org" *doi-utils-redirect*)
    (replace-regexp-in-string "doi" "doi/pdf" *doi-utils-redirect* )))

;;** Optical Society of America (OSA)
(defun osa-pdf-url (*doi-utils-redirect*)
  "Get url to the pdf from *DOI-UTILS-REDIRECT*."
  (when (string-match "^https://www.osapublishing.org" *doi-utils-redirect*)
    (replace-regexp-in-string "abstract.cfm" "viewmedia.cfm" *doi-utils-redirect* )))



;;** Publishers using Highwire Press metatags
;; For context and details, see:
;; https://webmasters.stackexchange.com/questions/72746/where-are-the-complete-set-of-highwire-press-metatags-defined

(defun highwire-pdf-url (*doi-utils-redirect*)
  "Typical URL:  http://biomechanical.asmedigitalcollection.asme.org/article.aspx?articleid=1427237

On this page the pdf might be here:     <meta name=\"citation_author\" content=\"Dalong Li\" /><meta name=\"citation_author_email\" content=\"dal40@pitt.edu\" /><meta name=\"citation_author\" content=\"Anne M. Robertson\" /><meta name=\"citation_author_email\" content=\"rbertson@pitt.edu\" /><meta name=\"citation_title\" content=\"A Structural Multi-Mechanism Damage Model for Cerebral Arterial Tissue\" /><meta name=\"citation_firstpage\" content=\"101013\" /><meta name=\"citation_doi\" content=\"10.1115/1.3202559\" /><meta name=\"citation_keyword\" content=\"Mechanisms\" /><meta name=\"citation_keyword\" content=\"Biological tissues\" /><meta name=\"citation_keyword\" content=\"Stress\" /><meta name=\"citation_keyword\" content=\"Fibers\" /><meta name=\"citation_journal_title\" content=\"Journal of Biomechanical Engineering\" /><meta name=\"citation_journal_abbrev\" content=\"J Biomech Eng\" /><meta name=\"citation_volume\" content=\"131\" /><meta name=\"citation_issue\" content=\"10\" /><meta name=\"citation_publication_date\" content=\"2009/10/01\" /><meta name=\"citation_issn\" content=\"0148-0731\" /><meta name=\"citation_publisher\" content=\"American Society of Mechanical Engineers\" /><meta name=\"citation_pdf_url\" content=\"http://biomechanical.asmedigitalcollection.asme.org/data/journals/jbendy/27048/101013_1.pdf\" />

It is in the citation_pdf_url.

It would be better to parse this, but here I just use a regexp.
"

  (when (or (string-match "^http\\(s?\\)://biomechanical.asmedigitalcollection.asme.org" *doi-utils-redirect*)
	    (string-match "^http\\(s?\\)://ojs.aaai.org" *doi-utils-redirect*)
	    (string-match "^http\\(s?\\)://aclanthology.org" *doi-utils-redirect*))
    (setq *doi-utils-waiting* 0)
    (url-retrieve
     *doi-utils-redirect*
     (lambda (status)
       (or (progn (goto-char (point-min))
		  (re-search-forward "citation_pdf_url\"? content=\"\\(.*\\)\"" nil t))
	   (progn (goto-char (point-min))
		  (re-search-forward "\"\\([^\"]*\\)\" name=\"?citation_pdf_url" nil t)))
       ;; (message-box (match-string 1))
       (setq *doi-utils-pdf-url* (match-string 1)
	     *doi-utils-waiting* nil)))
    (while (and *doi-utils-waiting* (< *doi-utils-waiting* 5))
      (setq *doi-utils-waiting* (+ *doi-utils-waiting* 0.1))
      (sleep-for 0.1))
    *doi-utils-pdf-url*))


;; Society for Industrial and Applied Mathematics (SIAM)
(defun siam-pdf-url (*doi-utils-redirect*)
  "Get url to the pdf from *DOI-UTILS-REDIRECT*."
  (when (string-match "^http\\(s?\\)://epubs.siam.org" *doi-utils-redirect*)
    (replace-regexp-in-string "/doi/" "/doi/pdf/" *doi-utils-redirect* )))

;; PLOS journals
;; https://plos.org/
(defun plos-pdf-url (*doi-utils-redirect*)
  "Get url to the pdf from *DOI-UTILS-REDIRECT*."
  (when (string-match "^http\\(s*\\)://journals.plos.org" *doi-utils-redirect*)
    (concat (replace-regexp-in-string (regexp-quote "/article?id=") "/article/file?id=" *doi-utils-redirect*) "&type=printable")))


;; https://www.frontiersin.org/articles/10.3389/fchem.2022.1037997/full
;; https://www.frontiersin.org/articles/10.3389/fchem.2022.1037997/pdf
(defun frontiers-pdf-url (*doi-utils-redirect*)
  (when (string-match "^http\\(s*\\)://www.frontiersin.org" *doi-utils-redirect*)
    (replace-regexp-in-string "/full" "/pdf" *doi-utils-redirect*)))




;; https://chemistry-europe.onlinelibrary.wiley.com/doi/10.1002/celc.201902035
;; https://chemistry-europe.onlinelibrary.wiley.com/doi/epdf/10.1002/celc.201902035
(defun chemistry-europe-pdf-url (*doi-utils-redirect*)
  (when (string-match "^http\\(s*\\)://chemistry-europe.onlinelibrary.wiley.com" *doi-utils-redirect*)
    (concat
     (replace-regexp-in-string "/doi" "/doi/pdfdirect" *doi-utils-redirect*)
     "?download=true")))


;; ** from issue #1081

(defun arxiv-pdf-url (*doi-utils-redirect*)
  "Get url to the pdf from *DOI-UTILS-REDIRECT*."
  (when (string-match-p "^https?://arxiv\\.org" *doi-utils-redirect*)
    (concat (replace-regexp-in-string "/abs/" "/pdf/" *doi-utils-redirect*)
	    ".pdf")))


(defun rss-pdf-url (*doi-utils-redirect*)
  "Get url to the pdf from *DOI-UTILS-REDIRECT*."
  (when (string-match-p "roboticsproceedings" *doi-utils-redirect*)
    (concat (replace-regexp-in-string "\\.html" ".pdf" *doi-utils-redirect*))))


(defun ieeestamp-pdf-url (*doi-utils-redirect*)
  "Get url to the pdf from *DOI-UTILS-REDIRECT*."
  (when (string-match "^https?://ieeexplore\\.ieee\\.org/document/\\([0-9]+\\)"
		      *doi-utils-redirect*)
    (concat "https://ieeexplore.ieee.org/stampPDF/getPDF.jsp?tp=&arnumber="
	    (match-string 1 *doi-utils-redirect*))))


;;** Add all functions

(setq doi-utils-pdf-url-functions
      (list
       'aps-pdf-url
       'science-pdf-url
       'nature-pdf-url
       ;; 'wiley-pdf-url
       'wiley-pdf-url-2
       'springer-chapter-pdf-url
       'springer-pdf-url
       'acs-pdf-url-1
       'acs-pdf-url-2
       'acs-pdf-url-3
       'iop-pdf-url
       'jstor-pdf-url
       'aip-pdf-url
       'aip-pdf-url-2
       'science-direct-pdf-url
       'linkinghub-elsevier-pdf-url
       'tandfonline-pdf-url
       'ecs-pdf-url
       'ecst-pdf-url
       'rsc-pdf-url
       'pnas-pdf-url
       'copernicus-pdf-url
       'sage-pdf-url
       'jneurosci-pdf-url
       'ieee-pdf-url
       'ieee2-pdf-url
       'ieee3-pdf-url
       'acm-pdf-url
       'osa-pdf-url
       'highwire-pdf-url
       'siam-pdf-url
       'agu-pdf-url
       'plos-pdf-url
       'frontiers-pdf-url
       'chemistry-europe-pdf-url 
       'generic-full-pdf-url
       'arxiv-pdf-url
       'rss-pdf-url
       'ieeestamp-pdf-url))

;;** Get the pdf url for a doi

(defun doi-utils-get-pdf-url (doi)
  "Return a url to a pdf for the DOI if one can be calculated.
Loops through the functions in `doi-utils-pdf-url-functions'
until one is found."
  (doi-utils-get-redirect doi)

  (unless *doi-utils-redirect*
    (error "No redirect found for %s" doi))
  (catch 'pdf-url
    (dolist (func doi-utils-pdf-url-functions)
      (let ((this-pdf-url (funcall func *doi-utils-redirect*)))
        (when this-pdf-url
          (throw 'pdf-url this-pdf-url))))))

;;** Finally, download the pdf
(defvar bibtex-completion-library-path)
(defvar bibtex-completion-bibliography)
(declare-function async-start "async")

;;;###autoload
(defun doi-utils-async-download-pdf ()
  "Download the PDF for bibtex entry at point asynchronously.
It is not fully async, only the download is. Fully async is
harder because you need to run `doi-utils-get-pdf-url' async
too. "
  (interactive)
  (require 'async)
  (save-excursion
    (bibtex-beginning-of-entry)
    (let (;; get doi, removing http://dx.doi.org/ if it is there.
          (doi (replace-regexp-in-string
                "https?://\\(dx.\\)?.doi.org/" ""
                (bibtex-autokey-get-field "doi")))
          (base-name (funcall doi-utils-pdf-filename-function))
          (pdf-url)
          (pdf-file))

      (setq pdf-file
	    (concat (cond
		     ((stringp bibtex-completion-library-path)
		      bibtex-completion-library-path)
		     ((= 1 (length bibtex-completion-library-path))
		      (car bibtex-completion-library-path))
		     (t
		      (completing-read "Dir: " bibtex-completion-library-path)))
		    base-name ".pdf"))

      (unless doi (error "No DOI found to get a pdf for"))

      (when (file-exists-p pdf-file)
	(error "%s already exists. Delete to re-download" pdf-file))

      ;; (doi-utils-get-pdf-url "10.1063/1.5019667")
      ;; If you get here, try getting the pdf file
      (async-start
       `(lambda ()
	  (setq package-user-dir ,package-user-dir)
	  (require 'package)
	  (package-initialize)
	  (setq load-path (list ,@load-path))
	  (require 'doi-utils)

	  (setq pdf-url (doi-utils-get-pdf-url ,doi))
	  (when pdf-url
	    (url-copy-file pdf-url ,pdf-file t)

	    (let* ((header (with-temp-buffer
			     (set-buffer-multibyte nil)
			     (insert-file-contents-literally ,pdf-file nil 0 5)
			     (buffer-string)))
		   (valid (and (stringp header)
			       (string-equal (encode-coding-string header 'utf-8) "%PDF-"))))
	      (if valid
		  (format "%s downloaded" ,pdf-file)
		(delete-file ,pdf-file)
		(require 'browse-url)
		(browse-url pdf-url)
		(message "Invalid pdf (file deleted). Header = %s" header)))))
       `(lambda (result)
	  (message "doi-utils-async-download-pdf: %s"  result))))))


;;;###autoload
(defun doi-utils-get-bibtex-entry-pdf (&optional arg)
  "Download pdf for entry at point if the pdf does not already exist locally.
The entry must have a doi. The pdf will be saved, by the name
%s.pdf where %s is the bibtex label. Files will not be
overwritten. The pdf will be checked to make sure it is a pdf,
and not some html failure page. You must have permission to
access the pdf. We open the pdf at the end if
`doi-utils-open-pdf-after-download' is non-nil.

With one prefix ARG, directly get the pdf from a file (through
`read-file-name') instead of looking up a DOI. With a double
prefix ARG, directly get the pdf from an open buffer (through
`read-buffer-to-switch') instead. These two alternative methods
work even if the entry has no DOI, and the pdf file is not
checked."
  (interactive "P")
  (save-excursion
    (bibtex-beginning-of-entry)
    (let (;; get doi, removing http://dx.doi.org/ if it is there.
          (doi (replace-regexp-in-string
                "https?://\\(dx.\\)?.doi.org/" ""
                (bibtex-autokey-get-field "doi")))
          (base-name (funcall doi-utils-pdf-filename-function))
          (pdf-url)
          (pdf-file))

      (setq pdf-file
	    (concat (cond
		     ((stringp bibtex-completion-library-path)
		      bibtex-completion-library-path)
		     ((= 1 (length bibtex-completion-library-path))
		      (car bibtex-completion-library-path))
		     (t
		      (completing-read "Dir: " bibtex-completion-library-path)))
		    base-name ".pdf"))
      ;; now get file if needed.
      (unless (file-exists-p pdf-file)
	(cond
	 ((and (not arg)
	       doi
	       (setq pdf-url (doi-utils-get-pdf-url doi)))
	  (url-copy-file pdf-url pdf-file)
	  ;; now check if we got a pdf
          (if (org-ref-pdf-p pdf-file)
              (message "%s saved" pdf-file)
            (delete-file pdf-file)
            (message "No pdf was downloaded.")
            (browse-url pdf-url)))
	 ((equal arg '(4))
	  (copy-file (expand-file-name (read-file-name "Pdf file: " nil nil t))
		     pdf-file))
	 ((equal arg '(16))
	  (with-current-buffer (read-buffer-to-switch "Pdf buffer: ")
	    (write-file pdf-file)))
	 (t
	  (message "We don't have a recipe for this journal.")))

	(when (file-exists-p pdf-file)
	  (bibtex-set-field "file" pdf-file))

	(when (and doi-utils-open-pdf-after-download (file-exists-p pdf-file))
	  (org-open-file pdf-file))))))

;;* Getting bibtex entries from a DOI

;; [[http://homepages.see.leeds.ac.uk/~eeaol/notes/2013/02/doi-metadata/][found]]
;; you can download metadata about a DOI from http://dx.doi.org. You just have
;; to construct the right http request to get it. Here is a function that gets
;; the metadata as a plist in emacs.
;;
;;
(defvar doi-utils-cache nil
  "Cache variable for storing data we can reuse.
A-list (doi . data) where doi is doi string, and data is what is
retrieved from it. This is transient, and disappears when you
restart Emacs. This mostly exists to prevent
`doi-utils-update-field' from needing to download the data for
every field.")


(defun doi-utils-clear-cache ()
  "Clear `doi-utils-cache'."
  (interactive)
  (setq doi-utils-cache '()))


(defun doi-utils-get-json-metadata (doi)
  "Try to get json metadata for DOI.  Open the DOI in a browser if we do not get it."
  (if-let* ((data (cdr (assoc doi doi-utils-cache))))
      ;; We have the data already, so we return it.
      data
    (let ((url-request-method "GET")
          (url-mime-accept-string "application/citeproc+json")
          (json-object-type 'plist)
          (json-data)
	  (url (concat doi-utils-dx-doi-org-url doi)))
      (with-temp-buffer
	(url-insert
	 (url-retrieve-synchronously url))
	(setq json-data (buffer-string))

	(when (or (string-match "<title>Error: DOI Not Found</title>" json-data)
		  (string-match "Resource not found" json-data)
		  (string-match "Status *406" json-data)
		  (string-match "400 Bad Request" json-data))
	  (browse-url (concat doi-utils-dx-doi-org-url doi))
	  (error "Something went wrong.  We got this response:
%s

Opening %s" json-data url))

	(setq data (json-read-from-string json-data))
	(cl-pushnew (cons doi data) doi-utils-cache)
	data))))


(defun doi-utils-get-json-metadata-curl (doi)
  "Try to get json metadata for DOI.  Open the DOI in a browser if we do not get it."
  (let ((json-object-type 'plist)
        (json-data)
	(url (concat doi-utils-dx-doi-org-url doi)))
    (with-temp-buffer
      (call-process "curl" nil t nil
                    "--location"
                    "--silent"
                    "--header"
                    "Accept: application/citeproc+json"
                    url)
      (setq json-data (buffer-string))
      (cond
       ((or (string-match "<title>Error: DOI Not Found</title>" json-data)
	    (string-match "Resource not found" json-data)
	    (string-match "Status *406" json-data)
	    (string-match "400 Bad Request" json-data))
	(browse-url url)
	(error "Something went wrong.  We got this response:
%s
Opening %s" json-data url))
       ;; everything seems ok with the data
       (t
	(json-read-from-string json-data))))))

;; We can use that data to construct a bibtex entry. We do that by defining a
;; template, and filling it in. I wrote this template expansion code which
;; makes it easy to substitute values like %{} in emacs lisp.


(defun doi-utils-expand-template (s)
  "Expand a string template S containing %{} with the eval of its contents."
  (replace-regexp-in-string "%{\\([^}]+\\)}"
			    (lambda (arg)
			      (let ((sexp (substring arg 2 -1)))
				(format "%s" (eval (read sexp)))))
			    s))


;; Now we define a function that fills in that template from the metadata.

;; As different bibtex types share common keys, it is advantageous to separate
;; data extraction from json, and the formatting of the bibtex entry.

;; We use eval-and-compile because we use the three following forms in the
;; `doi-utils-def-bibtex-type' macro.  Since the macro is expanded at compile
;; time, we need to ensure these defuns and defvars are evaluated at
;; compile-time.
(eval-and-compile
  (defvar doi-utils-json-metadata-extract
    '((type       (plist-get results :type))
      (author     (mapconcat (lambda (x)
			       (message "%s" x)
			       (if (plist-get x :name)
				   (plist-get x :name)
				 (concat (plist-get x :given) " " (plist-get x :family))))
                             (plist-get results :author) " and "))
      (title      (plist-get results :title))
      (subtitle   (plist-get results :subtitle))
      (journal    (plist-get results :container-title))
      (series     (plist-get results :container-title))
      (publisher  (plist-get results :publisher))
      (volume     (plist-get results :volume))
      (issue      (plist-get results :issue))
      (number     (plist-get results :issue))
      (year       (or (elt (elt (plist-get (plist-get results :issued) :date-parts) 0) 0)
                      (elt (elt (plist-get (plist-get results :approved) :date-parts) 0) 0)
                      ))
      ;; Some dates don't have a month in them.
      (month      (let ((date (elt
			       (plist-get (plist-get results :issued) :date-parts) 0)))
		    (if (>= (length date) 2)
			(elt date 1)
		      "-")))
      (pages      (or (plist-get results :page)
		      (plist-get results :article-number)))
      (doi        (plist-get results :DOI))
      (url        (plist-get results :URL))
      (booktitle  (plist-get results :container-title))
      (school     (or (plist-get results :school)
                      (plist-get (plist-get results :institution) :name)))
      ;; I am not sure how general this is. This gets the first name.
      (institution (plist-get (car (plist-get results :institution)) :name))))

  ;; Next, we need to define the different bibtex types. Each type has a bibtex
  ;; type (for output) and the type as provided in the doi record. Finally, we
  ;; have to declare the fields we want to output.

  (defvar doi-utils-bibtex-type-generators nil)

  (defun doi-utils-concat-prepare (lst &optional acc)
    "Minimize the number of args passed to `concat' from LST.
Given a list LST of strings and other expressions, which are
intended to be passed to `concat', concat any subsequent strings,
minimising the number of arguments being passed to `concat'
without changing the results.  ACC is the list of additional
expressions."
    (cond ((null lst) (nreverse acc))
          ((and (stringp (car lst))
                (stringp (car acc)))
           (doi-utils-concat-prepare (cdr lst) (cons (concat (car acc) (car lst))
                                                     (cdr acc))))
          (t (doi-utils-concat-prepare (cdr lst) (cons (car lst) acc))))))

(defmacro doi-utils-def-bibtex-type (name matching-types &rest fields)
  "Define a BibTeX type identified by (symbol) NAME.
MATCHING-TYPES is a list of strings.  FIELDS are symbols that
match to retrieval expressions in
`doi-utils-json-metadata-extract'.  This type will only be used
when the `:type' parameter in the JSON metadata is contained in
MATCHING-TYPES."
  `(push (lambda (type results)
           (when
               (or ,@(mapcar
                      (lambda (match-type)
                        `(string= type ,match-type)) matching-types))
             (let ,(mapcar (lambda (field)
                             (let ((field-expr
                                    (assoc field doi-utils-json-metadata-extract)))
                               (if field-expr
                                   ;; need to convert to string first
                                   `(,(car field-expr) (format "%s" ,(cadr field-expr)))
                                 (error "Unknown bibtex field type %s" field))))
                           fields)
               (concat
                ,@(doi-utils-concat-prepare
                   (org-ref--flatten-list
                    (list (concat "@" (symbol-name name) "{,\n")
                          ;; there seems to be some bug with mapcan,
                          ;; so we fall back to flatten
                          (mapcar (lambda (field)
                                    `("  " ,(symbol-name field) " = {" ,field "},\n"))
                                  fields)
                          "}\n")))))))
         doi-utils-bibtex-type-generators))

(doi-utils-def-bibtex-type article ("journal-article" "article-journal" "article")
                           author title journal year volume number pages doi url)

(doi-utils-def-bibtex-type inproceedings ("proceedings-article" "paper-conference")
                           author title booktitle year month pages doi url)

(doi-utils-def-bibtex-type book ("book" "edited-book")
                           author title series publisher year pages doi url)

(doi-utils-def-bibtex-type inbook ("chapter" "book-chapter" "reference-entry")
                           author title booktitle series publisher year pages doi url)

(doi-utils-def-bibtex-type phdthesis ("phdthesis" "thesis" "dissertation")
			   author title school publisher year)

(doi-utils-def-bibtex-type techreport ("report")
			   institution author title publisher year doi url)

;; this is what preprints in chemrxiv look like for now
(doi-utils-def-bibtex-type misc ("posted-content")
			   author title year doi url)



;; With the code generating the bibtex entry in place, we can glue it to the json retrieval code.

(defun doi-utils-doi-to-bibtex-string (doi)
  "Return a bibtex entry as a string for the DOI.  Not all types are supported yet."
  (let* ((results (funcall doi-utils-metadata-function doi))
         (type (plist-get results :type)))
    ;; (format "%s" results) ; json-data
    (or (cl-some (lambda (g) (funcall g type results)) doi-utils-bibtex-type-generators)
        (message "%s not supported yet\n%S." type results))))

;; That is just the string for the entry. To be useful, we need a function that
;; inserts the string into a buffer. This function will insert the string at the
;; cursor, clean the entry, try to get the pdf.

(defun doi-utils-insert-bibtex-entry-from-doi (doi)
  "Insert and clean bibtex entry from a DOI."
  (insert (doi-utils-doi-to-bibtex-string doi))
  (backward-char)
  ;; set date added for the record
  (let ((ts (funcall doi-utils-timestamp-format-function)))
    (when ts
      (bibtex-set-field doi-utils-timestamp-field
			ts)))
  (org-ref-clean-bibtex-entry)
  (when (buffer-file-name)
      (save-buffer)))


;;;###autoload
(defun doi-utils-add-bibtex-entry-from-doi (doi &optional bibfile)
  "Add DOI entry to end of a file in the current directory.
Pick the file ending with .bib or in .  If you have an active region that
starts like a DOI, that will be the initial prompt.  If no region
is selected and the first entry of the ‘kill-ring’ starts like a
DOI, then that is the initial prompt.  Otherwise, you have to type
or paste in a DOI.
Argument BIBFILE the bibliography to use."
  (interactive
   (list (read-string
          "DOI: "
          ;; now set initial input
          (doi-utils-maybe-doi-from-region-or-current-kill))))

  (unless bibfile
    (setq bibfile (completing-read "Bibfile: " (org-ref-possible-bibfiles))))
  ;; Wrap in save-window-excursion to restore your window arrangement after this
  ;; is done.
  (save-window-excursion
    (with-current-buffer
        (find-file-noselect bibfile)
      ;; Check if the doi already exists
      (goto-char (point-min))
      (if (re-search-forward (concat doi "\\_>") nil t)
          (message "%s is already in this file" doi)
        (goto-char (point-max))

	(when (not (looking-back "\n\n" (min 3 (point))))
	  (insert "\n\n"))

        (doi-utils-insert-bibtex-entry-from-doi doi)
        (save-buffer)))))

(defalias 'doi-add-bibtex-entry 'doi-utils-add-bibtex-entry-from-doi
  "Alias function for convenience.")

(defun doi-utils-maybe-doi-from-region-or-current-kill ()
  "Try to get a DOI from the active region or current kill."
  (let* ((the-active-region (if (region-active-p) ;; nil if no active region
                                (buffer-substring (region-beginning) (region-end))
                              nil))
         (the-current-kill (ignore-errors (current-kill 0 t)))  ;; nil if empty kill ring
         ;; DOI urls
         ;; Ex: https://doi.org/10.1109/MALWARE.2014.6999410
         ;; Ex: https://dx.doi.org/10.1007/978-3-319-60876-1_10
         (doi-url-prefix-regexp "^https?://\\(dx\\.\\)?doi\\.org/")
         ;; https://www.crossref.org/blog/dois-and-matching-regular-expressions/
         (doi-regexp "10\\.[0-9]\\{4,9\\}/[-._;()/:A-Z0-9]+$"))
    (cond
     ;; Check if a DOI can be found in the active region
     ;; DOI raw
     ;; Ex: 10.1109/MALWARE.2014.6999410
     ((and (stringp the-active-region)
           (org-ref--string-match (concat "^" doi-regexp) the-active-region))
      the-active-region)
     ;; DOI url
     ;; Ex: https://doi.org/10.1109/MALWARE.2014.6999410
     ((and (stringp the-active-region)
           (org-ref--string-match (concat doi-url-prefix-regexp doi-regexp) the-active-region))
      (replace-regexp-in-string doi-url-prefix-regexp "" the-active-region))
     ;; DOI url as customized
     ((and (stringp the-active-region)
           (org-ref--string-match (regexp-quote doi-utils-dx-doi-org-url) the-active-region))
      (replace-regexp-in-string (regexp-quote doi-utils-dx-doi-org-url) "" the-active-region))
     ;; Check if DOI can be found in the current kill
     ;; DOI raw
     ;; Ex: 10.1109/MALWARE.2014.6999410
     ((and (stringp the-current-kill)
           (org-ref--string-match (concat "^" doi-regexp) the-current-kill))
      the-current-kill)
     ;; DOI url
     ;; Ex: https://doi.org/10.1109/MALWARE.2014.6999410
     ((and (stringp the-current-kill)
           (org-ref--string-match (concat doi-url-prefix-regexp doi-regexp) the-current-kill))
      (replace-regexp-in-string doi-url-prefix-regexp "" the-current-kill))
     ;; DOI url as customized
     ((and (stringp the-current-kill)
           (org-ref--string-match (regexp-quote doi-utils-dx-doi-org-url) the-current-kill))
      (replace-regexp-in-string (regexp-quote doi-utils-dx-doi-org-url) "" the-current-kill))
     ;; otherwise, return nil
     (t
      nil))))

;;;###autoload
(defun doi-utils-doi-to-org-bibtex (doi)
  "Convert a DOI to an ‘org-bibtex’ form and insert it at point."
  (interactive "sDOI: ")
  (with-temp-buffer
    (insert (doi-utils-doi-to-bibtex-string doi))
    (bibtex-clean-entry)
    (kill-region (point-min) (point-max)))
  (org-bibtex-yank)
  (org-metaright)
  (org-metaright))

;;* Updating bibtex entries

;; I wrote this code because it is pretty common for me to copy bibtex entries
;; from ASAP articles that are incomplete, e.g. no page numbers because it is
;; not in print yet. I wanted a convenient way to update an entry from its DOI.
;; Basically, we get the metadata, and update the fields in the entry.

;; There is not bibtex set field function, so I wrote this one.


;;;###autoload
(defun bibtex-set-field (field value &optional nodelim)
  "Set FIELD to VALUE in bibtex file.  create field if it does not exist.
Optional argument NODELIM see `bibtex-make-field'."
  (interactive "sfield: \nsvalue: ")
  (bibtex-beginning-of-entry)
  (let ((found))
    (if (setq found (bibtex-search-forward-field field t))
        ;; we found a field
        (progn
          (goto-char (car (cdr found)))
          (when value
            (bibtex-kill-field)
            (bibtex-make-field field nil nil nodelim)
            (backward-char)
            (insert value)))
      ;; make a new field
      (bibtex-beginning-of-entry)
      (forward-line) (beginning-of-line)
      (bibtex-next-field nil)
      (forward-char)
      (bibtex-make-field field nil nil nodelim)
      (backward-char)
      (insert value))))


(defun plist-get-keys (plist)
  "Return keys in a PLIST."
  (cl-loop for (key _value) on plist by #'cddr collect key))


;;;###autoload
(defun doi-utils-update-bibtex-entry-from-doi (doi)
  "Update fields in a bibtex entry from the DOI.
Every field will be updated, so previous change will be lost."
  (interactive (list
                (or (replace-regexp-in-string
                     "https?://\\(dx.\\)?doi.org/" ""
                     (bibtex-autokey-get-field "doi"))
                    (read-string "DOI: "))))
  (let* ((results (funcall doi-utils-metadata-function doi))
         (type (plist-get results :type))
         (author (mapconcat
                  (lambda (x)
		    ;; There are two possible ways an author is named. The most
		    ;; common is with :given and :family, but sometimes there is
		    ;; :name instead.
		    (if (plist-get x :name)
			(plist-get x :name)
		      (concat (plist-get x :given)
			      " " (plist-get x :family))))
                  (plist-get results :author) " and "))
         (title (plist-get results :title))
         (journal (plist-get results :container-title))
         (year (format "%s"
                       (elt
                        (elt
                         (plist-get
                          (plist-get results :issued) :date-parts) 0) 0)))
         (volume (plist-get results :volume))
         (number (or (plist-get results :issue) ""))
         (pages (or (plist-get results :page) ""))
         (url (or (plist-get results :URL) ""))
         (doi (plist-get results :DOI))
         mapping)

    ;; map the json fields to bibtex fields. The code each field is mapped to is
    ;; evaluated.
    (setq mapping '((:author . (bibtex-set-field "author" author))
                    (:title . (bibtex-set-field "title" title))
                    (:container-title . (bibtex-set-field "journal" journal))
                    (:issued . (bibtex-set-field "year" year))
                    (:volume . (bibtex-set-field "volume" volume))
                    (:issue . (bibtex-set-field "number" number))
                    (:page . (bibtex-set-field "pages" pages))
                    (:DOI . (bibtex-set-field "doi" doi))
                    (:URL . (bibtex-set-field "url" url))))

    ;; now we have code to run for each entry. we map over them and evaluate the code
    (mapc
     (lambda (key)
       (eval (cdr (assoc key mapping))))
     (plist-get-keys results)))

  (org-ref-clean-bibtex-entry))


;; A downside to updating an entry is it overwrites what you have already fixed.
;; So, we next develop a function to update the field at point.


;;;###autoload
(defun doi-utils-update-field ()
  "Update the field at point in the bibtex entry.
Data is retrieved from the doi in the entry."
  (interactive)
  (let* ((doi (bibtex-autokey-get-field "doi"))
         (results (funcall doi-utils-metadata-function doi))
         (field (car (bibtex-find-text-internal nil nil ","))))
    (cond
     ((string= field "volume")
      (bibtex-set-field field (plist-get results :volume)))
     ((string= field "number")
      (bibtex-set-field field (plist-get results :issue)))
     ((string= field "pages")
      (bibtex-set-field field (or (plist-get results :page)
                                  (plist-get results :article-number))))
     ((string= field "year")
      (bibtex-set-field field (plist-get results :year)))
     (t
      (message "%s not supported yet." field)))))



;;* DOI functions for WOS

;; I came across this API http://wokinfo.com/media/pdf/OpenURL-guide.pdf to make
;; links to the things I am interested in here. Based on that document, here are
;; three links based on a doi:10.1021/jp047349j that take you to different Web
;; Of Science (WOS) pages.


;; 1. go to article in WOS: http://ws.isiknowledge.com/cps/openurl/service?url_ver=Z39.88-2004&rft_id=info:doi/10.1021/jp047349j
;; 2. citing articles: http://ws.isiknowledge.com/cps/openurl/service?url_ver=Z39.88-2004&rft_id=info%3Adoi%2F10.1021/jp047349j&svc_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Asch_svc&svc.citing=yes
;; 3. related articles: http://ws.isiknowledge.com/cps/openurl/service?url_ver=Z39.88-2004&rft_id=info%3Adoi%2F10.1021/jp047349j&svc_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Asch_svc&svc.related=yes

;; These are pretty easy to construct, so we can write functions that will
;; create them and open the url in our browser. There are some other options
;; that could be considered, but since we usually have a doi, it seems like the
;; best way to go for creating the links. Here are the functions.

;;;###autoload
(defun doi-utils-wos (doi)
  "Open Web of Science entry for DOI."
  (interactive "sDOI: ")
  (browse-url
   (format
    "http://ws.isiknowledge.com/cps/openurl/service?url_ver=Z39.88-2004&rft_id=info:doi/%s" doi)))

;;;###autoload
(defun doi-utils-wos-citing (doi)
  "Open Web of Science citing articles entry for DOI.
May be empty if none are found."
  (interactive "sDOI: ")
  (browse-url
   (concat
    "http://ws.isiknowledge.com/cps/openurl/service?url_ver=Z39.88-2004&rft_id=info%3Adoi%2F"
    doi
    "&svc_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Asch_svc&svc.citing=yes")))

;;;###autoload
(defun doi-utils-wos-related (doi)
  "Open Web of Science related articles page for DOI."
  (interactive "sDOI: ")
  (browse-url
   (concat "http://ws.isiknowledge.com/cps/openurl/service?url_ver=Z39.88-2004&rft_id=info%3Adoi%2F"
           doi
           "&svc_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Asch_svc&svc.related=yes")))


;;* DOI functions for ADS

;;;###autoload
(defun doi-utils-ads (doi)
  "Open ADS entry for DOI"
  (interactive "sDOI: ")
  (browse-url
   (concat
    "https://ui.adsabs.harvard.edu/abs/" "%22" doi "%22")))


;;* A new doi link for org-mode
;; The idea is to add a menu to the doi link, so rather than just clicking to open the article, you can do other things.
;; 1. open doi
;; 2. open in wos
;; 3. open citing articles
;; 4. open related articles
;; 5. open bibtex entry
;; 6. get bibtex entry


;;;###autoload
(defun doi-utils-open (doi)
  "Open DOI in browser."
  (interactive "sDOI: ")
  (browse-url (concat doi-utils-dx-doi-org-url doi)))


;;;###autoload
(defun doi-utils-open-bibtex (doi)
  "Search through variable `bibtex-completion-bibliography' for DOI."
  (interactive "sDOI: ")
  (cl-loop for f in (org-ref-normalize-bibtex-completion-bibliography)
	   when (progn (find-file f)
		       (when (search-forward doi (point-max) t)
			 (bibtex-beginning-of-entry)))
	   return f))


;;;###autoload
(defun doi-utils-crossref (doi)
  "Search DOI in CrossRef."
  (interactive "sDOI: ")
  (browse-url
   (format
    "http://search.crossref.org/?q=%s" doi)))


;;;###autoload
(defun doi-utils-google-scholar (doi)
  "Google scholar the DOI."
  (interactive "sDOI: ")
  (browse-url
   (format
    "http://scholar.google.com/scholar?q=%s" doi)))


;;;###autoload
(defun doi-utils-pubmed (doi)
  "Search Pubmed for the DOI."
  (interactive "sDOI: ")
  (browse-url
   (format
    "http://www.ncbi.nlm.nih.gov/pubmed/?term=%s"
    (url-hexify-string doi))))

(declare-function org-element-property "org-element")

(defun doi-utils--context-doi ()
  (org-element-property :path (org-element-context)))

(transient-define-prefix doi-link-follow-menu ()
  "DOI actions."
  [["Actions"
    ("o" "open" (lambda () (interactive)
                  (doi-utils-open (doi-utils--context-doi))))
    ("w" "wos" (lambda () (interactive)
                 (doi-utils-wos (doi-utils--context-doi))))
    ("c" "wos citing articles" (lambda () (interactive)
                                  (doi-utils-wos-citing (doi-utils--context-doi))))
    ("r" "wos related articles" (lambda () (interactive)
                                   (doi-utils-wos-related (doi-utils--context-doi))))
    ("a" "ads" (lambda () (interactive)
                 (doi-utils-ads (doi-utils--context-doi))))
    ("s" "Google Scholar" (lambda () (interactive)
                            (doi-utils-google-scholar (doi-utils--context-doi))))
    ("f" "CrossRef" (lambda () (interactive)
                      (doi-utils-crossref (doi-utils--context-doi))))
    ("p" "Pubmed" (lambda () (interactive)
                    (doi-utils-pubmed (doi-utils--context-doi))))
    ("b" "open in bibtex" (lambda () (interactive)
                             (doi-utils-open-bibtex (doi-utils--context-doi))))
    ("g" "get bibtex entry" (lambda () (interactive)
                               (doi-utils-add-bibtex-entry-from-doi (doi-utils--context-doi))))
    ("q" "quit" transient-quit-one)]])

(define-obsolete-function-alias 'doi-link-follow/body
  #'doi-link-follow-menu "3.1")


(org-link-set-parameters "doi"
			 :follow (lambda (_) (doi-link-follow-menu))
			 :export (lambda (doi desc format)
				   (cond
				    ((eq format 'html)
				     (format "<a href=\"%s%s\">%s</a>"
					     doi-utils-dx-doi-org-url
					     doi
					     (or desc (concat "doi:" doi))))
				    ((eq format 'latex)
				     (format "\\href{%s%s}{%s}"
					     doi-utils-dx-doi-org-url
					     doi
					     (or desc (concat "doi:" doi)))))))

;;* Getting a doi for a bibtex entry missing one

;; Some bibtex entries do not have a DOI, maybe because they were entered by
;; hand, or copied from a source that did not have it available. Here we develop
;; some functions to help you find the DOI using Crossref.

;; Here is our example bibtex entry.
;; #+BEGIN_SRC bibtex
;; @article{deml-2014-oxide,
;;   author =	 {Ann M. Deml and Vladan Stevanovi{\'c} and
;;                   Christopher L. Muhich and Charles B. Musgrave and
;;                   Ryan O'Hayre},
;;   title =	 {Oxide Enthalpy of Formation and Band Gap Energy As
;;                   Accurate Descriptors of Oxygen Vacancy Formation
;;                   Energetics},
;;   journal =	 {Energy Environ. Sci.},
;;   volume =	 7,
;;   number =	 6,
;;   pages =	 1996,
;;   year =	 2014,
;;   doi =		 {10.1039/c3ee43874k,
;;   url =		 {http://dx.doi.org/10.1039/c3ee43874k}},

;; }


;; The idea is to query Crossref in a way that is likely to give us a hit
;; relevant to the entry.

;; According to http://search.crossref.org/help/api we can send a query with a
;; free form citation that may give us something back. We do this to get a list
;; of candidates, which could be used to get the doi.

(declare-function org-ref-bib-citation "org-ref-bibtex")
;;;###autoload
(defun doi-utils-crossref-citation-query ()
  "Query Crossref with the title of the bibtex entry at point.
Get a list of possible matches. Choose one with completion."
  (interactive)
  (bibtex-beginning-of-entry)
  (let* ((entry (bibtex-parse-entry))
	 (raw-json-string)
         (json-string)
         (json-data)
         (doi (bibtex-autokey-get-field "doi")))
    (unless (string= "" doi)
      (error "Entry already has a doi field"))

    (with-current-buffer
        (url-retrieve-synchronously
         (concat
          "http://search.crossref.org/dois?q="
          (url-hexify-string (org-ref-bib-citation))))
      (save-excursion
      	(goto-char (point-min))
      	(while (re-search-forward "<i>\\|</i>" nil t)
      	  (replace-match ""))
	(goto-char (point-min))
	(while (re-search-forward "&amp;" nil t)
	  (replace-match "&"))
      	(goto-char (point-min))
      	(while (re-search-forward "&quot;" nil t)
      	  (replace-match "\\\"" nil t)))
      (setq raw-json-string (buffer-substring url-http-end-of-headers (point-max)))
      ;; decode json string
      (setq json-string (decode-coding-string (encode-coding-string raw-json-string 'utf-8) 'utf-8))
      (setq json-data (json-read-from-string json-string)))

    (let* ((name (format "Crossref hits for %s" (org-ref-bib-citation)))
           (candidates (mapcar (lambda (x)
                                 (cons
                                  (concat
                                   (cdr (assoc 'fullCitation x)))
                                  (cdr (assoc 'doi x))))
                               json-data))

	   (doi (cdr (assoc (completing-read "DOI: " candidates) candidates))))
      (bibtex-make-field "doi" t)
      (backward-char)
      ;; crossref returns doi url, but I prefer only a doi for the doi field
      (insert (replace-regexp-in-string "^https?://\\(dx.\\)?doi.org/" "" doi))
      (when (string= "" (bibtex-autokey-get-field "url"))
        (bibtex-make-field "url" t)
        (backward-char)
        (insert doi)))))



;;* Debugging a DOI

;; I wrote this function to help debug a DOI. This function generates an
;; org-buffer with the doi, gets the json metadata, shows the bibtex entry, and
;; the pdf link for it.
(defun doi-utils-get-json (doi)
  "Return json data as a string for DOI."
  (let ((url-request-method "GET")
        (url-mime-accept-string "application/citeproc+json")
        (json-data))
    (with-temp-buffer
      (url-insert
       (url-retrieve-synchronously
	(concat doi-utils-dx-doi-org-url doi)))
      (setq json-data (buffer-string))
      (if (string-match "Resource not found" json-data)
          (progn
            (browse-url (concat doi-utils-dx-doi-org-url doi))
            (error "Resource not found.  Opening website"))
	json-data))))


;;;###autoload
(defun doi-utils-debug (doi)
  "Generate an org-buffer showing data about DOI."
  (interactive "sDOI: ")
  (switch-to-buffer "*debug-doi*")
  (erase-buffer)
  (org-mode)
  (insert (concat "doi:" doi) "\n\n")
  (insert "* JSON
"
	  (let ((url-request-method "GET")
		(url-mime-accept-string "application/citeproc+json"))
	    (pp
	     (json-read-from-string (with-temp-buffer
				      (url-insert
				       (url-retrieve-synchronously
					(concat doi-utils-dx-doi-org-url doi)))
				      (buffer-string)))))
	  "\n\n")
  (goto-char (point-min)))

;;* Adding a bibtex entry from a crossref query

;; The idea here is to perform a query on Crossref, get a completion buffer of
;; candidates, and select the entry(ies) you want to add to your bibtex file.
;; You can select a region, e.g. a free form citation, or set of words, or you
;; can type the query in by hand.

;;;###autoload
(defun doi-utils-add-entry-from-crossref-query (query bibtex-file)
  "Search Crossref with QUERY and use completion to select an entry to add to BIBTEX-FILE."
  (interactive (list
                (read-string
                 "Query: "
                 ;; now set initial input
                 (cond
                  ;; If region is active assume we want it
                  ((region-active-p)
                   (replace-regexp-in-string
                    "\n" " "
                    (buffer-substring (region-beginning) (region-end))))
                  ;; type or paste it in
                  (t
                   nil)))
                (completing-read
                 "Bibfile: "
                 (append (org-ref--directory-files "." (lambda (f) (org-ref--file-ext-p f "bib")))
                         bibtex-completion-bibliography))))
  (let* ((json-data (with-temp-buffer
		      (url-insert
		       (url-retrieve-synchronously
			(concat
			 "https://api.crossref.org/works?query="
			 (url-hexify-string query))))
		      
		      (json-read-from-string (buffer-string))))
	 (name (format "Crossref hits for %s"
		       ;; remove carriage returns. They can make completion confusing.
		       (replace-regexp-in-string "\n" " " query)))
	 (candidates (let-alist json-data
		       (cl-loop for item across .message.items
				collect (let-alist item
					  (cons (format "%s, %s, %s, %s."
							(string-join .title " ")
							(string-join
							 (cl-loop for author across .author collect
								  (let-alist author
								    (format "%s %s"
									    .given .family)))
							 ", ")
							.publisher
							.created.date-parts)			       
						.DOI)))))
	 (doi (cdr (assoc (completing-read "Choice: " candidates) candidates))))

    (with-current-buffer (find-file-noselect bibtex-file)
      (doi-utils-add-bibtex-entry-from-doi
       (replace-regexp-in-string
	"^https?://\\(dx.\\)?doi.org/" "" doi)
       bibtex-file)
      (save-buffer))))


(defalias 'crossref-add-bibtex-entry 'doi-utils-add-entry-from-crossref-query
  "Alias function for convenience.")

;; * Convenience

(defun doi-utils-toggle-pdf-download ()
  "Toggle the setting of `doi-utils-download-pdf'.
I find this useful when downloading the pdfs slows down adding a
lot of references; then you just toggle it off."
  (interactive)
  (message "Setting doi-utils-download-pdf to %s"
	   (setq doi-utils-download-pdf (not doi-utils-download-pdf))))

;;* The end
(provide 'doi-utils)
;;; doi-utils.el ends here