+# -*- org-edit-src-content-indentation: 0; -*-
#+TITLE: DOI utilities for making bibtex entries and downloading pdfs
This package provides functionality to download PDFs and bibtex entries from a DOI, as well as to update a bibtex entry from a DOI. It depends slightly on org-ref, to determine where to save pdf files too, and where to insert bibtex entries in the default bibliography.
#+END_SRC
#+RESULTS:
-| :volume | 99 | :indexed | (:timestamp 1399964115538.0 :date-parts [[2014 5 13]]) | :publisher | American Physical Society (APS) | :source | CrossRef | :URL | http://dx.doi.org/10.1103/PhysRevLett.99.016105 | :ISSN | [0031-9007 1079-7114] | :DOI | 10.1103/physrevlett.99.016105 | :type | journal-article | :title | Scaling Properties of Adsorption Energies for Hydrogen-Containing Molecules on Transition-Metal Surfaces | :issue | 1 | :deposited | (:timestamp 1313712000000.0 :date-parts [[2011 8 19]]) | :reference-count | 26 | :container-title | Phys. Rev. Lett. | :author | [(:given F. :family Abild-Pedersen) (:given J. :family Greeley) (:given F. :family Studt) (:given J. :family Rossmeisl) (:given T. :family Munter) (:given P. :family Moses) (:given E. :family Skúlason) (:given T. :family Bligaard) (:given J. :family Nørskov)] | :prefix | http://id.crossref.org/prefix/10.1103 | :score | 1.0 | :issued | (:date-parts [[2007 7]]) | :subject | [Physics and Astronomy(all)] | :subtitle | [] |
+| :member | http://id.crossref.org/member/16 | :volume | 99 | :indexed | (:timestamp 1423435577602 :date-parts [[2015 2 8]]) | :publisher | American Physical Society (APS) | :source | CrossRef | :URL | http://dx.doi.org/10.1103/PhysRevLett.99.016105 | :ISSN | [0031-9007 1079-7114] | :DOI | 10.1103/physrevlett.99.016105 | :type | journal-article | :title | Scaling Properties of Adsorption Energies for Hydrogen-Containing Molecules on Transition-Metal Surfaces | :issue | 1 | :deposited | (:timestamp 1313712000000 :date-parts [[2011 8 19]]) | :reference-count | 26 | :container-title | Phys. Rev. Lett. | :author | [(:given F. :family Abild-Pedersen) (:given J. :family Greeley) (:given F. :family Studt) (:given J. :family Rossmeisl) (:given T. :family Munter) (:given P. :family Moses) (:given E. :family Skúlason) (:given T. :family Bligaard) (:given J. :family Nørskov)] | :prefix | http://id.crossref.org/prefix/10.1103 | :score | 1.0 | :issued | (:date-parts [[2007 7]]) | :subject | [Physics and Astronomy(all)] | :subtitle | [] |
+
+or for a book:
+#+BEGIN_SRC emacs-lisp :tangle no
+(doi-utils-get-json-metadata "10.1007/978-1-4612-4968-9")
+#+END_SRC
+
+#+RESULTS:
+| :member | nil | :indexed | (:timestamp 1423773021494 :date-parts [[2015 2 12]]) | :publisher | Springer New York | :source | CrossRef | :URL | http://dx.doi.org/10.1007/978-1-4612-4968-9 | :ISBN | [http://id.crossref.org/isbn/978-0-387-96347-1 http://id.crossref.org/isbn/978-1-4612-4968-9] | :ISSN | [0172-6056] | :DOI | 10.1007/978-1-4612-4968-9 | :type | book | :title | Constructive Combinatorics | :deposited | (:timestamp 1378684800000 :date-parts [[2013 9 9]]) | :reference-count | 0 | :container-title | Undergraduate Texts in Mathematics | :author | [(:given Dennis :family Stanton) (:given Dennis :family White)] | :prefix | none | :score | 1.0 | :issued | (:date-parts [[1986]]) | :subtitle | [] |
We can use that data to construct a bibtex entry. We do that by defining a template, and filling it in. I wrote this template expansion code which makes it easy to substitute values like %{} in emacs lisp.
Now we define a function that fills in that template from the metadata.
+As different bibtex types share common keys, it is advantageous to separate data extraction from json, and the formatting of the bibtex entry.
+
+#+BEGIN_SRC emacs-lisp :tangle doi-utils.el
+(setq doi-utils-json-metadata-extract
+ '((type (plist-get results :type))
+ (author (mapconcat (lambda (x) (concat (plist-get x :given) " " (plist-get x :family)))
+ (plist-get results :author) " and "))
+ (title (plist-get results :title))
+ (subtitle (plist-get results :subtitle))
+ (journal (plist-get results :container-title))
+ (series (plist-get results :container-title))
+ (publisher (plist-get results :publisher))
+ (volume (plist-get results :volume))
+ (issue (plist-get results :issue))
+ (number (plist-get results :issue))
+ (year (elt (elt (plist-get (plist-get results :issued) :date-parts) 0) 0))
+ (month (elt (elt (plist-get (plist-get results :issued) :date-parts) 0) 1))
+ (pages (plist-get results :page))
+ (doi (plist-get results :DOI))
+ (url (plist-get results :URL))
+ (booktitle (plist-get results :container-title))))
+#+END_SRC
+
+Next, we need to define the different bibtex types. Each type has a bibtex type (for output) and the type as provided in the doi record. Finally, we have to declare the fields we want to output.
+
+#+BEGIN_SRC emacs-lisp :tangle doi-utils.el :results none
+(setq doi-utils-bibtex-type-generators nil)
+
+(defun doi-utils-concat-prepare (lst &optional acc)
+ "Given a list `lst' of strings and other expressions, which are
+intented to passed to `concat', concat any subsequent strings,
+minimising the number of arguments being passed to `concat'
+without changing the results."
+ (cond ((null lst) (nreverse acc))
+ ((and (stringp (car lst))
+ (stringp (car acc)))
+ (doi-utils-concat-prepare (cdr lst) (cons (concat (car acc) (car lst))
+ (cdr acc))))
+ (t (doi-utils-concat-prepare (cdr lst) (cons (car lst) acc)))))
+
+(defmacro doi-utils-def-bibtex-type (name matching-types &rest fields)
+ "Define a BibTeX type identified by (symbol) `name' with
+`fields' (given as symbols), matching to retrieval expressions in
+`doi-utils-json-metadata-extract'. This type will only be used
+when the `:type' parameter in the JSON metadata is contained in
+`matching-types' - a list of strings."
+ `(push (lambda (type results)
+ (when (or ,@(mapcar (lambda (match-type) `(string= type ,match-type)) matching-types))
+ (let ,(mapcar (lambda (field)
+ (let ((field-expr (assoc field doi-utils-json-metadata-extract)))
+ (if field-expr
+ ;; need to convert to string first
+ `(,(car field-expr) (format "%s" ,(cadr field-expr)))
+ (error "unknown bibtex field type %s" field))))
+ fields)
+ (concat
+ ,@(doi-utils-concat-prepare
+ (-flatten
+ (list (concat "@" (symbol-name name) "{,\n")
+ ;; there seems to be some bug with mapcan,
+ ;; so we fall back to flatten
+ (mapcar (lambda (field)
+ `(" " ,(symbol-name field) " = {" ,field "},\n"))
+ fields)
+ "}\n")))))))
+ doi-utils-bibtex-type-generators))
+
+(doi-utils-def-bibtex-type article ("journal-article" "article-journal")
+ author title journal year volume number pages doi url)
+
+(doi-utils-def-bibtex-type inproceedings ("proceedings-article")
+ author title booktitle year month pages doi url)
+
+(doi-utils-def-bibtex-type book ("book")
+ author title series publisher year pages doi url)
+
+(doi-utils-def-bibtex-type inbook ("book-chapter")
+ author title booktitle series publisher year pages doi url)
+
+#+END_SRC
+
+With the code generating the bibtex entry in place, we can glue it to the json retrieval code.
#+BEGIN_SRC emacs-lisp :tangle doi-utils.el
(defun doi-utils-doi-to-bibtex-string (doi)
- "return a bibtex entry as a string for the doi. Only articles are currently supported"
- (let (type
- results
- author
- title
- booktitle
- journal
- year
- volume
- number
- pages
- month
- url
- json-data)
- (setq results (doi-utils-get-json-metadata doi)
- json-data (format "%s" results)
- type (plist-get results :type)
- author (mapconcat (lambda (x) (concat (plist-get x :given) " " (plist-get x :family)))
- (plist-get results :author) " and ")
- title (plist-get results :title)
- journal (plist-get results :container-title)
- volume (plist-get results :volume)
- issue (plist-get results :issue)
- year (elt (elt (plist-get (plist-get results :issued) :date-parts) 0) 0)
- pages (plist-get results :page)
- doi (plist-get results :DOI)
- url (plist-get results :URL))
- (cond
- ((or (string= type "journal-article") (string= type "article-journal"))
- (doi-utils-expand-template "@article{,
- author = {%{author}},
- title = {%{title}},
- journal = {%{journal}},
- year = {%{year}},
- volume = {%{volume}},
- number = {%{issue}},
- pages = {%{pages}},
- doi = {%{doi}},
- url = {%{url}},
-}"))
-
- ((string= type "proceedings-article")
- (setq booktitle (plist-get results :container-title))
- (doi-utils-expand-template "@inproceedings{,
- author = {%{author}},
- title = {%{title}},
- booktitle = {%{booktitle}},
- year = {%{year}},
- month = {%{month}},
- pages = {%{pages}},
- doi = {%{doi}},
- url = {%{url}},
-}"))
-
- (t (message-box "%s not supported yet." type)))))
+ "return a bibtex entry as a string for the doi. Not all types are supported yet."
+ (let* ((results (doi-utils-get-json-metadata doi))
+ (type (plist-get results :type)))
+ ;(format "%s" results) ; json-data
+ (or (some (lambda (g) (funcall g type results)) doi-utils-bibtex-type-generators)
+ (message "%s not supported yet\n%S." type results))))
#+END_SRC
#+RESULTS:
#+RESULTS:
#+begin_example
@article{,
- author = {F. Abild-Pedersen and J. Greeley and F. Studt and J. Rossmeisl and T. Munter and P. Moses and E. Skúlason and T. Bligaard and J. Nørskov},
- title = {Scaling Properties of Adsorption Energies for Hydrogen-Containing Molecules on Transition-Metal Surfaces},
- journal = {Phys. Rev. Lett.},
- year = {2007},
- volume = {99},
- number = {1},
- pages = {nil},
- doi = {10.1103/physrevlett.99.016105},
- url = {http://dx.doi.org/10.1103/PhysRevLett.99.016105},
+ author = {F. Abild-Pedersen and J. Greeley and F. Studt and J. Rossmeisl and T. Munter and P. Moses and E. Skúlason and T. Bligaard and J. Nørskov},
+ title = {Scaling Properties of Adsorption Energies for Hydrogen-Containing Molecules on Transition-Metal Surfaces},
+ journal = {Phys. Rev. Lett.},
+ year = {2007},
+ volume = {99},
+ number = {1},
+ pages = {nil},
+ doi = {10.1103/physrevlett.99.016105},
+ url = {http://dx.doi.org/10.1103/PhysRevLett.99.016105},
+}
+#+end_example
+
+and for a book:
+
+#+BEGIN_SRC emacs-lisp :tangle no
+(doi-utils-doi-to-bibtex-string "10.1007/978-1-4612-4968-9")
+#+END_SRC
+
+#+RESULTS:
+#+begin_example
+@book{,
+ author = {Dennis Stanton and Dennis White},
+ title = {Constructive Combinatorics},
+ series = {Undergraduate Texts in Mathematics},
+ publisher = {Springer New York},
+ year = {1986},
+ pages = {nil},
+ doi = {10.1007/978-1-4612-4968-9},
+ url = {http://dx.doi.org/10.1007/978-1-4612-4968-9},
}
#+end_example
(interactive "sDOI :")
(insert (doi-utils-doi-to-bibtex-string doi))
(backward-char)
+ ;; set date added for the record
+ (bibtex-set-field "DATE_ADDED" (current-time-string))
(if (bibtex-key-in-head nil)
(org-ref-clean-bibtex-entry t)
(org-ref-clean-bibtex-entry))
(doi-utils-get-bibtex-entry-pdf)
(save-selected-window
(org-ref-open-bibtex-notes)))
+
#+END_SRC
It may be you are in some other place when you want to add a bibtex entry. This next function will open the first entry in org-ref-default-bibliography go to the end, and add the entry. You can sort it later.
(s-match "^10" (buffer-substring
(region-beginning)
(region-end))))
- (buffer-susbstring (region-beginning) (region-end)))
+ (buffer-substring (region-beginning) (region-end)))
;; if the first entry in the kill-ring looks
;; like a DOI, let's use it.
((if (s-match "^10" (car kill-ring))
(insert value))))
#+END_SRC
-The updating function looks like this. We get all the keys from the json plist metadata, and update the fields if they exist.
+The updating function for a whole entry looks like this. We get all the keys from the json plist metadata, and update the fields if they exist.
#+BEGIN_SRC emacs-lisp :tangle doi-utils.el
(defun plist-get-keys (plist)
(org-ref-clean-bibtex-entry t)
(org-ref-clean-bibtex-entry)))
#+END_SRC
+
+A downside to updating an entry is it overwrites what you have already fixed. So, we next develop a function to update the field at point.
+
+#+BEGIN_SRC emacs-lisp
+(defun doi-utils-update-field ()
+ (interactive)
+ (let* ((doi (bibtex-autokey-get-field "doi"))
+ (results (doi-utils-get-json-metadata doi))
+ (field (car (bibtex-find-text-internal nil nil ","))))
+ (cond
+ ((string= field "volume")
+ (bibtex-set-field field (plist-get results :volume)))
+ ((string= field "number")
+ (bibtex-set-field field (plist-get results :issue)))
+ ((string= field "pages")
+ (bibtex-set-field field (plist-get results :page)))
+ ((string= field "year")
+ (bibtex-set-field field (plist-get results :year)))
+ (t
+ (message "%s not supported yet." field)))))
+#+END_SRC
+
+
* DOI functions for WOS
I came across this API http://wokinfo.com/media/pdf/OpenURL-guide.pdf to make links to the things I am interested in here. Based on that document, here are three links based on a doi:10.1021/jp047349j that take you to different Web Of Science (WOS) pages.
+* Debugging a DOI
+I wrote this function to help debug a DOI. This function generates an org-buffer with the doi, gets the json metadata, shows the bibtex entry, and the pdf link for it.
+
+#+BEGIN_SRC emacs-lisp :tangle doi-utils.el
+(defun doi-utils-debug (doi)
+ "Generate an org-buffer showing data about DOI."
+ (interactive "sDOI: ")
+ (switch-to-buffer "*debug-doi*")
+ (erase-buffer)
+ (org-mode)
+ (insert (concat "doi:" doi) "\n\n")
+ (insert "* JSON
+" (format "%s" (doi-utils-get-json-metadata doi)) "
+
+* Bibtex
+
+" (doi-utils-doi-to-bibtex-string doi) "
+
+* PDF
+" (doi-utils-get-pdf-url doi)))
+#+END_SRC
+
+#+RESULTS:
+: doi-utils-debug
+
* Adding a bibtex entry from a crossref query
+The idea here is to perform a query on Crossref, get a helm buffer of candidates, and select the entry(ies) you want to add to your bibtex file. You can select a region, e.g. a free form citation, or set of words, or you can type the query in by hand.
#+BEGIN_SRC emacs-lisp :tangle doi-utils.el
(defun doi-utils-add-entry-from-crossref-query (query bibtex-file)
(cond
;; If region is active assume we want it
((region-active-p)
- (buffer-susbstring (region-beginning) (region-end)))
+ (replace-regexp-in-string
+ "\n" " "
+ (buffer-substring (region-beginning) (region-end))))
;; type or paste it in
(t
nil)))
(setq json-string (buffer-substring url-http-end-of-headers (point-max)))
(setq json-data (json-read-from-string json-string)))
- (let* ((name (format "Crossref hits for %s" query))
+ (let* ((name (format "Crossref hits for %s"
+ ;; remove carriage returns. they cause problems in helm.
+ (replace-regexp-in-string "\n" " " query)))
(helm-candidates (mapcar (lambda (x)
(cons
(concat
;; just return the candidate
(action . (("Insert bibtex entry" . (lambda (doi)
(doi-utils-add-bibtex-entry-from-doi
- (replace-regexp-in-string "^http://dx.doi.org/" "" doi))))
+ (replace-regexp-in-string "^http://dx.doi.org/" "" doi) ,bibtex-file)))
("Open url" . (lambda (doi)
(browse-url doi))))))))
(helm :sources '(source)))))
#+END_EXAMPLE
-#+BEGIN_SRC emacs-lisp :var data=json :results value raw
+#+BEGIN_SRC emacs-lisp :var data=json :results value raw :tangle no
(let ((json-object-type 'plist)
(json (json-read-from-string data)))
(aref json 0))
Here is a list of helm candidates
-#+BEGIN_SRC emacs-lisp :var data=json :results code
+#+BEGIN_SRC emacs-lisp :var data=json :results code :tangle no
(let (;(json-object-type 'plist)
(json (json-read-from-string data)))
(mapcar (lambda (x) (cons (assoc 'fullCitation x) x)) json))
#+END_SRC
#+RESULTS:
-#+BEGIN_SRC emacs-lisp
+#+BEGIN_SRC emacs-lisp :tangle no
+
(((fullCitation . "Ann M. Deml, Vladan Stevanovi\304\207, Christopher L. Muhich, Charles B. Musgrave, Ryan O'Hayre, 2014, 'Oxide enthalpy of formation and band gap energy as accurate descriptors of oxygen vacancy formation energetics', <i>Energy & Environmental Science</i>, vol. 7, no. 6, p. 1996")
(year . "2014")
(coins . "ctx_ver=Z39.88-2004&rft_id=info%3Adoi%2Fhttp%3A%2F%2Fdx.doi.org%2F10.1039%2Fc3ee43874k&rfr_id=info%3Asid%2Fcrossref.org%3Asearch&rft.atitle=Oxide+enthalpy+of+formation+and+band+gap+energy+as+accurate+descriptors+of+oxygen+vacancy+formation+energetics&rft.jtitle=Energy+%26+Environmental+Science&rft.date=2014&rft.volume=7&rft.issue=6&rft.spage=1996&rft.aufirst=Ann+M.&rft.aulast=Deml&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Ajournal&rft.genre=article&rft.au=Ann+M.+Deml&rft.au=+Vladan+Stevanovi%C4%87&rft.au=+Christopher+L.+Muhich&rft.au=+Charles+B.+Musgrave&rft.au=+Ryan+O%27Hayre")
* ISBN utility
-This is not really a doi utility, but I am putting it here for now since it is just a single function. It looks up an ISBN and takes you to a page that has a bibtex entry. I am not crazy about that, but I have not found an isbn metadata source yet.
+These are not really doi utilities, but for now I am putting them here.
+
+I found this on the web. It can be handy, but the bibtex entry has a lot of stuff in it.
#+BEGIN_SRC emacs-lisp :tangle doi-utils.el
-(defun isbn-to-bibtex (isbn)
+(defun isbn-to-bibtex-lead (isbn)
"Search lead.to for ISBN bibtex entry. You have to copy the entry if it is on the page to your bibtex file."
- (interactive "ISBN: ")
+ (interactive "sISBN: ")
(browse-url
(format "http://lead.to/amazon/en/?key=%s+&si=all&op=bt&bn=&so=sa&ht=us" isbn)))
#+END_SRC
+
+Here we get isbn metadata and build a bibtex entry.
+http://xisbn.worldcat.org/xisbnadmin/doc/api.htm#getmetadata
+
+
+#+BEGIN_SRC emacs-lisp :tangle doi-utils.el
+(defun isbn-to-bibtex (isbn bibfile)
+ "Get bibtex entry for ISBN and insert it into BIBFILE unless an
+entry with the generated key already exists in the file."
+ (interactive
+ (list
+ (read-input
+ "ISBN: "
+ ;; now set initial input
+ (cond
+ ;; If region is active and it starts with a number, we use it
+ ((and (region-active-p)
+ (s-match "^[0-9]" (buffer-substring (region-beginning) (region-end))))
+ (buffer-substring (region-beginning) (region-end)))
+ ;; if first entry in kill ring starts with a number assume it is an isbn
+ ;; and use it as the guess
+ ((if (s-match "^[0-9]" (car kill-ring))
+ (car kill-ring)))
+ ;; type or paste it in
+ (t
+ nil)))
+ (ido-completing-read
+ "Bibfile: "
+ (append (f-entries "." (lambda (f) (f-ext? f "bib")))
+ org-ref-default-bibliography))))
+
+ (let* ((results (with-current-buffer
+ (url-retrieve-synchronously
+ (format
+ "http://xisbn.worldcat.org/webservices/xid/isbn/%s?method=getMetadata&format=json&fl=*"
+ isbn))
+ (json-read-from-string
+ (buffer-substring url-http-end-of-headers (point-max)))))
+ (status (cdr (nth 1 results)))
+ (metadata (aref (cdar results) 0))
+ (new-entry)
+ (new-key))
+
+ ;; check if we got something
+ (unless (string= "ok" status)
+ (error "Status is %s" status))
+
+ ;; construct an alphabetically sorted bibtex entry. I assume ISBN numbers go
+ ;; with book entries.
+ (setq new-entry
+ (concat "\n@book{,\n"
+ (mapconcat
+ 'identity
+ (loop for field in (-sort 'string-lessp (mapcar 'car metadata))
+ collect
+ (format " %s={%s}," field (cdr (assoc field metadata))))
+ "\n")
+ "\n}\n"))
+
+ ;; build entry in temp buffer to get the key so we can check for duplicates
+ (setq new-entry (with-temp-buffer
+ (insert new-entry)
+ (org-ref-clean-bibtex-entry)
+ (setq new-key (bibtex-key-in-head))
+ (buffer-string)))
+ (find-file bibfile)
+ (goto-char (point-min))
+ (when (search-forward new-key nil t)
+ (beep)
+ (setq new-key (read-input
+ (format "%s already exists. Enter new key (C-g to cancel): " new-key)
+ new-key)))
+ (goto-char (point-max))
+ (insert new-entry)
+ ;; set key. It is simplest to just replace it, even if it is the same.
+ (bibtex-beginning-of-entry)
+ (re-search-forward bibtex-entry-maybe-empty-head)
+ (if (match-beginning bibtex-key-in-head)
+ (delete-region (match-beginning bibtex-key-in-head)
+ (match-end bibtex-key-in-head)))
+ (insert new-key)
+ (bibtex-fill-entry)
+ (save-buffer)))
+#+END_SRC
+
+
+
* end of file
#+BEGIN_SRC emacs-lisp :tangle doi-utils.el
(provide 'doi-utils)
#+RESULTS:
: Loaded doi-utils.el
-
-
-
-
-
-#+BEGIN_SRC emacs-lisp
-(setq data '(("John" . "john@email.com")
- ("Jim" . "jim@email.com")
- ("Jane" . "jane@email.com")
- ("Jill" . "jill@email.com")))
-
-(setq some-helm-source
- `((name . "HELM at the Emacs")
- (candidates . ,(mapcar 'car data))
- (action . (lambda (candidate)
- (message-box "%s" (cdr (assoc candidate data)))))))
-
-(message-box "you chose %s" (helm :sources '(some-helm-source)))
-#+END_SRC
-
-#+RESULTS:
-: you chose jim@email.com