change python config, add jupyter and ein
This commit is contained in:
435
lisp/ein/ein-jupyter.el
Normal file
435
lisp/ein/ein-jupyter.el
Normal file
@@ -0,0 +1,435 @@
|
||||
;;; ein-jupyter.el --- Manage the jupyter notebook server -*- lexical-binding:t -*-
|
||||
|
||||
;; Copyright (C) 2017 John M. Miller
|
||||
|
||||
;; Authors: John M. Miller <millejoh at mac.com>
|
||||
|
||||
;; This file is NOT part of GNU Emacs.
|
||||
|
||||
;; ein-jupyter.el is free software: you can redistribute it and/or modify
|
||||
;; it under the terms of the GNU General Public License as published by
|
||||
;; the Free Software Foundation, either version 3 of the License, or
|
||||
;; (at your option) any later version.
|
||||
|
||||
;; ein-jupyter.el is distributed in the hope that it will be useful,
|
||||
;; but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
;; GNU General Public License for more details.
|
||||
|
||||
;; You should have received a copy of the GNU General Public License
|
||||
;; along with ein-jupyter.el. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
;;; Commentary:
|
||||
|
||||
;;; Code:
|
||||
|
||||
(require 'ein-core)
|
||||
(require 'ein-notebooklist)
|
||||
(require 'ein-dev)
|
||||
(require 'exec-path-from-shell nil t)
|
||||
(autoload 'ein:gat-chain "ein-gat")
|
||||
(autoload 'ein:gat-project "ein-gat")
|
||||
(autoload 'ein:gat-region "ein-gat")
|
||||
(autoload 'ein:gat-zone "ein-gat")
|
||||
|
||||
(defcustom ein:jupyter-use-containers nil
|
||||
"Take EIN in a different direcsh."
|
||||
:group 'ein
|
||||
:type 'boolean)
|
||||
|
||||
(defcustom ein:jupyter-docker-image "jupyter/datascience-notebook"
|
||||
"Docker pull whichever jupyter image you prefer. This defaults to
|
||||
the `jupyter docker stacks` on hub.docker.com.
|
||||
|
||||
Optionally append ':tag', e.g., ':latest' in the customary way."
|
||||
:group 'ein
|
||||
:type 'string)
|
||||
|
||||
(defcustom ein:jupyter-docker-mount-point "/home/jovyan/ipynb"
|
||||
"Where in docker image to mount `ein:jupyter-default-notebook-directory'."
|
||||
:group 'ein
|
||||
:type 'string)
|
||||
|
||||
(defcustom ein:jupyter-docker-additional-switches "-e JUPYTER_ENABLE_LAB=no --rm"
|
||||
"Additional options to the `docker run` call.
|
||||
|
||||
Note some options like '-v' and '-network' are imposed by EIN."
|
||||
:group 'ein
|
||||
:type 'string)
|
||||
|
||||
(defcustom ein:jupyter-cannot-find-jupyter nil
|
||||
"Use purcell's `exec-path-from-shell'"
|
||||
:group 'ein
|
||||
:type 'boolean)
|
||||
|
||||
(defcustom ein:jupyter-server-command "jupyter"
|
||||
"The default command to start a jupyter notebook server.
|
||||
Changing this to `jupyter-notebook' requires customizing
|
||||
`ein:jupyter-server-use-subcommand' to nil."
|
||||
:group 'ein
|
||||
:type 'string
|
||||
:set-after '(ein:jupyter-cannot-find-jupyter)
|
||||
:set (lambda (symbol value)
|
||||
(set-default symbol value)
|
||||
(when (and (featurep 'exec-path-from-shell)
|
||||
ein:jupyter-cannot-find-jupyter
|
||||
(memq window-system '(mac ns x)))
|
||||
(eval `(let (,@(when (boundp 'exec-path-from-shell-check-startup-files)
|
||||
(list 'exec-path-from-shell-check-startup-files)))
|
||||
(exec-path-from-shell-initialize))))))
|
||||
|
||||
(defcustom ein:jupyter-default-server-command ein:jupyter-server-command
|
||||
"Obsolete alias for `ein:jupyter-server-command'"
|
||||
:group 'ein
|
||||
:type 'string
|
||||
:set-after '(ein:jupyter-server-command)
|
||||
:set (lambda (symbol value)
|
||||
(set-default symbol value)
|
||||
(set-default 'ein:jupyter-server-command value)))
|
||||
|
||||
;;;###autoload
|
||||
(defcustom ein:jupyter-server-use-subcommand "notebook"
|
||||
"For JupyterLab 3.0+ change the subcommand to \"server\".
|
||||
Users of \"jupyter-notebook\" (as opposed to \"jupyter notebook\") select Omit."
|
||||
:group 'ein
|
||||
:type '(choice (string :tag "Subcommand" "notebook")
|
||||
(const :tag "Omit" nil)))
|
||||
|
||||
(defcustom ein:jupyter-server-args '("--no-browser")
|
||||
"Add any additional command line options you wish to include
|
||||
with the call to the jupyter notebook."
|
||||
:group 'ein
|
||||
:type '(repeat string))
|
||||
|
||||
(defcustom ein:jupyter-default-notebook-directory nil
|
||||
"Default location of ipynb files."
|
||||
:group 'ein
|
||||
:type 'directory)
|
||||
|
||||
(defcustom ein:jupyter-default-kernel 'first-alphabetically
|
||||
"With which of ${XDG_DATA_HOME}/jupyter/kernels to create new notebooks."
|
||||
:group 'ein
|
||||
:type (append
|
||||
'(choice (other :tag "First alphabetically" first-alphabetically))
|
||||
(condition-case err
|
||||
(mapcar
|
||||
(lambda (x) `(const :tag ,(cdr x) ,(car x)))
|
||||
(cl-loop
|
||||
for (k . spec) in
|
||||
(alist-get
|
||||
'kernelspecs
|
||||
(let ((json-object-type 'alist))
|
||||
(json-read-from-string ;; intentionally not ein:json-read-from-string
|
||||
(shell-command-to-string
|
||||
(format "2>/dev/null %s kernelspec list --json"
|
||||
ein:jupyter-server-command)))))
|
||||
collect `(,k . ,(alist-get 'display_name (alist-get 'spec spec)))))
|
||||
(error (ein:log 'warn "ein:jupyter-default-kernel: %s" err)
|
||||
'((string :tag "Ask"))))))
|
||||
|
||||
(defconst *ein:jupyter-server-process-name* "ein server")
|
||||
(defconst *ein:jupyter-server-buffer-name*
|
||||
(format "*%s*" *ein:jupyter-server-process-name*))
|
||||
(defvar-local ein:jupyter-server-notebook-directory nil
|
||||
"Keep track of prevailing --notebook-dir argument.")
|
||||
|
||||
(defun ein:jupyter-running-notebook-directory ()
|
||||
(when (ein:jupyter-server-process)
|
||||
(buffer-local-value 'ein:jupyter-server-notebook-directory
|
||||
(get-buffer *ein:jupyter-server-buffer-name*))))
|
||||
|
||||
(defun ein:jupyter-get-default-kernel (kernels)
|
||||
(cond (ein:%notebooklist-new-kernel%
|
||||
(ein:$kernelspec-name ein:%notebooklist-new-kernel%))
|
||||
((eq ein:jupyter-default-kernel 'first-alphabetically)
|
||||
(car (car kernels)))
|
||||
((stringp ein:jupyter-default-kernel)
|
||||
ein:jupyter-default-kernel)
|
||||
(t
|
||||
(symbol-name ein:jupyter-default-kernel))))
|
||||
|
||||
(defun ein:jupyter-process-lines (_url-or-port command &rest args)
|
||||
"If URL-OR-PORT registered as a k8s url, preface COMMAND ARGS
|
||||
with `kubectl exec'."
|
||||
(if-let ((found (executable-find command)))
|
||||
(with-temp-buffer
|
||||
(let ((status (apply #'call-process found nil t nil args)))
|
||||
(if (zerop status)
|
||||
(progn
|
||||
(goto-char (point-min))
|
||||
(let (lines)
|
||||
(while (not (eobp))
|
||||
(setq lines (cons (buffer-substring-no-properties
|
||||
(line-beginning-position)
|
||||
(line-end-position))
|
||||
lines))
|
||||
(forward-line 1))
|
||||
(nreverse lines)))
|
||||
(prog1 nil
|
||||
(ein:log 'warn "ein:jupyter-process-lines: '%s %s' returned %s"
|
||||
found (ein:join-str " " args) status)))))
|
||||
(prog1 nil
|
||||
(ein:log 'warn "ein:jupyter-process-lines: cannot find %s" command))))
|
||||
|
||||
(defsubst ein:jupyter-server-process ()
|
||||
"Return the emacs process object of our session."
|
||||
(get-buffer-process (get-buffer *ein:jupyter-server-buffer-name*)))
|
||||
|
||||
(defun ein:jupyter-server--run (buf user-cmd dir &optional args)
|
||||
(get-buffer-create buf)
|
||||
(let* ((cmd (if ein:jupyter-use-containers "docker" user-cmd))
|
||||
(vargs (cond (ein:jupyter-use-containers
|
||||
(split-string
|
||||
(format "run --network host -v %s:%s %s %s"
|
||||
dir
|
||||
ein:jupyter-docker-mount-point
|
||||
ein:jupyter-docker-additional-switches
|
||||
ein:jupyter-docker-image)))
|
||||
(t
|
||||
(append (split-string (or ein:jupyter-server-use-subcommand ""))
|
||||
(when dir
|
||||
(list (format "--notebook-dir=%s"
|
||||
(convert-standard-filename dir))))
|
||||
args
|
||||
(let ((copy (cl-copy-list ein:jupyter-server-args)))
|
||||
(when (ein:debug-p)
|
||||
(cl-pushnew "--debug" copy :test #'equal))
|
||||
copy)))))
|
||||
(proc (apply #'start-process
|
||||
*ein:jupyter-server-process-name* buf cmd vargs)))
|
||||
(ein:log 'info "ein:jupyter-server--run: %s %s" cmd (ein:join-str " " vargs))
|
||||
(set-process-query-on-exit-flag proc nil)
|
||||
proc))
|
||||
|
||||
(defun ein:jupyter-my-url-or-port ()
|
||||
(when-let ((my-pid (aand (ein:jupyter-server-process) (process-id it))))
|
||||
(catch 'done
|
||||
(dolist (json (ein:jupyter-crib-running-servers))
|
||||
(cl-destructuring-bind (&key pid url &allow-other-keys)
|
||||
json
|
||||
(when (equal my-pid pid)
|
||||
(throw 'done (ein:url url))))))))
|
||||
|
||||
(defun ein:jupyter-server-ready-p ()
|
||||
(when (ein:jupyter-server-process)
|
||||
(with-current-buffer *ein:jupyter-server-buffer-name*
|
||||
(save-excursion
|
||||
(goto-char (point-max))
|
||||
(re-search-backward (format "Process %s" *ein:jupyter-server-process-name*)
|
||||
nil "") ;; important if we start-stop-start
|
||||
(re-search-forward
|
||||
"\\([[:alnum:]]+\\) is\\( now\\)? running"
|
||||
nil t)))))
|
||||
|
||||
(defun ein:jupyter-server-login-and-open (url-or-port &optional callback)
|
||||
"Log in and open a notebooklist buffer for a running jupyter notebook server.
|
||||
|
||||
Determine if there is a running jupyter server (started via a
|
||||
call to `ein:jupyter-server-start') and then try to guess if
|
||||
token authentication is enabled. If a token is found use it to
|
||||
generate a call to `ein:notebooklist-login' and once
|
||||
authenticated open the notebooklist buffer via a call to
|
||||
`ein:notebooklist-open'."
|
||||
(if-let ((token (ein:notebooklist-token-or-password url-or-port)))
|
||||
(ein:notebooklist-login url-or-port callback nil nil token)
|
||||
(ein:log 'error "`(ein:notebooklist-token-or-password %s)` must return non-nil"
|
||||
url-or-port)))
|
||||
|
||||
(defsubst ein:set-process-sentinel (proc url-or-port)
|
||||
"URL-OR-PORT might get redirected.
|
||||
This is currently only the case for jupyterhub. Once login
|
||||
handshake provides the new URL-OR-PORT, we set various state as
|
||||
pertains our singleton jupyter server process here."
|
||||
|
||||
;; Would have used `add-function' if it didn't produce gv-ref warnings.
|
||||
(set-process-sentinel
|
||||
proc
|
||||
(apply-partially (lambda (url-or-port* sentinel proc* event)
|
||||
(aif sentinel (funcall it proc* event))
|
||||
(funcall #'ein:notebooklist-sentinel url-or-port* proc* event))
|
||||
url-or-port (process-sentinel proc))))
|
||||
|
||||
;;;###autoload
|
||||
(defun ein:jupyter-crib-token (url-or-port)
|
||||
"Shell out to jupyter for its credentials knowledge. Return list
|
||||
of (PASSWORD TOKEN)."
|
||||
(aif (cl-loop for line in
|
||||
(apply #'ein:jupyter-process-lines url-or-port
|
||||
ein:jupyter-server-command
|
||||
(append
|
||||
(split-string (or ein:jupyter-server-use-subcommand ""))
|
||||
'("list" "--json")))
|
||||
with token0
|
||||
with password0
|
||||
when (cl-destructuring-bind
|
||||
(&key password url token &allow-other-keys)
|
||||
(ein:json-read-from-string line)
|
||||
(prog1 (equal (ein:url url) url-or-port)
|
||||
(setq password0 password) ;; t or :json-false
|
||||
(setq token0 token)))
|
||||
return (list password0 token0))
|
||||
it (list nil nil)))
|
||||
|
||||
;;;###autoload
|
||||
(defun ein:jupyter-crib-running-servers ()
|
||||
"Shell out to jupyter for running servers."
|
||||
(cl-loop for line in
|
||||
(apply #'ein:jupyter-process-lines nil
|
||||
ein:jupyter-server-command
|
||||
(append
|
||||
(split-string (or ein:jupyter-server-use-subcommand ""))
|
||||
'("list" "--json")))
|
||||
collecting (ein:json-read-from-string line)))
|
||||
|
||||
;;;###autoload
|
||||
(defun ein:jupyter-server-start (server-command
|
||||
notebook-directory
|
||||
&optional no-login-p login-callback port)
|
||||
"Start SERVER-COMMAND with `--notebook-dir' NOTEBOOK-DIRECTORY.
|
||||
|
||||
Login after connection established unless NO-LOGIN-P is set.
|
||||
LOGIN-CALLBACK takes two arguments, the buffer created by
|
||||
`ein:notebooklist-open--finish', and the url-or-port argument
|
||||
of `ein:notebooklist-open*'.
|
||||
|
||||
With \\[universal-argument] prefix arg, prompt the user for the
|
||||
server command."
|
||||
(interactive
|
||||
(list (let ((default-command (executable-find ein:jupyter-server-command)))
|
||||
(if (and (not ein:jupyter-use-containers)
|
||||
(or current-prefix-arg (not default-command)))
|
||||
(let (command result)
|
||||
(while (not (setq
|
||||
result
|
||||
(executable-find
|
||||
(setq
|
||||
command
|
||||
(read-string
|
||||
(format
|
||||
"%sServer command: "
|
||||
(if command
|
||||
(format "[%s not executable] " command)
|
||||
""))
|
||||
nil nil ein:jupyter-server-command))))))
|
||||
result)
|
||||
default-command))
|
||||
(let ((default-dir ein:jupyter-default-notebook-directory)
|
||||
result)
|
||||
(while (or (not result) (not (file-directory-p result)))
|
||||
(setq result (read-directory-name
|
||||
(format "%sNotebook directory: "
|
||||
(if result
|
||||
(format "[%s not a directory]" result)
|
||||
""))
|
||||
default-dir default-dir t)))
|
||||
result)
|
||||
nil
|
||||
(lambda (buffer _url-or-port)
|
||||
(pop-to-buffer buffer))
|
||||
nil))
|
||||
(when (ein:jupyter-server-process)
|
||||
(error "ein:jupyter-server-start: First `M-x ein:stop'"))
|
||||
(let ((proc (ein:jupyter-server--run *ein:jupyter-server-buffer-name*
|
||||
server-command
|
||||
notebook-directory
|
||||
(when (numberp port)
|
||||
`("--port" ,(format "%s" port)
|
||||
"--port-retries" "0")))))
|
||||
(cl-loop repeat 30
|
||||
until (ein:jupyter-server-ready-p)
|
||||
do (sleep-for 0 500)
|
||||
finally do
|
||||
(if-let ((buffer (get-buffer *ein:jupyter-server-buffer-name*))
|
||||
(url-or-port (ein:jupyter-my-url-or-port)))
|
||||
(with-current-buffer buffer
|
||||
(setq ein:jupyter-server-notebook-directory
|
||||
(convert-standard-filename notebook-directory))
|
||||
(add-hook 'kill-buffer-query-functions
|
||||
(lambda () (or (not (ein:jupyter-server-process))
|
||||
(ein:jupyter-server-stop t url-or-port)))
|
||||
nil t))
|
||||
(ein:log 'warn "Jupyter server failed to start, cancelling operation")))
|
||||
(when-let ((login-p (not no-login-p))
|
||||
(url-or-port (ein:jupyter-my-url-or-port)))
|
||||
(unless login-callback
|
||||
(setq login-callback #'ignore))
|
||||
(add-function :after (var login-callback)
|
||||
(apply-partially (lambda (proc* _buffer url-or-port)
|
||||
(ein:set-process-sentinel proc* url-or-port))
|
||||
proc))
|
||||
(ein:jupyter-server-login-and-open
|
||||
url-or-port
|
||||
login-callback))))
|
||||
|
||||
;;;###autoload
|
||||
(defalias 'ein:run 'ein:jupyter-server-start)
|
||||
|
||||
;;;###autoload
|
||||
(defalias 'ein:stop 'ein:jupyter-server-stop)
|
||||
|
||||
(defvar ein:gat-urls)
|
||||
(defvar ein:gat-aws-region)
|
||||
;;;###autoload
|
||||
(defun ein:jupyter-server-stop (&optional ask-p url-or-port)
|
||||
(interactive
|
||||
(list t (awhen (ein:get-notebook)
|
||||
(ein:$notebook-url-or-port it))))
|
||||
(let ((my-url-or-port (ein:jupyter-my-url-or-port))
|
||||
(all-p t))
|
||||
(dolist (url-or-port
|
||||
(if url-or-port (list url-or-port) (ein:notebooklist-keys))
|
||||
(prog1 all-p
|
||||
(when (and (null (ein:notebooklist-keys))
|
||||
(ein:shared-output-healthy-p))
|
||||
(kill-buffer (ein:shared-output-buffer)))))
|
||||
(let* ((gat-dir (alist-get (intern url-or-port)
|
||||
(awhen (bound-and-true-p ein:gat-urls) it)))
|
||||
(my-p (string= url-or-port my-url-or-port))
|
||||
(close-p (or (not ask-p)
|
||||
(prog1 (y-or-n-p (format "Close %s?" url-or-port))
|
||||
(message "")))))
|
||||
(if (not close-p)
|
||||
(setq all-p nil)
|
||||
(ein:notebook-close-notebooks
|
||||
(lambda (notebook)
|
||||
(string= url-or-port (ein:$notebook-url-or-port notebook)))
|
||||
t)
|
||||
(cl-loop repeat 10
|
||||
until (null (seq-some (lambda (proc)
|
||||
(cl-search "request curl"
|
||||
(process-name proc)))
|
||||
(process-list)))
|
||||
do (sleep-for 0 500))
|
||||
(cond (my-p
|
||||
(-when-let* ((proc (ein:jupyter-server-process))
|
||||
(pid (process-id proc)))
|
||||
(run-at-time 2 nil
|
||||
(lambda ()
|
||||
(signal-process pid (if (eq system-type 'windows-nt) 9 15))))
|
||||
;; NotebookPasswordApp::shutdown_server() also ignores req response.
|
||||
(ein:query-singleton-ajax (ein:url url-or-port "api/shutdown")
|
||||
:type "POST")))
|
||||
(gat-dir
|
||||
(with-current-buffer (ein:notebooklist-get-buffer url-or-port)
|
||||
(-when-let* ((gat-chain-args `("gat" nil
|
||||
"--project" ,(ein:gat-project)
|
||||
"--region" ,(ein:gat-region)
|
||||
"--zone" ,(ein:gat-zone)))
|
||||
(now (truncate (float-time)))
|
||||
(gat-log-exec (append gat-chain-args
|
||||
(list "log" "--after" (format "%s" now)
|
||||
"--vendor" (aif (bound-and-true-p ein:gat-vendor) it "aws")
|
||||
"--nextunit" "shutdown.service")))
|
||||
(magit-process-popup-time 0))
|
||||
(ein:gat-chain (current-buffer) nil gat-log-exec :notebook-dir gat-dir)
|
||||
;; NotebookPasswordApp::shutdown_server() also ignores req response.
|
||||
(ein:query-singleton-ajax (ein:url url-or-port "api/shutdown")
|
||||
:type "POST")))))
|
||||
;; `ein:notebooklist-sentinel' frequently does not trigger
|
||||
(ein:notebooklist-list-remove url-or-port)
|
||||
(maphash (lambda (k _v) (when (equal (car k) url-or-port)
|
||||
(remhash k *ein:notebook--pending-query*)))
|
||||
*ein:notebook--pending-query*)
|
||||
(kill-buffer (ein:notebooklist-get-buffer url-or-port)))))))
|
||||
|
||||
(provide 'ein-jupyter)
|
||||
Reference in New Issue
Block a user