;;; ein-jupyter.el --- Manage the jupyter notebook server -*- lexical-binding:t -*- ;; Copyright (C) 2017 John M. Miller ;; Authors: John M. Miller ;; This file is NOT part of GNU Emacs. ;; ein-jupyter.el is free software: you can redistribute it and/or modify ;; it under the terms of the GNU General Public License as published by ;; the Free Software Foundation, either version 3 of the License, or ;; (at your option) any later version. ;; ein-jupyter.el is distributed in the hope that it will be useful, ;; but WITHOUT ANY WARRANTY; without even the implied warranty of ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ;; GNU General Public License for more details. ;; You should have received a copy of the GNU General Public License ;; along with ein-jupyter.el. If not, see . ;;; Commentary: ;;; Code: (require 'ein-core) (require 'ein-notebooklist) (require 'ein-dev) (require 'exec-path-from-shell nil t) (autoload 'ein:gat-chain "ein-gat") (autoload 'ein:gat-project "ein-gat") (autoload 'ein:gat-region "ein-gat") (autoload 'ein:gat-zone "ein-gat") (defcustom ein:jupyter-use-containers nil "Take EIN in a different direcsh." :group 'ein :type 'boolean) (defcustom ein:jupyter-docker-image "jupyter/datascience-notebook" "Docker pull whichever jupyter image you prefer. This defaults to the `jupyter docker stacks` on hub.docker.com. Optionally append ':tag', e.g., ':latest' in the customary way." :group 'ein :type 'string) (defcustom ein:jupyter-docker-mount-point "/home/jovyan/ipynb" "Where in docker image to mount `ein:jupyter-default-notebook-directory'." :group 'ein :type 'string) (defcustom ein:jupyter-docker-additional-switches "-e JUPYTER_ENABLE_LAB=no --rm" "Additional options to the `docker run` call. Note some options like '-v' and '-network' are imposed by EIN." :group 'ein :type 'string) (defcustom ein:jupyter-cannot-find-jupyter nil "Use purcell's `exec-path-from-shell'" :group 'ein :type 'boolean) (defcustom ein:jupyter-server-command "jupyter" "The default command to start a jupyter notebook server. Changing this to `jupyter-notebook' requires customizing `ein:jupyter-server-use-subcommand' to nil." :group 'ein :type 'string :set-after '(ein:jupyter-cannot-find-jupyter) :set (lambda (symbol value) (set-default symbol value) (when (and (featurep 'exec-path-from-shell) ein:jupyter-cannot-find-jupyter (memq window-system '(mac ns x))) (eval `(let (,@(when (boundp 'exec-path-from-shell-check-startup-files) (list 'exec-path-from-shell-check-startup-files))) (exec-path-from-shell-initialize)))))) (defcustom ein:jupyter-default-server-command ein:jupyter-server-command "Obsolete alias for `ein:jupyter-server-command'" :group 'ein :type 'string :set-after '(ein:jupyter-server-command) :set (lambda (symbol value) (set-default symbol value) (set-default 'ein:jupyter-server-command value))) ;;;###autoload (defcustom ein:jupyter-server-use-subcommand "notebook" "For JupyterLab 3.0+ change the subcommand to \"server\". Users of \"jupyter-notebook\" (as opposed to \"jupyter notebook\") select Omit." :group 'ein :type '(choice (string :tag "Subcommand" "notebook") (const :tag "Omit" nil))) (defcustom ein:jupyter-server-args '("--no-browser") "Add any additional command line options you wish to include with the call to the jupyter notebook." :group 'ein :type '(repeat string)) (defcustom ein:jupyter-default-notebook-directory nil "Default location of ipynb files." :group 'ein :type 'directory) (defcustom ein:jupyter-default-kernel 'first-alphabetically "With which of ${XDG_DATA_HOME}/jupyter/kernels to create new notebooks." :group 'ein :type (append '(choice (other :tag "First alphabetically" first-alphabetically)) (condition-case err (mapcar (lambda (x) `(const :tag ,(cdr x) ,(car x))) (cl-loop for (k . spec) in (alist-get 'kernelspecs (let ((json-object-type 'alist)) (json-read-from-string ;; intentionally not ein:json-read-from-string (shell-command-to-string (format "2>/dev/null %s kernelspec list --json" ein:jupyter-server-command))))) collect `(,k . ,(alist-get 'display_name (alist-get 'spec spec))))) (error (ein:log 'warn "ein:jupyter-default-kernel: %s" err) '((string :tag "Ask")))))) (defconst *ein:jupyter-server-process-name* "ein server") (defconst *ein:jupyter-server-buffer-name* (format "*%s*" *ein:jupyter-server-process-name*)) (defvar-local ein:jupyter-server-notebook-directory nil "Keep track of prevailing --notebook-dir argument.") (defun ein:jupyter-running-notebook-directory () (when (ein:jupyter-server-process) (buffer-local-value 'ein:jupyter-server-notebook-directory (get-buffer *ein:jupyter-server-buffer-name*)))) (defun ein:jupyter-get-default-kernel (kernels) (cond (ein:%notebooklist-new-kernel% (ein:$kernelspec-name ein:%notebooklist-new-kernel%)) ((eq ein:jupyter-default-kernel 'first-alphabetically) (car (car kernels))) ((stringp ein:jupyter-default-kernel) ein:jupyter-default-kernel) (t (symbol-name ein:jupyter-default-kernel)))) (defun ein:jupyter-process-lines (_url-or-port command &rest args) "If URL-OR-PORT registered as a k8s url, preface COMMAND ARGS with `kubectl exec'." (if-let ((found (executable-find command))) (with-temp-buffer (let ((status (apply #'call-process found nil t nil args))) (if (zerop status) (progn (goto-char (point-min)) (let (lines) (while (not (eobp)) (setq lines (cons (buffer-substring-no-properties (line-beginning-position) (line-end-position)) lines)) (forward-line 1)) (nreverse lines))) (prog1 nil (ein:log 'warn "ein:jupyter-process-lines: '%s %s' returned %s" found (ein:join-str " " args) status))))) (prog1 nil (ein:log 'warn "ein:jupyter-process-lines: cannot find %s" command)))) (defsubst ein:jupyter-server-process () "Return the emacs process object of our session." (get-buffer-process (get-buffer *ein:jupyter-server-buffer-name*))) (defun ein:jupyter-server--run (buf user-cmd dir &optional args) (get-buffer-create buf) (let* ((cmd (if ein:jupyter-use-containers "docker" user-cmd)) (vargs (cond (ein:jupyter-use-containers (split-string (format "run --network host -v %s:%s %s %s" dir ein:jupyter-docker-mount-point ein:jupyter-docker-additional-switches ein:jupyter-docker-image))) (t (append (split-string (or ein:jupyter-server-use-subcommand "")) (when dir (list (format "--notebook-dir=%s" (convert-standard-filename dir)))) args (let ((copy (cl-copy-list ein:jupyter-server-args))) (when (ein:debug-p) (cl-pushnew "--debug" copy :test #'equal)) copy))))) (proc (apply #'start-process *ein:jupyter-server-process-name* buf cmd vargs))) (ein:log 'info "ein:jupyter-server--run: %s %s" cmd (ein:join-str " " vargs)) (set-process-query-on-exit-flag proc nil) proc)) (defun ein:jupyter-my-url-or-port () (when-let ((my-pid (aand (ein:jupyter-server-process) (process-id it)))) (catch 'done (dolist (json (ein:jupyter-crib-running-servers)) (cl-destructuring-bind (&key pid url &allow-other-keys) json (when (equal my-pid pid) (throw 'done (ein:url url)))))))) (defun ein:jupyter-server-ready-p () (when (ein:jupyter-server-process) (with-current-buffer *ein:jupyter-server-buffer-name* (save-excursion (goto-char (point-max)) (re-search-backward (format "Process %s" *ein:jupyter-server-process-name*) nil "") ;; important if we start-stop-start (re-search-forward "\\([[:alnum:]]+\\) is\\( now\\)? running" nil t))))) (defun ein:jupyter-server-login-and-open (url-or-port &optional callback) "Log in and open a notebooklist buffer for a running jupyter notebook server. Determine if there is a running jupyter server (started via a call to `ein:jupyter-server-start') and then try to guess if token authentication is enabled. If a token is found use it to generate a call to `ein:notebooklist-login' and once authenticated open the notebooklist buffer via a call to `ein:notebooklist-open'." (if-let ((token (ein:notebooklist-token-or-password url-or-port))) (ein:notebooklist-login url-or-port callback nil nil token) (ein:log 'error "`(ein:notebooklist-token-or-password %s)` must return non-nil" url-or-port))) (defsubst ein:set-process-sentinel (proc url-or-port) "URL-OR-PORT might get redirected. This is currently only the case for jupyterhub. Once login handshake provides the new URL-OR-PORT, we set various state as pertains our singleton jupyter server process here." ;; Would have used `add-function' if it didn't produce gv-ref warnings. (set-process-sentinel proc (apply-partially (lambda (url-or-port* sentinel proc* event) (aif sentinel (funcall it proc* event)) (funcall #'ein:notebooklist-sentinel url-or-port* proc* event)) url-or-port (process-sentinel proc)))) ;;;###autoload (defun ein:jupyter-crib-token (url-or-port) "Shell out to jupyter for its credentials knowledge. Return list of (PASSWORD TOKEN)." (aif (cl-loop for line in (apply #'ein:jupyter-process-lines url-or-port ein:jupyter-server-command (append (split-string (or ein:jupyter-server-use-subcommand "")) '("list" "--json"))) with token0 with password0 when (cl-destructuring-bind (&key password url token &allow-other-keys) (ein:json-read-from-string line) (prog1 (equal (ein:url url) url-or-port) (setq password0 password) ;; t or :json-false (setq token0 token))) return (list password0 token0)) it (list nil nil))) ;;;###autoload (defun ein:jupyter-crib-running-servers () "Shell out to jupyter for running servers." (cl-loop for line in (apply #'ein:jupyter-process-lines nil ein:jupyter-server-command (append (split-string (or ein:jupyter-server-use-subcommand "")) '("list" "--json"))) collecting (ein:json-read-from-string line))) ;;;###autoload (defun ein:jupyter-server-start (server-command notebook-directory &optional no-login-p login-callback port) "Start SERVER-COMMAND with `--notebook-dir' NOTEBOOK-DIRECTORY. Login after connection established unless NO-LOGIN-P is set. LOGIN-CALLBACK takes two arguments, the buffer created by `ein:notebooklist-open--finish', and the url-or-port argument of `ein:notebooklist-open*'. With \\[universal-argument] prefix arg, prompt the user for the server command." (interactive (list (let ((default-command (executable-find ein:jupyter-server-command))) (if (and (not ein:jupyter-use-containers) (or current-prefix-arg (not default-command))) (let (command result) (while (not (setq result (executable-find (setq command (read-string (format "%sServer command: " (if command (format "[%s not executable] " command) "")) nil nil ein:jupyter-server-command)))))) result) default-command)) (let ((default-dir ein:jupyter-default-notebook-directory) result) (while (or (not result) (not (file-directory-p result))) (setq result (read-directory-name (format "%sNotebook directory: " (if result (format "[%s not a directory]" result) "")) default-dir default-dir t))) result) nil (lambda (buffer _url-or-port) (pop-to-buffer buffer)) nil)) (when (ein:jupyter-server-process) (error "ein:jupyter-server-start: First `M-x ein:stop'")) (let ((proc (ein:jupyter-server--run *ein:jupyter-server-buffer-name* server-command notebook-directory (when (numberp port) `("--port" ,(format "%s" port) "--port-retries" "0"))))) (cl-loop repeat 30 until (ein:jupyter-server-ready-p) do (sleep-for 0 500) finally do (if-let ((buffer (get-buffer *ein:jupyter-server-buffer-name*)) (url-or-port (ein:jupyter-my-url-or-port))) (with-current-buffer buffer (setq ein:jupyter-server-notebook-directory (convert-standard-filename notebook-directory)) (add-hook 'kill-buffer-query-functions (lambda () (or (not (ein:jupyter-server-process)) (ein:jupyter-server-stop t url-or-port))) nil t)) (ein:log 'warn "Jupyter server failed to start, cancelling operation"))) (when-let ((login-p (not no-login-p)) (url-or-port (ein:jupyter-my-url-or-port))) (unless login-callback (setq login-callback #'ignore)) (add-function :after (var login-callback) (apply-partially (lambda (proc* _buffer url-or-port) (ein:set-process-sentinel proc* url-or-port)) proc)) (ein:jupyter-server-login-and-open url-or-port login-callback)))) ;;;###autoload (defalias 'ein:run 'ein:jupyter-server-start) ;;;###autoload (defalias 'ein:stop 'ein:jupyter-server-stop) (defvar ein:gat-urls) (defvar ein:gat-aws-region) ;;;###autoload (defun ein:jupyter-server-stop (&optional ask-p url-or-port) (interactive (list t (awhen (ein:get-notebook) (ein:$notebook-url-or-port it)))) (let ((my-url-or-port (ein:jupyter-my-url-or-port)) (all-p t)) (dolist (url-or-port (if url-or-port (list url-or-port) (ein:notebooklist-keys)) (prog1 all-p (when (and (null (ein:notebooklist-keys)) (ein:shared-output-healthy-p)) (kill-buffer (ein:shared-output-buffer))))) (let* ((gat-dir (alist-get (intern url-or-port) (awhen (bound-and-true-p ein:gat-urls) it))) (my-p (string= url-or-port my-url-or-port)) (close-p (or (not ask-p) (prog1 (y-or-n-p (format "Close %s?" url-or-port)) (message ""))))) (if (not close-p) (setq all-p nil) (ein:notebook-close-notebooks (lambda (notebook) (string= url-or-port (ein:$notebook-url-or-port notebook))) t) (cl-loop repeat 10 until (null (seq-some (lambda (proc) (cl-search "request curl" (process-name proc))) (process-list))) do (sleep-for 0 500)) (cond (my-p (-when-let* ((proc (ein:jupyter-server-process)) (pid (process-id proc))) (run-at-time 2 nil (lambda () (signal-process pid (if (eq system-type 'windows-nt) 9 15)))) ;; NotebookPasswordApp::shutdown_server() also ignores req response. (ein:query-singleton-ajax (ein:url url-or-port "api/shutdown") :type "POST"))) (gat-dir (with-current-buffer (ein:notebooklist-get-buffer url-or-port) (-when-let* ((gat-chain-args `("gat" nil "--project" ,(ein:gat-project) "--region" ,(ein:gat-region) "--zone" ,(ein:gat-zone))) (now (truncate (float-time))) (gat-log-exec (append gat-chain-args (list "log" "--after" (format "%s" now) "--vendor" (aif (bound-and-true-p ein:gat-vendor) it "aws") "--nextunit" "shutdown.service"))) (magit-process-popup-time 0)) (ein:gat-chain (current-buffer) nil gat-log-exec :notebook-dir gat-dir) ;; NotebookPasswordApp::shutdown_server() also ignores req response. (ein:query-singleton-ajax (ein:url url-or-port "api/shutdown") :type "POST"))))) ;; `ein:notebooklist-sentinel' frequently does not trigger (ein:notebooklist-list-remove url-or-port) (maphash (lambda (k _v) (when (equal (car k) url-or-port) (remhash k *ein:notebook--pending-query*))) *ein:notebook--pending-query*) (kill-buffer (ein:notebooklist-get-buffer url-or-port))))))) (provide 'ein-jupyter)