kiwix.el

   1 ;;; kiwix.el --- Searching offline Wikipedia through Kiwix.  -*- lexical-binding: t; -*-
   2 ;; -*- coding: utf-8 -*-
   3
   4 ;; Copyright (C) 2019-2020  Free Software Foundation, Inc.
   5
   6 ;; Author: stardiviner <numbchild@gmail.com>
   7 ;; Maintainer: stardiviner <numbchild@gmail.com>
   8 ;; Keywords: kiwix wikipedia
   9 ;; URL: https://github.com/stardiviner/kiwix.el
  10 ;; Created: 23th July 2016
  11 ;; Version: 1.0.1
  12 ;; Package-Requires: ((emacs "24.4") (request "0.3.0"))
  13
  14 ;; This file is part of GNU Emacs.
  15
  16 ;; GNU Emacs is free software: you can redistribute it and/or modify
  17 ;; it under the terms of the GNU General Public License as published by
  18 ;; the Free Software Foundation, either version 3 of the License, or
  19 ;; (at your option) any later version.
  20
  21 ;; GNU Emacs is distributed in the hope that it will be useful,
  22 ;; but WITHOUT ANY WARRANTY; without even the implied warranty of
  23 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  24 ;; GNU General Public License for more details.
  25
  26 ;; You should have received a copy of the GNU General Public License
  27 ;; along with GNU Emacs.  If not, see <http://www.gnu.org/licenses/>.
  28
  29 ;;; Commentary:
  30
  31 ;; This currently only works for GNU/Linux, not tested for Mac OS X and Windows.
  32
  33 ;;;; Kiwix installation
  34 ;;
  35 ;; https://github.com/stardiviner/kiwix.el/#install
  36
  37 ;;;; Config:
  38 ;;
  39 ;; (use-package kiwix
  40 ;;   :ensure t
  41 ;;   :after org
  42 ;;   :bind (:map document-prefix ("w" . kiwix-at-point))
  43 ;;   :init (setq kiwix-server-use-docker t
  44 ;;               kiwix-server-port 8080
  45 ;;               kiwix-default-library "wikipedia_zh_all_2015-11.zim"))
  46
  47 ;;;; Usage:
  48 ;;
  49 ;; 1. [M-x kiwix-launch-server] to launch Kiwix server.
  50 ;; 2. [M-x kiwix-at-point] to search the word under point or the region selected string.
  51
  52 ;;; Code:
  53 \f
  54
  55 (require 'cl-lib)
  56 (require 'request)
  57 (require 'subr-x)
  58 (require 'thingatpt)
  59 (require 'json)
  60
  61 (declare-function helm "helm")
  62 (declare-function helm-build-async-source "helm")
  63 (declare-function ivy-read "ivy")
  64
  65
  66 (defgroup kiwix-mode nil
  67   "Kiwix customization options."
  68   :group 'kiwix-mode)
  69
  70 (defcustom kiwix-server-use-docker nil
  71   "Using Docker container for kiwix-serve or not?"
  72   :type 'boolean
  73   :safe #'booleanp)
  74
  75 (defcustom kiwix-server-port 8000
  76   "Specify default kiwix-serve server port."
  77   :type 'number
  78   :safe #'numberp)
  79
  80 (defcustom kiwix-server-url (format "http://127.0.0.1:%s" kiwix-server-port)
  81   "Specify Kiwix server URL."
  82   :type 'string)
  83
  84 (defcustom kiwix-server-command
  85   (cond
  86    ((file-executable-p "/usr/bin/kiwix-serve") "/usr/bin/kiwix-serve")
  87    ((eq system-type 'gnu/linux) "/usr/lib/kiwix/bin/kiwix-serve")
  88    ((eq system-type 'darwin)
  89     (warn "You need to specify Mac OS X Kiwix path. And send a PR to my repo."))
  90    ((eq system-type 'windows-nt)
  91     (warn "You need to specify Windows Kiwix path. And send a PR to my repo.")))
  92   "Specify kiwix server command."
  93   :type 'string)
  94
  95 (defun kiwix-dir-detect ()
  96   "Detect Kiwix profile directory exist."
  97   (let ((kiwix-dir "~/.www.kiwix.org/kiwix"))
  98     (if (and (file-directory-p kiwix-dir) (file-readable-p kiwix-dir))
  99         kiwix-dir
 100       (warn "ERROR: Kiwix profile directory \"~/.www.kiwix.org/kiwix\" is not accessible.")
 101       nil)))
 102
 103 (defcustom kiwix-default-data-profile-name
 104   (when (kiwix-dir-detect)
 105     (car (directory-files "~/.www.kiwix.org/kiwix" nil ".*\\.default\\'")))
 106   "Specify the default Kiwix data profile path."
 107   :type 'string)
 108
 109 (defcustom kiwix-default-data-dir
 110   (when (kiwix-dir-detect)
 111     (concat "~/.www.kiwix.org/kiwix/" kiwix-default-data-profile-name))
 112   "Specify the default Kiwix data directory."
 113   :type 'string
 114   :safe #'stringp)
 115
 116 (defcustom kiwix-default-library-dir
 117   (file-name-directory (concat kiwix-default-data-dir "/data/library/library.xml"))
 118   "Kiwix libraries path."
 119   :type 'string
 120   :safe #'stringp)
 121
 122 (defcustom kiwix-default-completing-read (cond ((fboundp 'ivy-read) 'ivy)
 123                                                ((fboundp 'helm) 'helm))
 124   "Kiwix default completion frontend.
 125 Currently Ivy (`ivy') and Helm (`helm') both supported."
 126   :type 'symbol
 127   :safe #'symbolp)
 128
 129 (defcustom kiwix-default-browser-function browse-url-browser-function
 130   "Set default browser for open kiwix query result URL."
 131   :type '(choice
 132           (const :tag "browse-url default function" browse-url-default-browser)
 133           (const :tag "EWW" eww-browse-url)
 134           (const :tag "EAF web browser" eaf-open-browser)
 135           (const :tag "Firefox web browser" browse-url-firefox)
 136           (const :tag "Google Chrome web browser" browse-url-chrome)
 137           (const :tag "Conkeror web browser" browse-url-conkeror)
 138           (const :tag "xwidget browser" xwidget-webkit-browse-url))
 139   :safe #'symbolp)
 140
 141 (defun kiwix--get-library-name (file)
 142   "Extract library name from library file."
 143   (replace-regexp-in-string "\\.zim\\'" "" file))
 144
 145 (defun kiwix-get-libraries ()
 146   "Check out all available Kiwix libraries."
 147   (when (kiwix-dir-detect)
 148     (mapcar #'kiwix--get-library-name
 149             (directory-files kiwix-default-library-dir nil ".*\\.zim\\'"))))
 150
 151 (defvar kiwix-libraries (kiwix-get-libraries)
 152   "A list of Kiwix libraries.")
 153
 154 (defun kiwix-libraries-refresh ()
 155   "A helper function to refresh available Kiwx libraries."
 156   (setq kiwix-libraries (kiwix-get-libraries)))
 157
 158 (defvar kiwix--selected-library nil
 159   "Global variable of currently select library used in anonymous function.
 160 Like in function `kiwix-ajax-search-hints'.")
 161
 162 ;; - examples:
 163 ;; - "wikipedia_en_all" - "wikipedia_en_all_2016-02"
 164 ;; - "wikipedia_zh_all" - "wikipedia_zh_all_2015-17"
 165 ;; - "wiktionary_en_all" - "wiktionary_en_all_2015-17"
 166 ;; - "wiktionary_zh_all" - "wiktionary_zh_all_2015-17"
 167 ;; - "wikipedia_en_medicine" - "wikipedia_en_medicine_2015-17"
 168
 169 (defun kiwix-select-library (&optional filter)
 170   "Select Kiwix library name."
 171   (kiwix-libraries-refresh)
 172   (completing-read "Kiwix library: " kiwix-libraries nil t filter))
 173
 174 (defcustom kiwix-default-library "wikipedia_en_all.zim"
 175   "The default kiwix library when library fragment in link not specified."
 176   :type 'string
 177   :safe #'stringp)
 178
 179 (defcustom kiwix-mode-prefix nil
 180   "Specify kiwix-mode keybinding prefix before loading."
 181   :type 'kbd)
 182
 183 ;; update kiwix server url and port
 184 (defun kiwix-server-url-update ()
 185   "Update `kiwix-server-url' everytime used.
 186 In order to fix user config setting port after kiwix already initialized."
 187   (setq kiwix-server-url (format "http://127.0.0.1:%s" kiwix-server-port)))
 188
 189 ;; launch Kiwix server
 190 ;;;###autoload
 191 (defun kiwix-launch-server ()
 192   "Launch Kiwix server."
 193   (interactive)
 194   (let ((library-path kiwix-default-library-dir))
 195     (if kiwix-server-use-docker
 196         (start-process
 197          "kiwix-server"
 198          " *kiwix server*"
 199          "docker"
 200          "container" "run" "-d"
 201          "--name" "kiwix-serve"
 202          "-v" (concat (file-name-directory library-path) ":" "/data")
 203          "kiwix/kiwix-serve"
 204          "--library" "library.xml")
 205       (start-process
 206        "kiwix-server"
 207        " *kiwix server*"
 208        kiwix-server-command
 209        "--port" kiwix-server-port
 210        "--daemon"
 211        "--library" (concat library-path "library.xml")))))
 212
 213 (defun kiwix-capitalize-first (string)
 214   "Only capitalize the first word of STRING."
 215   (concat (string (upcase (aref string 0))) (substring string 1)))
 216
 217 (defun kiwix-query (query &optional selected-library)
 218   "Search `QUERY' in `LIBRARY' with Kiwix."
 219   (kiwix-server-url-update)
 220   (let* ((library (or selected-library (kiwix--get-library-name kiwix-default-library)))
 221          (url (concat kiwix-server-url "/search?content=" library "&pattern=" (url-hexify-string query)))
 222          (browse-url-browser-function kiwix-default-browser-function))
 223     (browse-url url)))
 224
 225 (defun kiwix-docker-check ()
 226   "Make sure Docker image 'kiwix/kiwix-server' is available."
 227   (let ((docker-image (replace-regexp-in-string
 228                        "\n" ""
 229                        (shell-command-to-string
 230                         "docker image ls kiwix/kiwix-serve | sed -n '2p' | cut -d ' ' -f 1"))))
 231     (string-equal docker-image "kiwix/kiwix-serve")))
 232
 233 (defvar kiwix-server-available? nil
 234   "The kiwix-server current available?")
 235
 236 (defun kiwix-ping-server ()
 237   "Ping Kiwix server to set `kiwix-server-available?' global state variable."
 238   (when kiwix-server-use-docker
 239     (unless (kiwix-docker-check)
 240       (async-shell-command "docker pull kiwix/kiwix-serve")))
 241   (let ((inhibit-message t))
 242     (kiwix-server-url-update)
 243     (request kiwix-server-url
 244       :type "GET"
 245       :sync t
 246       :parser (lambda () (libxml-parse-html-region (point-min) (point-max)))
 247       :error (cl-function
 248               (lambda (&rest args &key error-thrown &allow-other-keys)
 249                 (setq kiwix-server-available? nil)
 250                 (when (string-equal (cdr error-thrown) "exited abnormally with code 7\n")
 251                   (warn "kiwix.el failed to connect to host. exited abnormally with status code: 7."))))
 252       :success (cl-function
 253                 (lambda (&key _data &allow-other-keys)
 254                   (setq kiwix-server-available? t)))
 255       :status-code '((404 . (lambda (&rest _) (message (format "Endpoint %s does not exist." url))))
 256                      (500 . (lambda (&rest _) (message (format "Error from  %s." url))))))))
 257
 258 (defun kiwix-ajax-search-hints (input &optional selected-library)
 259   "Instantly AJAX request to get available Kiwix entry keywords
 260 list and return a list result."
 261   (kiwix-server-url-update)
 262   (kiwix-ping-server)
 263   (when (and input kiwix-server-available?)
 264     (let* ((library (or selected-library
 265                         (kiwix--get-library-name (or kiwix--selected-library
 266                                                      kiwix-default-library))))
 267            (ajax-api (format "%s/suggest?content=%s&term="
 268                              kiwix-server-url
 269                              library))
 270            (ajax-url (concat ajax-api input))
 271            (data (request-response-data
 272                   (let ((inhibit-message t))
 273                     (request ajax-url
 274                       :type "GET"
 275                       :sync t
 276                       :headers '(("Content-Type" . "application/json"))
 277                       :parser #'json-read
 278                       :success (cl-function
 279                                 (lambda (&key data &allow-other-keys)
 280                                   data)))))))
 281       (if (vectorp data) (mapcar #'cdar data)))))
 282
 283 (defun kiwix--get-thing-at-point ()
 284   "Get region select text or symbol at point."
 285   (if mark-active
 286       (buffer-substring
 287        (region-beginning) (region-end))
 288     (thing-at-point 'symbol)))
 289
 290 ;;;###autoload
 291 (defun kiwix-at-point ()
 292   "Search for the symbol at point with `kiwix-query'."
 293   (interactive)
 294   (unless (kiwix-ping-server)
 295     (kiwix-launch-server))
 296   (if kiwix-server-available?
 297       (progn
 298         (setq kiwix--selected-library (kiwix-select-library))
 299         (let* ((library kiwix--selected-library)
 300                (query (pcase kiwix-default-completing-read
 301                         ('helm
 302                          (require 'helm)
 303                          (helm :source (helm-build-async-source "kiwix-helm-search-hints"
 304                                          :candidates-process
 305                                          (lambda (input)
 306                                            (apply #'kiwix-ajax-search-hints
 307                                                   input `(,kiwix--selected-library))))
 308                                :input (kiwix--get-thing-at-point)
 309                                :buffer "*helm kiwix completion candidates*"))
 310                         ('ivy
 311                          (require 'ivy)
 312                          (ivy-read "Kiwix related entries: "
 313                                    (lambda (input)
 314                                      (apply #'kiwix-ajax-search-hints
 315                                             input `(,kiwix--selected-library)))
 316                                    :predicate nil
 317                                    :require-match nil
 318                                    :initial-input (kiwix--get-thing-at-point)
 319                                    :preselect nil
 320                                    :def nil
 321                                    :history nil
 322                                    :keymap nil
 323                                    :update-fn 'auto
 324                                    :sort t
 325                                    :dynamic-collection t
 326                                    :caller 'ivy-done)))))
 327           (message (format "library: %s, query: %s" library query))
 328           (if (or (null library)
 329                   (string-empty-p library)
 330                   (null query)
 331                   (string-empty-p query))
 332               (error "Your query is invalid")
 333             (kiwix-query query library))))
 334     (warn "kiwix-serve is not available, please start it at first."))
 335   (setq kiwix-server-available? nil))
 336
 337 ;;===============================================================================
 338
 339 (defun kiwix-mode-enable ()
 340   "Enable kiwix-mode."
 341   )
 342
 343 (defun kiwix-mode-disable ()
 344   "Disable kiwix-mode."
 345   )
 346
 347 (defvar kiwix-mode-map
 348   (let ((map (make-sparse-keymap)))
 349     map)
 350   "kiwix-mode map.")
 351
 352 ;;;###autoload
 353 (define-minor-mode kiwix-mode
 354   "Kiwix global minor mode for searching Kiwix serve."
 355   :global t
 356   :lighter " Kiwix"
 357   (if kiwix-mode (kiwix-mode-enable) (kiwix-mode-disable)))
 358
 359 \f
 360 (provide 'kiwix)
 361
 362 ;;; kiwix.el ends here