Babcore: Emacs Customizations everyone should have

Update (2017-05): babcore is at 0.2, but I cannot currently update the marmalade package.

C-g: abort

;; Copyright (C) 2013 Arne Babenhauserheide

;; Author: Arne Babenhauserheide (and various others in Emacswiki and elsewhere).
;; Maintainer: Arne Babenhauserheide
;; Created 03 April 2013
;; Version: 0.1.0
;; Version Keywords: core configuration

;; This program is free software; you can redistribute it and/or
;; modify it under the terms of the GNU General Public License
;; as published by the Free Software Foundation; either version 3
;; of the License, or (at your option) any later version.

;; This program is distributed in the hope that it will be useful,
;; but WITHOUT ANY WARRANTY; without even the implied warranty of
;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
;; GNU General Public License for more details.

;; You should have received a copy of the GNU General Public License
;; along with this program. If not, see <http://www.gnu.org/licenses/>.

;;; Commentary:
;; Quick Start / installation:
;; 1. Download this file and put it next to other files Emacs includes
;; 2. Add this to you .emacs file and restart emacs:
;; 
;;      (require 'babcore)
;;
;; Alternatively install via package.el:
;;
;;      (require 'package)
;;      (add-to-list 'package-archives '("marmalade" . "http://marmalade-repo.org/packages/"))
;;      (package-refresh-contents)
;;      (package-install 'babcore)
;; 
;; Use Case: Use a common core configuration so you can avoid the
;;   tedious act of gathering all the basic stuff over the years and
;;   can instead concentrate on the really cool new stuff Emacs offers
;;   you.
;;

;;; Change Log:

;; 2016-06-05 - 0.1.0: replace desktop with better savehist config and
;;                     cleanup babcore. Replace flymake with flycheck.
;;                     Remove the eval-region key-chord. Simplify
;;                     x-urgent. Fix switching back from full-screen
;;                     mode. Remove babcore-shell-execute, since
;;                     async-shell-command (M-&) is a built-in which
;;                     does the job better. Add C-M-. as third alias
;;                     for goto-last-change. Add find-file-as-root and
;;                     a few fixes for encumbering behavior.
;; 2013-11-02 - Disable clipboard sync while exporting with org-mode
;;              org-export-dispatch
;; 2013-10-22 - More useful frame titles
;; 2013-04-03 - Minor adjustments
;; 2013-02-29 - Initial release

;;; Code:

;; Convenient package handling in emacs

(require 'package)
;; use packages from marmalade
(add-to-list 'package-archives '("marmalade" . "http://marmalade-repo.org/packages/"))
;; and the old elpa repo
(add-to-list 'package-archives '("elpa-old" . "http://tromey.com/elpa/"))
;; and automatically parsed versiontracking repositories.
(add-to-list 'package-archives '("melpa" . "http://melpa.milkbox.net/packages/"))

;; Make sure a package is installed
(defun package-require (package)
  "Install a PACKAGE unless it is already installed 
or a feature with the same name is already active.

Usage: (package-require 'package)"
  ; try to activate the package with at least version 0.
  (package-activate package '(0))
  ; try to just require the package. Maybe the user has it in his local config

  (condition-case nil
      (require package)
    ; if we cannot require it, it does not exist, yet. So install it.
    (error (progn
             (package-install package)
             (require package)))))

;; Initialize installed packages
(package-initialize)  
;; package init not needed, since it is done anyway in emacs 24 after reading the init
;; but we have to load the list of available packages, if it is not available, yet.
(when (not package-archive-contents)
  (with-timeout (15 (message "updating package lists failed due to timeout"))
    (package-refresh-contents)))

;; Flycheck: On the fly syntax checking
(package-require 'flycheck)
(add-hook 'after-init-hook #'global-flycheck-mode)
; stronger error display
(defface flycheck-error
  '((t (:foreground "red" :underline (:color "Red1" :style wave) :weight bold)))
  "Flycheck face for errors"
  :group "flycheck")

;; Inline auto completion and suggestions
(package-require 'auto-complete)
;; avoid competing with org-mode templates.
(add-hook 'org-mode-hook
          (lambda ()
            (make-local-variable 'ac-stop-words)
            (loop for template in org-structure-template-alist do
                  (add-to-list 'ac-stop-words 
                               (concat "<" (car template))))))

; use ido mode for file and buffer Completion when switching buffers
(require 'ido)
(ido-mode t)

;; Convenient printing
(require 'printing)
(pr-update-menus t)
; make sure we use localhost as cups server
(setenv "CUPS_SERVER" "localhost")
(package-require 'cups)

; use allout minor mode to have outlining everywhere.
(allout-mode)

; syntax highlighting everywhere
(global-font-lock-mode 1)

; Activate org-mode
(require 'org)
; and some more org stuff

; http://orgmode.org/guide/Activation.html#Activation

; The following lines are always needed.  Choose your own keys.
(add-to-list 'auto-mode-alist '("\\.org\\'" . org-mode))

; And add babel inline code execution
; babel, for executing code in org-mode.
(org-babel-do-load-languages
 'org-babel-load-languages
 ; load all language marked with (lang . t).
 '((C . t)
   (R . t)
   (asymptote)
   (awk)
   (calc)
   (clojure)
   (comint)
   (css)
   (ditaa . t)
   (dot . t)
   (emacs-lisp . t)
   (fortran)
   (gnuplot . t)
   (haskell)
   (io)
   (java)
   (js)
   (latex)
   (ledger)
   (lilypond)
   (lisp)
   (matlab)
   (maxima)
   (mscgen)
   (ocaml)
   (octave)
   (org . t)
   (perl)
   (picolisp)
   (plantuml)
   (python . t)
   (ref)
   (ruby)
   (sass)
   (scala)
   (scheme)
   (screen)
   (sh . t)
   (shen)
   (sql)
   (sqlite)))

; Add proper word wrapping
(global-visual-line-mode t)

; go to the last change
(package-require 'goto-chg)
(global-set-key [(control .)] 'goto-last-change)
; M-. can conflict with etags tag search. But C-. can get overwritten
; by flyspell-auto-correct-word. And goto-last-change needs a really
; fast key.
(global-set-key [(meta .)] 'goto-last-change)
; ensure that even in worst case some goto-last-change is available
(global-set-key [(control meta .)] 'goto-last-change)

(require 'flyspell)
; Make german umlauts work.
(setq locale-coding-system 'utf-8)
(set-terminal-coding-system 'utf-8)
(set-keyboard-coding-system 'utf-8)
(set-selection-coding-system 'utf-8)
(prefer-coding-system 'utf-8)

;aspell und flyspell
(setq-default ispell-program-name "aspell")

;make aspell faster but less correctly
(setq ispell-extra-args '("--sug-mode=ultra" "-w" "äöüÄÖÜßñ"))
(setq ispell-list-command "list")

; control-lock-mode, so we can enter a vi style command-mode with standard emacs keys.
(package-require 'control-lock)
; also bind M-ü and M-ä to toggling control lock.
(global-set-key (kbd "M-ü") 'control-lock-toggle)
(global-set-key (kbd "C-ü") 'control-lock-toggle)
(global-set-key (kbd "M-ä") 'control-lock-toggle)
(global-set-key (kbd "C-ä") 'control-lock-toggle)
(global-set-key (kbd "C-z") 'control-lock-toggle)

; use key chords invoke commands
(package-require 'key-chord)
(key-chord-mode 1)
; buffer actions
(key-chord-define-global "vb"     'eval-buffer)
(key-chord-define-global "cy"     'yank-pop)
(key-chord-define-global "cg"     "\C-c\C-c")
; frame actions
(key-chord-define-global "xo"     'other-window);
(key-chord-define-global "x1"     'delete-other-windows)
(key-chord-define-global "x0"     'delete-window)
(defun kill-this-buffer-if-not-modified ()
  (interactive)
  ; taken from menu-bar.el
  (if (menu-bar-non-minibuffer-window-p)
      (kill-buffer-if-not-modified (current-buffer))
    (abort-recursive-edit)))
(key-chord-define-global "xk"     'kill-this-buffer-if-not-modified)
; file actions
(key-chord-define-global "bf"     'ido-switch-buffer)
(key-chord-define-global "cf"     'ido-find-file)
(key-chord-define-global "vc"     'vc-next-action)

emacsclient -e "(show-frame)"

(defun show-frame (&optional frame)
  "Show the current Emacs frame or the FRAME given as argument.

And make sure that it really shows up!"
  (raise-frame)
  ; yes, you have to call this twice. Don’t ask me why…
  ; select-frame-set-input-focus calls x-focus-frame and does a bit of
  ; additional magic.
  (select-frame-set-input-focus (selected-frame))
  (select-frame-set-input-focus (selected-frame)))

;; let emacs blink when something interesting happens.
;; in KDE this marks the active Emacs icon in the tray.
(defun x-urgency-hint (frame arg &optional source)
  "Set the x-urgency hint for the frame to arg: 

- If arg is nil, unset the urgency.
- If arg is any other value, set the urgency.

If you unset the urgency, you still have to visit the frame to make the urgency setting disappear (at least in KDE)."
    (let* ((wm-hints (append (x-window-property 
                "WM_HINTS" frame "WM_HINTS" source nil t) nil))
     (flags (car wm-hints)))
    (setcar wm-hints
        (if arg
        (logior flags #x100)
          (logand flags (lognot #x100))))
    (x-change-window-property "WM_HINTS" wm-hints frame "WM_HINTS" 32 t)))

(defun x-urgent (&optional arg)
  "Mark the current emacs frame as requiring urgent attention. 

With a prefix argument which does not equal a boolean value of nil, remove the urgency flag (which might or might not change display, depending on the window manager)."
  (interactive "P")
  (let (frame (selected-frame))
  (x-urgency-hint frame (not arg))))

; fullscreen, taken from http://www.emacswiki.org/emacs/FullScreen#toc26
; should work for X und OSX with emacs 23.x (TODO find minimum version).
; for windows it uses (w32-send-sys-command #xf030) (#xf030 == 61488)
(defvar babcore-fullscreen-p nil
  "Check if fullscreen is on or off")
(defvar babcore-stored-frame-width nil 
  "width of the frame before going fullscreen")
(defvar babcore-stored-frame-height nil
  "width of the frame before going fullscreen")

(defun babcore-non-fullscreen ()
  (interactive)
  (if (fboundp 'w32-send-sys-command)
      ;; WM_SYSCOMMAND restore #xf120
      (w32-send-sys-command 61728)
    (progn (set-frame-parameter nil 'fullscreen nil)
           (set-frame-parameter nil 'width 
                                (if babcore-stored-frame-width
                                    babcore-stored-frame-width 82))
           (sleep-for 0 1) ; 1ms sleep: workaround to avoid unsetting the width in the next command
           (set-frame-parameter nil 'height
                                (if babcore-stored-frame-height 
                                    babcore-stored-frame-height 42)))))

(defun babcore-fullscreen ()
  (interactive)
  (setq babcore-stored-frame-width (frame-width))
  (setq babcore-stored-frame-height (frame-height))
  (if (fboundp 'w32-send-sys-command)
      ;; WM_SYSCOMMAND maximaze #xf030
      (w32-send-sys-command 61488)
    (set-frame-parameter nil 'fullscreen 'fullboth)))

(defun toggle-fullscreen ()
  (interactive)
  (setq babcore-fullscreen-p (not babcore-fullscreen-p))
  (if babcore-fullscreen-p
      (babcore-non-fullscreen)
    (babcore-fullscreen)))

(global-set-key [f11] 'toggle-fullscreen)

; Default KDE keybindings to make emacs nicer integrated into KDE. 

; can treat C-m as its own mapping.
; (define-key input-decode-map "\C-m" [?\C-1])

(defun revert-buffer-preserve-modes ()
  (interactive)
  (revert-buffer t nil t))

; C-m shows/hides the menu bar - thanks to http://stackoverflow.com/questions/2298811/how-to-turn-off-alternative-enter-with-ctrlm-in-linux
; f5 reloads
(defconst kde-default-keys-minor-mode-map
  (let ((map (make-sparse-keymap)))
    (set-keymap-parent map text-mode-map)
    (define-key map [f5] 'revert-buffer-preserve-modes)
    (define-key map [?\C-1] 'menu-bar-mode)
    (define-key map [?\C-+] 'text-scale-increase)
    (define-key map [?\C--] 'text-scale-decrease) ; shadows 'negative-argument which is also available via M-- and C-M--, though.
    (define-key map [C-kp-add] 'text-scale-increase)
    (define-key map [C-kp-subtract] 'text-scale-decrease)
    map)
  "Keymap for `kde-default-keys-minor-mode'.")

;; Minor mode for keypad control
(define-minor-mode kde-default-keys-minor-mode
  "Adds some default KDE keybindings"
  :global t
  :init-value t
  :lighter ""
  :keymap 'kde-default-keys-minor-mode-map
  )

;; Set the frame title as by http://www.emacswiki.org/emacs/FrameTitle
(setq frame-title-format (list "%b ☺ " (user-login-name) "@" (system-name) "%[ - GNU %F " emacs-version)
      icon-title-format (list "%b ☻ " (user-login-name) "@" (system-name) " - GNU %F " emacs-version))

M-x ucs-insert

;; Highlight TODO and FIXME in comments 
(package-require 'fic-ext-mode)
(defun add-something-to-mode-hooks (mode-list something)
  "helper function to add a callback to multiple hooks"
  (dolist (mode mode-list)
    (add-hook (intern (concat (symbol-name mode) "-mode-hook")) something)))

(add-something-to-mode-hooks '(c++ tcl emacs-lisp python text markdown latex) 'fic-ext-mode)

; save the current macro as reusable function.
(defun save-current-kbd-macro-to-dot-emacs (name)
  "Save the current macro as named function definition inside
your initialization file so you can reuse it anytime in the
future."
  (interactive "SSave Macro as: ")
  (name-last-kbd-macro name)
  (save-excursion 
    (find-file-literally user-init-file)
    (goto-char (point-max))
    (insert "\n\n;; Saved macro\n")
    (insert-kbd-macro name)
    (insert "\n")))

; Activate transparent GnuPG encryption.
(require 'epa-file)
(epa-file-enable)

(setq backup-by-copying t      ; don't clobber symlinks
      backup-directory-alist
      '(("." . "~/.local/share/emacs-saves"))    ; don't litter my fs tree
      delete-old-versions t
      kept-new-versions 6
      kept-old-versions 2
      version-control t)       ; use versioned backups

; save the place in files
(require 'saveplace)
(setq-default save-place t)

; save minibuffer history
(require 'savehist)
;; increase the default history cutoff
(setq history-length 500)
(savehist-mode t)
(setq savehist-additional-variables
      '(regexp-search-ring
        register-alist))

(setq x-select-enable-clipboard t)

; When I have x-select-enable-clipboard enabled, compiling an org-mode file to PDF locks
; KDE - I think it does so by filling up the clipboard.
(defadvice org-export-dispatch-no-clipboard-advice (around org-export-dispatch)
  "Do not clobber the system-clipboard while compiling an org-mode file with `org-export`."
  (let ((select-active-regions nil)
        (x-select-enable-clipboard nil)
        (x-select-enable-primary nil)
        (interprogram-cut-function nil)
        (interprogram-paste-function nil))
    ad-do-it))
(ad-activate 'org-export-dispatch-no-clipboard-advice t)

(package-require 'legalese)

;;; Open files as root - quickly
(defcustom find-file-root-prefix "/sudo:root@localhost:"
"Tramp root prefix to use.")

(defun find-file-as-root ()
  "Like `ido-find-file, but automatically edit the file with
root-privileges (using tramp/sudo), if the file is not writable by
user."
  (interactive)
  (let ((file (ido-read-file-name "Edit as root: ")))
    (unless (file-writable-p file)
      (setq file (concat find-file-root-prefix file)))
    (find-file file)))
;; or some other keybinding...
;; (global-set-key (kbd "C-x F") 'djcb-find-file-as-root)

(defun find-current-as-root ()
  "Reopen current file as root"
  (interactive)
  (set-visited-file-name (concat find-file-root-prefix (buffer-file-name)))
  (setq buffer-read-only nil))

;;;;;;;;;;;;;
;;; Fixes ;;;
;;;;;;;;;;;;;

;; Make comint recognize passwords in virtually all languages.
(defcustom comint-password-prompt-regexp
  (concat
   "\\(^ *\\|"
   (regexp-opt
    '("Enter" "enter" "Enter same" "enter same" "Enter the" "enter the"
      "Old" "old" "New" "new" "'s" "login"
      "Kerberos" "CVS" "UNIX" " SMB" "LDAP" "[sudo]" "Repeat" "Bad") t)
   " +\\)"
   (regexp-opt
    '("Adgangskode" "adgangskode" "Contrasenya" "contrasenya" "Contraseña" "contraseña" "Geslo" "geslo" "Hasło" "hasło" "Heslo" "heslo" "Iphasiwedi" "iphasiwedi" "Jelszó" "jelszó" "Lozinka" "lozinka" "Lösenord" "lösenord" "Mot de passe " "Mot de Passe " "mot de Passe " "mot de passe " "Mật khẩu " "mật khẩu" "Parola" "parola" "Pasahitza" "pasahitza" "Pass phrase" "pass Phrase" "pass phrase" "Passord" "passord" "Passphrase" "passphrase" "Password" "password" "Passwort" "passwort" "Pasvorto" "pasvorto" "Response" "response" "Salasana" "salasana" "Senha" "senha" "Wachtwoord" "wachtwoord" "slaptažodis" "slaptažodis" "Лозинка" "лозинка" "Пароль" "пароль" "ססמה" "كلمة السر" "गुप्तशब्द" "शब्दकूट" "গুপ্তশব্দ" "পাসওয়ার্ড" "ਪਾਸਵਰਡ" "પાસવર્ડ" "ପ୍ରବେଶ ସଙ୍କେତ" "கடவுச்சொல்" "సంకేతపదము" "ಗುಪ್ತಪದ" "അടയാളവാക്ക്" "රහස්පදය" "ពាក្យសម្ងាត់ ៖ " "パスワード" "密码" "密碼" "암호"))
   "\\(?:\\(?:, try\\)? *again\\| (empty for no passphrase)\\| (again)\\)?\
\\(?: for [^:]+\\)?:\\s *\\'")
  "Regexp matching prompts for passwords in the inferior process.
This is used by `comint-watch-for-password-prompt'."
  :version "24.3"
  :type 'regexp
  :group 'comint)

;; Mark all AC_* and AS_* functions as builtin.
(add-hook 'autoconf-mode-hook 
          (lambda () 
            (add-to-list 'autoconf-font-lock-keywords '("\\(\\(AC\\|AS\\|AM\\)_.+?\\)\\((\\|\n\\)" (1 font-lock-builtin-face)))))

; tell emacs to ignore alt-gr clicks needed for M4 in the Neo Layout.
(define-key special-event-map (kbd "<key-17>") 'ignore)
(define-key special-event-map (kbd "<M-key-17>") 'ignore)

; yank-pop should yank if the last command was no yank.
(defun yank-pop (&optional arg)
  "Replace just-yanked stretch of killed text with a different stretch.
At such a time, the region contains a stretch of reinserted
previously-killed text.  `yank-pop' deletes that text and inserts in its
place a different stretch of killed text.

With no argument, the previous kill is inserted.
With argument N, insert the Nth previous kill.
If N is negative, this is a more recent kill.

The sequence of kills wraps around, so that after the oldest one
comes the newest one.

When this command inserts killed text into the buffer, it honors
`yank-excluded-properties' and `yank-handler' as described in the
doc string for `insert-for-yank-1', which see."
  (interactive "*p")
  (if (not (eq last-command 'yank))
      (yank)
    (setq this-command 'yank)
    (unless arg (setq arg 1))
    (let ((inhibit-read-only t)
          (before (< (point) (mark t))))
      (if before
          (funcall (or yank-undo-function 'delete-region) (point) (mark t))
        (funcall (or yank-undo-function 'delete-region) (mark t) (point)))
      (setq yank-undo-function nil)
      (set-marker (mark-marker) (point) (current-buffer))
      (insert-for-yank (current-kill arg))
      ;; Set the window start back where it was in the yank command,
      ;; if possible.
      (set-window-start (selected-window) yank-window-start t)
      (if before
          ;; This is like exchange-point-and-mark, but doesn't activate the mark.
          ;; It is cleaner to avoid activation, even though the command
          ;; loop would deactivate the mark because we inserted text.
          (goto-char (prog1 (mark t)
                       (set-marker (mark-marker) (point) (current-buffer))))))
    nil))

(setq visible-bell t)

(provide 'babcore)
;;; babcore.el ends here

Note: As almost everything on this page, this text and code is available under the GPLv3 or later.

Conveniently convert CamelCase to words_with_underscores using a small emacs hack

I currently cope with refactoring in an upstream project to which I maintain some changes which upstream does not merge. One nasty part is that the project converted from CamelCase for function names to words_with_underscores. And that created lots of merge errors.

Today I finally decided to speed up my work.

The first thing I needed was a function to convert a string in CamelCase to words_with_underscores. Since I’m lazy, I used google, and that turned up the CamelCase page of Emacswiki - and with it the following string functions:

(defun split-name (s)
  (split-string
   (let ((case-fold-search nil))
     (downcase
      (replace-regexp-in-string "\\([a-z]\\)\\([A-Z]\\)" "\\1 \\2" s)))
   "[^A-Za-z0-9]+"))
(defun underscore-string (s) (mapconcat 'downcase   (split-name s) "_"))

Quite handy - and elegantly executed. Now I just need to make this available for interactive use. For this, Emacs Lisp offers many useful ways to turn Editor information into program information, called interactive codes - in my case the region-code: "r". This gives the function the beginning and the end of the currently selected region as arguments.

With this, I created an interactive function which de-camelcases and underscores the selected region:

(defun underscore-region (begin end) (interactive "r")
  (let* ((word (buffer-substring begin end))
         (underscored (underscore-string word)))
    (save-excursion
      (widen) ; break out of the subregion so we can fix every usage of the function
      (replace-string word underscored nil (point-min) (point-max)))))

And now we’re almost there. Just create a macro which searches for a function, selects its name, de-camelcaeses and underscores it and then replaces every usage of the CamelCase name by the underscored name. This isn’t perfect refactoring (can lead to errors), but it’s fast and I see every change it does.

C-x C-(
C-s def 
M-x mark-word
M-x underscore-region
C-x C-)

That’s it, now just call the macro repeatedly.

C-x eeeeee…

Now check the diff to fix where this 13 lines hack got something wrong ( like changing __init__ into _init_ - I won’t debug this, you’ve been warned ☺).

Happy Hacking!

Custom link completion for org-mode in 25 lines (emacs)

Update (2013-01-23): The new org-mode removed (org-make-link), so I replaced it with (concat) and uploaded a new example-file: org-custom-link-completion.el.
Happy Hacking!

1 Intro

I recently set up custom completion for two of my custom link types in Emacs org-mode. When I wrote on identi.ca about that, Greg Tucker-Kellog said that he’d like to see that. So I decided, I’d publish my code.

The link types I regularly need are papers (PDFs of research papers I take notes about) and bib (the bibtex entries for the papers). The following are my custom link definitions :

(setq org-link-abbrev-alist
      '(("bib" . "~/Dokumente/Uni/Doktorarbeit-inverse-co2-ch4/aufschriebe/ref.bib::%s")
       ("notes" . "~/Dokumente/Uni/Doktorarbeit-inverse-co2-ch4/aufschriebe/papers.org::#%s")
       ("papers" . "~/Dokumente/Uni/Doktorarbeit-inverse-co2-ch4/aufschriebe/papers/%s.pdf")))

For some weeks I had copied the info into the links by hand. Thus an entry about a paper looks like the following.

* Title [[bib:identifier]] [[papers:name_without_suffix]]

This already suffices to be able to click the links for opening the PDF or showing the bibtex entry. Entering the links was quite inconvenient, though.

(defun string-replace (this withthat in)
  "replace THIS with WITHTHAT' in the string IN"
  (with-temp-buffer
    (insert in)
    (goto-char (point-min))
    (replace-string this withthat)
    (buffer-substring (point-min) (point-max))))

<<string-replace>>
(defun org-papers-complete-link (&optional arg)
  "Create a papers link using completion."
  (let (file link)
       (setq file (read-file-name "papers: " "papers/"))
       <<cleanup-link>>
    link))

(let ((pwd (file-name-as-directory (expand-file-name ".")))
  (pwd1 (file-name-as-directory (abbreviate-file-name
                 (expand-file-name ".")))))
  (setq file (string-replace "papers/" "" file))
  (setq file (string-replace pwd "" (string-replace pwd1 "" file)))
  (setq file (string-replace ".pdf" "" file))
  (setq link (concat "papers:" file)))

<<reftex-setup>>
(defun org-bib-complete-link (&optional arg)
  "Create a bibtex link using reftex autocompletion."
  (concat "bib:" (reftex-do-citation nil t nil)))

(defun org-mode-reftex-setup ()
  (interactive)
  (and (buffer-file-name) (file-exists-p (buffer-file-name))
       (progn
        ; Reftex should use the org file as master file. See C-h v TeX-master for infos.
        (setq TeX-master t)
        (turn-on-reftex)
        ; don’t ask for the tex master on every start.
        (reftex-parse-all)
        ;add a custom reftex cite format to insert links
        (reftex-set-cite-format
         '((?b . "[[bib:%l][%l-bib]]")
           (?n . "[[notes:%l][%l-notes]]")
           (?p . "[[papers:%l][%l-paper]]")
           (?t . "%t")
           (?h . "** %t\n:PROPERTIES:\n:Custom_ID: %l\n:END:\n[[papers:%l][%l-paper]]")))))
  (define-key org-mode-map (kbd "C-c )") 'reftex-citation)
  (define-key org-mode-map (kbd "C-c (") 'org-mode-reftex-search))

(add-hook 'org-mode-hook 'org-mode-reftex-setup)

<<org-papers-complete-link>>
<<org-bib-complete-link>>

Easily converting ris-citations to bibtex with emacs and bibutils

The problem

Nature only gives me ris-formatted citations, but I use bibtex.

Also ris is far from human readable.

The background

ris can be reformatted to bibtext, but doing that manually disturbs my workflow when getting references while taking note about a paper in emacs.

I tend to search online for references, often just using google scholar, so when I find a ris reference, the first data I get for the ris-citation is a link.

The solution

Making it possible

bibutils¹ can convert ris to an intermediate xml format and then convert that to bibtex.

wget -O reference.ris RIS_URL
cat reference.ris | ris2xml | xml2bib >> ref.bib

This solves the problem, but it is not convenient, because I have to switch to the terminal, download the file, convert it and append the result to my bibtex file.

Making it convenient

With the first step, getting the ris-citation is quite inconvenient. I need 3 steps just for getting a citation.

But those steps are always the same, and since I use Emacs, I can automate and integrate them very easily. So I created a simple function in emacs, which takes the url of a ris citation, converts it to bibtex and appends the result to my local bibtex file. Now I get a ris citation with a simple call to

M-x ris-citation-to-bib

Then I enter the url and the function appends the citation to my bibtex file.²

Feel free to integrate it into your own emacs setup (additionally to the GPLv3 you can use any license used by emacswiki or worg).

(defun ris-citation-to-bib (&optional ris-url) 
  "get a ris citation as bibtex in one step. Just call M-x
ris-citation-to-bib and enter the ris url. 
Requires bibutils: http://sourceforge.net/p/bibutils/home/Bibutils/ 
"
  (interactive "Mris-url: ")
  (save-excursion
    (let ((bib-file "/home/arne/aufschriebe/ref.bib")
          (bib-buffer (get-buffer "ref.bib"))
          (ris-buffer (url-retrieve-synchronously ris-url)))
      ; firstoff check if we have the bib buffer. If yes, move point to the last line.
      (if (not (member bib-buffer (buffer-list)))
          (setq bib-buffer (find-file-noselect bib-file)))
      (progn 
        (set-buffer bib-buffer)
        (goto-char (point-max)))
      (if ris-buffer
          (set-buffer ris-buffer))
      (shell-command-on-region (point-min) (point-max) "ris2xml | xml2bib" ris-buffer)
      (let ((pmin (- (search-forward "@") 1))
            (pmax (search-forward "}

")))
        (if (member bib-buffer (buffer-list))
            (progn 
              (append-to-buffer bib-buffer pmin pmax)
              (kill-buffer ris-buffer)
              (set-buffer bib-buffer)
              (save-buffer)
          ))))))

Happy Hacking!

PS: When I don’t have the URL (many thanks to journals giving me only a download button), I open the file, select the content and hit M-| (shell-command-on-region) with ris2xml | xml2bib (searching backwards via C-r ris so I to avoid typing the exact command) and get the bibtex version in the results buffer.

El Kanban Org: parse org-mode todo-states to use org-tables as Kanban tables

Kanban for emacs org-mode.

Update (2020): Kanban moved to sourcehut: https://hg.sr.ht/~arnebab/kanban.el

Update (2013-04-13): Kanban.el now lives in its own repository: on bitbucket and on a statically served http-repo (to be independent from unfree software).

Update (2013-04-10): Thanks to Han Duply, kanban links now work for entries from other files. And I uploaded kanban.el on marmalade.

Some time ago I learned about kanban, and the obvious next step was: “I want to have a kanban board from org-mode”. I searched for it, but did not find any. Not wanting to give up on the idea, I implemented my own :)

The result are two functions: kanban-todo and kanban-zero.

“Screenshot” :)

TODO	DOING	DONE
Refactor in such a way that the
	let Presentation manage dumb sprites
return all actions on every command:
Make the UiState adhere the list of
Turn the model into a pure state

kanban-todo

kanban-todo provides your TODO items as kanban-fields. You can move them in the table without having duplicates, so all the state maintenance is done in the kanban table. Once you are finished, you mark them as done and delete them from the table.

To set it up, put kanban.el somewhere in your load path and (require 'kanban) (more recent but potentially unstable version). Then just add a table like the following:

|   |   |   |
|---+---+---|
|   |   |   |
|   |   |   |
|   |   |   |
|   |   |   |
#+TBLFM: $1='(kanban-todo @# @2$2..@>$>)::@1='(kanban-headers $#)

Click C-c C-c with the point on the TBLFMT line to update the table.

The important line is the #+TBLFM. That says “use my TODO items in the TODO column, except if they are in another column” and “add kanban headers for my TODO states”

The kanban-todo function takes an optional parameter match, which you can use to restrict the kanban table to given tags. The syntax is the same as for org-mode matchers. The third argument allows you to provide a scope, for example a list of files.

To only set the scope, use nil for the matcher.

See C-h f org-map-entries and C-h v org-agenda-files for details.

kanban-zero

kanban-zero is a zero-state Kanban: All state is managed in org-mode and the table only displays the kanban items.

To set it up, put kanban.el somwhere in your load path and (require 'kanban). Then just add a table like the following:

|   |   |   |
|---+---+---|
|   |   |   |
|   |   |   |
|   |   |   |
|   |   |   |
#+TBLFM: @2$1..@>$>='(kanban-zero @# $#)::@1='(kanban-headers $#)

The important line is the #+TBLFM. That says “show my org items in the appropriate column” and “add kanban headers for my TODO states”.

Click C-c C-c with the point on the TBLFMT line to update the table.

The kanban-zero function takes an optional parameter match, which you can use to restrict the kanban table to given tags. The syntax is the same as for org-mode matchers. The third argument allows you to provide a scope, for example a list of files.

To only set the scope, use nil for the matcher.

An example for matcher and scope would be:

#+TBLFM: @2$1..@>$>='(kanban-zero @# $# "1w6" '("/home/arne/.emacs.d/private/org/emacs-plan.org"))::@1='(kanban-headers $#)

See C-h f org-map-entries and C-h v org-agenda-files for details.

Contribute

To contribute to kanban.el, just change the file and write a comment about your changes. Maybe I’ll setup a repo on Bitbucket at some point…

Example

In the Hexbattle game-draft, I use kanban to track my progress:

Table of Contents

• 1 Kanban
• 2 refactor Hexbattle
  • 2.1 Structure plan:
  • 2.2 First all logic from UiState into the model. Model gets all state which is independent of the UI. Any necessary interaction in UiState calls model methods. √
  • 2.3 Refactor in such a way that the UiState and (even more importantly) the input do not need to call the model directly anymore.
    • 2.3.1 let Presentation manage dumb sprites for all chars which are active. Then I can remove a lot of UiState dependent functionality from the chars.
    • 2.3.2 return all actions on every command: command(…) → commands (instance of Commands).
  • 2.4 Make the UiState adhere the list of possible commands.
  • 2.5 Turn the model into a pure state machine.

1 Kanban

❢	STARTED	✔
Refactor in such a way that the
	let Presentation manage dumb sprites
return all actions on every command:
Make the UiState adhere the list of
Turn the model into a pure state

2 refactor Hexbattle    1w6

… and so on …

Advanced usage

“Graphical” TODO states

To make the todo states easier to grok directly you can use unicode symbols for them. Example:

#+SEQ_TODO: ❢ ☯ ⧖ | ☺ ✔ DEFERRED ✘

| ❢ | ☯ | ⧖ | ☺ |
|---+---+---+---|
|   |   |   |   |
#+TBLFM: @1='(kanban-headers $#)::@2$1..@>$>='(kanban-zero @# $#)

In my setup they are ❢ (todo) ☯ (doing) ⧖ (waiting) and ☺ (to report). Not shown in the kanban Table are ✔ (finished), ✘ (dropped) and deferred (later), because they don’t require any action from me, so I don’t need to see them all the time.

Collecting kanban entries via SSH

If you want to create a shared kanban table, you can use the excellent transparent network access options from Emacs tramp to collect kanban entries directly via SSH.

To use that, simply pass an explicit list of files to kanban-zero as 4th argument (if you don’t use tag matching just use nil as 3rd argument). "/ssh:host:path/to/file.org" retrieves the file ~/path/to/file.org from the host.

| ❢ | ☯ |
|---+---|
|   |   |
#+TBLFM: @1='(kanban-headers $#)::@2$1..@>$>='(kanban-zero @# $# nil (list (buffer-file-name) "/ssh:localhost:plan.org"))

Caveeat: all included kanban files have to use at least some of the same todo states: kanban.el only retrieves TODO states which are used in the current buffer.

How to show the abstract before the table of contents in org-mode

#+OPTIONS: toc:nil

#+BEGIN_ABSTRACT
Abstract
#+END_ABSTRACT
#+TOC: headlines 2

# Local Variables:
# org-export-with-toc: nil
# End:

#+BEGIN_ABSTRACT
Abstract
#+END_ABSTRACT
#+LATEX: \tableofcontents

IRC-chat via Tor with Emacs on Gentoo

As example: Connecting to #youbroketheinternet.

emerge privoxy torsocks net-vpn/tor
# rc-config start privoxy tor
# rc-update add privoxy default
# rc-update add tor default
mkdir -p ~/.local/EMACS_TOR_HOME/.emacs.d
echo "(require 'socks)" >> ~/.local/EMACS_TOR_HOME/.emacs.d/init.el
HOME=~/.local/EMACS_TOR_HOME torify emacs --title "Emacs-torified"
# M-x customize-variable RET socks-server RET
#   host: localhost
#   port: 9050
#   type: Socks v5
#   (C-x C-s to save and set)
# M-x erc-select
#   server loupsycedyglgamf.onion
#   port 67
# the welcome channel is good to go.

See https://www.emacswiki.org/emacs/ErcProxy#toc2

and http://youbroketheinternet.org/#overlay

Insert a scaled screenshot in emacs org-mode

@marjoleink asked on identi.ca¹, if it is possible to use emacs org-mode for showing scaled screenshots inline while writing. Since I thought I’d enjoy some hacking, I decided to take the challenge.

It does not do auto-scaling of embedded images, as far as I know, but the use case of screenshots can be done with a simple function (add this to your ~/.emacs or ~/.emacs.d/init.el):

(defun org-insert-scaled-screenshot ()
  "Insert a scaled screenshot 
for inline display 
into your org-mode buffer."
  (interactive)
  (let ((filename 
         (concat "screenshot-" 
                 (substring 
                  (shell-command-to-string 
                   "date +%Y%m%d%H%M%S")
                  0 -1 )
                 ".png")))
    (let ((scaledname 
           (concat filename "-width300.png")))

      (shell-command 
       (concat "import -window root " 
               filename))
      (shell-command 
       (concat "convert -adaptive-resize 300 " 
               filename " " scaledname))
      (insert (concat "[[./" scaledname "]]")))))

Now just call M-x org-redisplay-inline-images to see the screenshot (or add it to the function).

In action:

Have fun with Emacs - and happy hacking!

PS: In case it’s not obvious: The screenshot shows emacs just as the screenshot is being shot - with the method shown here ☺)

Minimal example for literate programming with noweb in emacs org-mode

If you want to use the literate programming features in emacs org-mode, you can try this minimal example to get started: Activate org-babel-tangle, then put this into the file noweb-test.org:

Minimal example for noweb in org-mode

* Assign 

First we assign abc:

#+begin_src python :noweb-ref assign_abc
abc = "abc"
#+end_src

* Use

Then we use it in a function:

#+begin_src python :noweb tangle :tangle noweb-test.py
def x():
  <<assign_abc>>
  return abc

print(x())
#+end_src

noweb-test.org

Hit C-c C-c to evaluate the source block. Hit C-c C-v C-t to put the expanded code into the file noweb-test.py.

The exported code looks like this:

def x():
  abc = "abc"
  return abc

print(x())

noweb-test.py

(html generated with org-export-as-html-to-buffer and slightly reniced to escape the additional parsing I have on my site)

And with org-export-as-pdf we get this:

noweb-test.pdf

Add :results output to the #+begin_src line of the second block to see the print results under that block when you hit C-c C-c in the block.

You can also use properties of headlines for giving the noweb-ref. Org-mode can then even concatenate several source blocks into one noweb reference. Just hit C-c C-x p to set a property (or use M-x org-set-property), then set noweb-ref to the name you want to use to embed all blocks under this heading together.

Note: org-babel prefixes each line of an included code-block with the prefix used for the reference (here <<assign_abc>>). This way you can easily include blocks inside python functions.

Note: To keep noweb-references literally in the output or similar, have a look at the different options to :noweb.

Note: To do this with shell-code, it’s useful to change the noweb markers to {{{ and }}}, because << and >> are valid shell-syntax, so they disturb the highlighting in sh-mode. Also confirming the evaluation every time makes plain exporting problematic. To fix this, just add the following somewhere in the file (to keep this simple, just add it to the end):

# Local Variables:
# org-babel-noweb-wrap-start: "{{{"
# org-babel-noweb-wrap-end: "}}}"
# org-confirm-babel-evaluate: nil
# org-export-allow-bind-keywords: t
# End:

Have fun with Emacs and org-mode!

Org-mode with Parallel Babel

Update 2017: a block with sem -j ... seems to block in recent versions of Emacs until all subtasks are done. It would be great if someone could figure out why (though it likely is the right thing to do). To circumvent that, you can daemonize the job in sem, but that might have unwanted side-effects: sem "[job] &"

; Execute babel source blocks asynchronously by just opening a new emacs.
(defun bab/org-babel-execute-src-block-new-emacs ()
  "Execute the current source block in a separate emacs,
so we do not block the current emacs."
  (interactive)
  (let ((line (line-number-at-pos))
        (file (buffer-file-name)))
    (async-shell-command (concat 
                          "TERM=vt200 emacs -nw --find-file " 
                          file 
                          " --eval '(goto-line "
                          (number-to-string line) 
                          ")' --eval "
     "'(let ((org-confirm-babel-evaluate nil))(org-babel-execute-src-block t))' "
                          "--eval '(kill-emacs 0)'"))))

; Export as pdf asynchronously by just opening a new emacs.
(defun bab/org-beamer-export-new-emacs ()
  "Export the current file in a separate emacs,
so we do not block the current emacs."
  (interactive)
  (let ((line (line-number-at-pos))
        (file (buffer-file-name)))
    (async-shell-command (concat 
                          "TERM=vt200 emacs -nw --find-file " 
                          file 
                          " --eval '(goto-line " 
                          (number-to-string line) 
                          ")' --eval "
     "'(let ((org-confirm-babel-evaluate nil))(org-beamer-export-to-pdf))' "
                          "--eval '(kill-emacs 0)'"))))

#+BEGIN_SRC sh :exports none
  oldPWD=$(pwd)
  cd ~/tm5tools/plotting
  filename="./obsheat-increasing.png" >/dev/null 2>/dev/null
  sem -j -1 ./plotstation.py -c ~/sun-work/ct-production-out-5x7e300m1.0 -C "aircraft" -c ~/sun-work/ct-production-out-5x7e300m1.0no-aircraft -C "continuous"  --obsheat --station allnoaa --title "\"Reducing observation coverage\"" -o ${oldPWD}/${filename}
  cd -
#+END_SRC

for i in {1..100}; do 
    sem -j -1 [code] # run N-1 processes with N as the number of
                     # pocessors in my computer
done

sem --wait

sem -j -1 ./plotstation.py --title "\"Reducing observation coverage\""

(setq org-src-fontify-natively t)

sem -j -1 [escaped code]

Publish a single file with emacs org-mode

(setq private-publish-ftp-proj (concat "/ftp:" USER "@" HOST ":arnebab/proj/"))

(setq org-publish-project-alist
      '(("orgmode-single-file"
         :base-directory "~/.emacs.d/private/journal"
         :publishing-directory (concat private-publish-ftp-proj "orgmode-single-file/")
         :base-extension "org"
         :publishing-function org-html-publish-to-html
         :completion-function (lambda () (rename-file 
                                          (concat private-publish-ftp-proj 
                                                  "orgmode-single-file/2013-11-25-Mo-publish-single-file-org-mode.html") 
                                          (concat private-publish-ftp-proj 
                                                  "orgmode-single-file/index.html") t))
         :section-numbers nil
         :with-toc t
         :html-preamble t
         :exclude ".*"
         :include ["2013-11-25-Mo-publish-single-file-org-mode.org"])))

Read your python module documentation from emacs

Update 2021: Fixed links that died with Bitbuckets hosting.

I just found the excellent pydoc-info mode for emacs from Jon Waltman. It allows me to hit C-h S in a python file and enter a module name to see the documentation right away. If the point is on a symbol (=module or class or function), I can just hit enter to see its docs.

In its default configuration (see the Readme) it “only” reads the python documentation. This alone is really cool when writing new python code, but it s not enough, since I often use third party modules.

And now comes the treat: If those modules use sphinx for documentation (≥1.1), I can integrate them just like the standard python documentation!

It took me some time to get it right, but now I have all the documentation for the inverse modelling framework I contribute to directly at my fingertips: Just hit C-h S ENTER when I’m on some symbol and a window shows me the docs:

The text in this image is from Wouter Peters. Used here as short citation which should be legal almost everywhere under citation rules.

I want to save you the work of figuring out how to do that yourself, so here’s a short guide for integrating the documentation for your python program into emacs.

Integrating your own documentation into emacs

The prerequisite for integrating your own documentation is to use sphinx for documenting your code. See their tutorial for info how to set it up. As soon as sphinx works for you, follow this guide to integrate your docs in your emacs.

Install pydoc-info

First get pydoc-info and the python infofile (adapt this to your local setup):

# get the mode
cd ~/.emacs.d/libs
hg clone https://hg.sr.ht/~arnebab/pydoc-info
# and the pregenerated info-file for python
wget http://www.draketo.de/dateien/python.info.gz
gunzip python.info.gz
sudo cp python.info /usr/share/info
sudo install-info --info-dir=/usr/share/info python.info

To build the info file for python yourself, have a look at the Readme.

Turn your documentation into info

Now turn your own documentation into an info document and install it.

Sphinx uses a core configuration file named conf.py. Add the following to that file, replacing all values but index and False by the appropriate names for you project:

# One entry per manual page. 
# list of tuples (startdocname, 
# targetname, title, author, dir_entry, 
# description, category, toctree_only).
texinfo_documents = [
  ('index', # startdocname, keep this!
   'TARGETNAME', # targetname
   u'Long Title', # title
   u'Author Name', # author
   'Name in the Directory Index of Info', # dir_entry
   u'Long Description', # description
   'Software Development', # cathegory
   False), # better keep this, too, i think.
]

Then call sphinx and install the info files like this (maybe adapted to your local setup):

sphinx-build -b texinfo source/ texinfo/ 
cd texinfo
sudo install-info --info-dir=/usr/share/info TARGETNAME.info
sudo cp TARGETNAME.info /usr/share/info/

Activate pydoc-info, including your documentation

Finally add the following to your .emacs (or wherever you store your personal adaptions):

; Show python-documentation as info-pages via C-h S
(setq load-path (cons "~/.emacs.d/libs/pydoc-info" load-path))
(require 'pydoc-info)
(info-lookup-add-help
   :mode 'python-mode
   :parse-rule 'pydoc-info-python-symbol-at-point
   :doc-spec
   '(("(python)Index" pydoc-info-lookup-transform-entry)
     ("(TARGETNAME)Index" pydoc-info-lookup-transform-entry)))

Recipes for presentations with beamer latex using emacs org-mode

I wrote some recipes for creating the kinds of slides I need with emacs org-mode export to beamer latex.

Update: Read ox-beamer to see how to adapt this to work with the new export engine in org-mode 0.8.

The recipes as PDF (21 slides, 247 KiB)

The org-mode sources (12.2 KiB)

Below is an html export of the org-mode file. Naturally it does not look as impressive as the real slides, but it captures all the sources, so I think it has some value.

Note: To be able to use the simple block-creation commands, you need to add #+startup: beamer to the header of your file or explicitely activate org-beamer with M-x org-beamer-mode.

PS: I hereby allow use of these slides under any of the licenses used by worg and/or the emacs wiki.

Hello World

#+LaTeX_CLASS: beamer
#+BEAMER_FRAME_LEVEL: 2

* Hello
** Hello GNU
Nice to see you!

Recipes
#+LaTeX_CLASS: beamer
#+BEAMER_FRAME_LEVEL: 2
* Introduction
** org-mode + beamer =  love
*** Code :BMCOL:
    :PROPERTIES:
    :BEAMER_col: 0.7
    :END:
<example block>
*** Simple block  :BMCOL:B_block:
    :PROPERTIES:
    :BEAMER_col: 0.3
    :BEAMER_env: block
    :END:
it's that easy!

** Two columns - in commands
*** Commands
C-c C-b | 0.7
C-c C-b b
C-n
<eTAB (write example) C-n C-n
*** Result
C-c C-b | 0.3
C-c C-b b
even easier - and faster!

*** Column 1 :B_ignoreheading:BMCOL:
    :PROPERTIES:
    :BEAMER_env: ignoreheading
    :BEAMER_col: 0.5
    :END:

*** One
*** Three                                                           

*** Column 2 :BMCOL:B_ignoreheading:
    :PROPERTIES:
    :BEAMER_col: 0.5
    :BEAMER_env: ignoreheading
    :END:

*** Two
*** Four

*** 
C-c C-b | 0.5 # column
C-c C-b i # ignore heading
*** One 
C-c C-b b # block
*** Three 
C-c C-b b
*** 
C-c C-b | 0.5
C-c C-b i
*** Two 
C-c C-b b
*** Four 
C-c C-b b

*** Code                                                      :B_block:BMCOL:
    :PROPERTIES:
    :BEAMER_env: block
    :BEAMER_col: 0.5
    :BEAMER_envargs: C[t]
    :END:

*** Result                                                    :B_block:BMCOL:
    :PROPERTIES:
    :BEAMER_env: block
    :BEAMER_col: 0.5
    :END:
pretty nice!

#+begin_quote
Emacs org-mode is a 
great presentation tool - 
Fast to beautiful slides.
- Arne Babenhauserheide
#+end_quote

\( 1 + 2 = 3 \) is clear

\[ 1 + 2 \cdot 3 = 7 \]

\( \LaTeX \) gives a space 
after math mode.

\LaTeX{} does it, too.

\LaTeX does not.

At the end of a sentence 
both work.
Try \LaTeX. Or try \LaTeX{}.

Only \( \LaTeX \) and \( \LaTeX{} \) 
also work with HTML export.

: #+bla: foo
: * Example Header

#+begin_example
content
#+end_example

#+bla: foo
* Example Header

content

<Title>

#+startup: beamer
#+LaTeX_CLASS: beamer
#+LaTeX_CLASS_OPTIONS: [bigger]
#+AUTHOR: <empty for none, if missing: inferred>
#+DATE: <empty for none, if missing: today>
#+BEAMER_FRAME_LEVEL: 2
#+TITLE: <causes <Title> to be regular content!>

(org-babel-do-load-languages ; babel, for executing 
 'org-babel-load-languages   ; code in org-mode.
 '((sh . t)
   (emacs-lisp . t)))

(require 'org-latex) ; latex export 
(add-to-list         ; with highlighting
  'org-export-latex-packages-alist '("" "minted"))
(add-to-list 
  'org-export-latex-packages-alist '("" "color"))
(setq org-export-latex-listings 'minted)

(custom-set-variables ; in ~/.emacs, only one instance 
 '(org-export-latex-classes (quote ; in the init file!
    (("beamer" "\\documentclass{beamer}" 
        org-beamer-sectioning))))
 '(org-latex-to-pdf-process (quote 
    ((concat "pdflatex -interaction nonstopmode" 
             "-shell-escape -output-directory %o %f") 
     "bibtex $(basename %b)" 
     (concat "pdflatex -interaction nonstopmode" 
             "-shell-escape -output-directory %o %f")
     (concat "pdflatex -interaction nonstopmode" 
             "-shell-escape -output-directory %o %f")))))

emerge emacs

emerge dev-tex/latex-beamer app-text/texlive

emerge dev-python/pygments

Sending email to many people with Emacs Wanderlust

I recently needed to send an email to many people¹.

Putting all of them into the BCC field did not work (mail rejected by provider) and when I split it into 2 emails, many did not see my mail because it was flagged as potential spam (they were not in the To-Field)².

I did not want to put them all into the To-Field, because that would have spread their email-addresses around, which many would not want³.

So I needed a different solution. Which I found in the extensibility of emacs and wanderlust⁴. It now carries the name wl-draft-send-to-multiple-receivers-from-buffer.

You simply write the email as usual via wl-draft, then put all email addresses you want write to into a buffer and call M-x wl-draft-send-to-multiple-receivers-from-buffer. It asks you about the buffer with email addresses, then shows you all addresses and asks for confirmation.

Then it sends one email after the other, with a randomized wait of 0-10 seconds between messages to avoid flagging as spam.

If you want to use it, just add the following to your .emacs:

(defun wl-draft-clean-mail-address (address)
  (replace-regexp-in-string "," "" address))

(defun wl-draft-send-to-multiple-receivers (addresses)
  (loop for address in addresses do
        (progn 
          (wl-user-agent-insert-header "To" 
            (wl-draft-clean-mail-address address))
          (let ((wl-interactive-send nil))
            (wl-draft-send))
          (sleep-for (random 10)))))

(defun wl-draft-send-to-multiple-receivers-from-buffer 
    (&optional addresses-buffer-name)
  "Send a mail to multiple recipients - one recipient at a time"
  (interactive "BBuffer with one address per line")
  (let ((addresses nil))
    (with-current-buffer addresses-buffer-name
      (setq addresses (split-string (buffer-string) "\n")))
    (if (y-or-n-p (concat "Send this mail to " 
            (mapconcat 'identity addresses ", ")))
        (wl-draft-send-to-multiple-receivers addresses))))

Happy Hacking!

Simple Emacs DarkRoom

I just realized that I let myself be distracted by all kinds of not-so-useful stuff instead of finally getting to type the text I already wanted to transcribe from stenografic at the beginning of … last week.

Screenshot!

Let’s take a break for a screenshot of the final version, because that’s what we really want to gain from this article: a distraction-free screenshot as distraction from the text :)

As you can see, the distractions are removed — the screenshot is completely full screen and only the text is left. If you switch to the minibuffer (i.e. via M-x), the status bar (modeline) is shown.

Background

To remove the distractions I looked again at WriteRoom and DarkRoom and similar which show just the text I want to write. More exactly: I thought about looking at them again, but at second thought I decided to see if I could not just customize emacs to do the same, backed with all the power you get from several decades of being THE editor for many great hackers.

It took some googling and reading emacs wiki, and then some Lisp-hacking, but finally it’s 4 o’clock in the morning and I’m writing this in my own darkroom mode¹, toggled on and off by just hitting F11.

Implementation

I build on hide-mode-line (livejournal post or webonastick) as well as the full-screen info in the emacs wiki.

The whole code just takes 76 lines of code plus 26 lines comments and whitespace:

;;;; Activate distraction free editing with F11

; hide mode line, from http://dse.livejournal.com/66834.html / http://webonastick.com
(autoload 'hide-mode-line "hide-mode-line" nil t)
; word counting
(require 'wc)

(defun count-words-and-characters-buffer ()
  "Display the number of words and characters in the current buffer."
  (interactive)
  (message (concat "The current buffer contains "
           (number-to-string
            (wc-non-interactive (point-min) (point-max)))
           " words and "
           (number-to-string 
            (- (point-max) (point-min)))
           " letters.")))

; fullscreen, taken from http://www.emacswiki.org/emacs/FullScreen#toc26
; should work for X und OSX with emacs 23.x (TODO find minimum version).
; for windows it uses (w32-send-sys-command #xf030) (#xf030 == 61488)
(defvar babcore-fullscreen-p t "Check if fullscreen is on or off")
(setq babcore-stored-frame-width nil)
(setq babcore-stored-frame-height nil)

(defun babcore-non-fullscreen ()
  (interactive)
  (if (fboundp 'w32-send-sys-command)
      ;; WM_SYSCOMMAND restore #xf120
      (w32-send-sys-command 61728)
    (progn (set-frame-parameter nil 'width 
                                (if babcore-stored-frame-width
                                    babcore-stored-frame-width 82))
           (set-frame-parameter nil 'height
                                (if babcore-stored-frame-height 
                                    babcore-stored-frame-height 42))
           (set-frame-parameter nil 'fullscreen nil))))

(defun babcore-fullscreen ()
  (interactive)
  (setq babcore-stored-frame-width (frame-width))
  (setq babcore-stored-frame-height (frame-height))
  (if (fboundp 'w32-send-sys-command)
      ;; WM_SYSCOMMAND maximaze #xf030
      (w32-send-sys-command 61488)
    (set-frame-parameter nil 'fullscreen 'fullboth)))

(defun toggle-fullscreen ()
  (interactive)
  (setq babcore-fullscreen-p (not babcore-fullscreen-p))
  (if babcore-fullscreen-p
      (babcore-non-fullscreen)
    (babcore-fullscreen)))

(global-set-key [f11] 'toggle-fullscreen)

; simple darkroom with fullscreen, fringe, mode-line, menu-bar and scroll-bar hiding.
(defvar darkroom-enabled nil)
; TODO: Find out if menu bar is enabled when entering darkroom. If yes: reenable.
(defvar darkroom-menu-bar-enabled nil)

(defun toggle-darkroom ()
  (interactive)
  (if (not darkroom-enabled)
      (setq darkroom-enabled t)
    (setq darkroom-enabled nil))
  (hide-mode-line)
  (if darkroom-enabled
      (progn
        (toggle-fullscreen)
        ; if the menu bar was enabled, reenable it when disabling darkroom
        (if menu-bar-mode
            (setq darkroom-menu-bar-enabled t)
          (setq darkroom-menu-bar-enabled nil))
        ; save the frame configuration to be able to restore to the exact previous state.
        (if darkroom-menu-bar-enabled
            (menu-bar-mode -1))
        (scroll-bar-mode -1)
        (let ((fringe-width 
               (* (window-width (get-largest-window)) 
                  (/ (- 1 0.61803) (1+ (count-windows)))))
              (char-width-pixels 6))
        ; 8 pixels is the default, 6 is the average char width in pixels
        ; for some fonts:
        ; http://www.gnu.org/software/emacs/manual/html_node/emacs/Fonts.html
           (set-fringe-mode (truncate (* fringe-width char-width-pixels))))
    
        (add-hook 'after-save-hook 'count-words-and-characters-buffer))
    
    (progn 
      (if darkroom-menu-bar-enabled
          (menu-bar-mode))
      (scroll-bar-mode t)
      (set-fringe-mode nil)
      (remove-hook 'after-save-hook 'count-words-and-characters-buffer)
      (toggle-fullscreen))))

; Activate with M-F11 -> enhanced fullscreen :)
(global-set-key [M-f11] 'toggle-darkroom)

(provide 'activate-darkroom)

Also I now activated cua-mode to make it easier to interact with other programs: C-c and C-x now copy/cut when the mark is active. Otherwise they are the usual prefix keys. To force them to be the prefix keys, I can use control-shift-c/-x. I thought this would disturb me, but it does not.

To make it faster, I also told cua-mode to have a maximum delay of 50ms, so I don’t feel the delay. Essentially I just put this in my ~/.emacs:

(cua-mode t)
(setq cua-prefix-override-inhibit-delay 0.005)

Epilog

Well, did this get me to transcribe the text? Not really, since I spent the time building my own DarkRoom/WriteRoom, but I enjoyed the little hacking and it might help me get it done tomorrow - and get far more other stuff done.

And it is really fun to write in DarkRoom mode ;)

PS: If you like the simple darkroom, please leave a comment!

I hereby declare that anyone is allowed to use this post and the screenshot under the same licensing as if it had been written in emacswiki.

Staying sane with Emacs (when facing drudge work)

I have to sift through 6 really boring config files. To stay sane, I call in Emacs for support.

My task looks like this:

(click for full size)

In the lower left window I check the identifier in the table I have to complete (left column), then I search for all instances of that identifier in the right window and insert the instrument type, the SIGMA (uncertainty due to representation error defined for the type of the instrument and the location of the site) and note whether the site is marked as assimilated in the config file.

Then I also check all the other config files and note whether the site is assimilated there.

Drudge work. There are people who can do this kind of work. My wife would likely be able to do it faster without tool support than I can do it with tool support. But I’m really bad at that: When the task gets too boring I tend to get distracted - for example by writing this article.

To get the task done anyway, I create tools which make it more enjoyable. And with Emacs that’s actually quite easy, because Emacs provides most required tools out of the box.

Firstoff: My workflow before adding tools was like this:

• hit xo to switch from the lower left window to the config file at the right.
• Use M-x occur then type the station identifier. This displays all occurances of the station identifier within the config file in the upper left window.
• Hit xo twice to switch to the lower left window again.
• Type the information into the file.
• Switch to the next line and repeat the process.

I now want to simplify this to a single command per line. I’ll use F9 as key, because that isn’t yet used for other things in my Emacs setup and because it is easy to reach and my default keybinding as “useful shortcut for this file”. Other single-keystroke options would be F7 and F8. All other F-keys are already used :)

To make this easy, I define a macro:

• Move to the line above the line I want to edit.
• Start Macro-recording with C-x C-(.
• Go to the beginning of the next line with C-n and C-a.
• Activate the mark with C-SPACE and select the whole identifier with M-f.
• Make the identifier lowercase with M-x downcase-region, copy it with M-w and undo the downcasing with C-x u (or use the undo key; I defined one in my xmodmap).
• Switch to the config file with C-x o
• Search the buffer with M-x occur, inserting the identifier with C-y.
• Hit C-x o C-x o (yes, twice) to get back into the list of sites.
• Move to the end of the instrument column with M-f and kill the word with C-BACKSPACE.
• Save the macro with C-x C-).
• Bind kmacro-call-macro to F9 with M-x local-set-key F9 kmacro-call-macro.

Done.

My workflow is now reduced to this:

• Hit F9
• Type the information.
• Repeat.

I’m pretty sure that this will save me more time today than I spent writing this text ☺

Happy hacking!

Tutorial: Writing scientific papers for ACP using emacs org-mode

Update 2023: I no longer work at the University. Nowadays I would use this setup as starting point, but with more focus on using org for reproducibility (I have German instructions for that) when the first version is ready for submission, export it as LaTeX, and work directly on that, because you’ll need detailed changes in LaTeX.

  (require 'org)
  (require 'org-latex)
  (require 'ox-latex)
  (setq org-latex-packages-alist 
        (quote (("" "color" t) ("" "minted" t) ("" "parskip" t)))
        org-latex-pdf-process 
        (quote (
"pdflatex -interaction nonstopmode -shell-escape -output-directory %o %f" 
"bibtex $(basename %b)" 
"pdflatex -interaction nonstopmode -shell-escape -output-directory %o %f" 
"pdflatex -interaction nonstopmode -shell-escape -output-directory %o %f")))
  (add-to-list 'org-latex-classes
               `("copernicus_discussions"
                 "\\documentclass{copernicus_discussions}
               [NO-DEFAULT-PACKAGES]
               [PACKAGES]
               [EXTRA]"
                 ("\\section{%s}" . "\\section*{%s}")
                 ("\\subsection{%s}" "\\newpage" "\\subsection*{%s}" "\\newpage")
                 ("\\subsubsection{%s}" . "\\subsubsection*{%s}")
                 ("\\paragraph{%s}" . "\\paragraph*{%s}")
                 ("\\subparagraph{%s}" . "\\subparagraph*{%s}"))
               )

(require 'reftex-cite)
(defun org-mode-reftex-setup ()
  (interactive)
  (and (buffer-file-name) (file-exists-p (buffer-file-name))
       (progn
        ; Reftex should use the org file as master file. See C-h v TeX-master for infos.
        (setq TeX-master t)
        (turn-on-reftex)
        ; enable auto-revert-mode to update reftex when bibtex file changes on disk
        (global-auto-revert-mode t) ; careful: this can kill the undo
                                    ; history when you change the file
                                    ; on-disk.
        (reftex-parse-all)
        ; add a custom reftex cite format to insert links
        ; This also changes any call to org-citation!
        (reftex-set-cite-format
         '((?c . "\\citet{%l}") ; natbib inline text
           (?i . "\\citep{%l}") ; natbib with parens
           ))))
  (define-key org-mode-map (kbd "C-c )") 'reftex-citation)
  (define-key org-mode-map (kbd "C-c (") 'org-mode-reftex-search))

(add-hook 'org-mode-hook 'org-mode-reftex-setup)

(add-to-list 'org-latex-packages-alist '("" "minted"))
(add-to-list 'org-latex-packages-alist '("" "color"))
(setq org-latex-listings 'minted)

; add emacs lisp support for minted
(setq org-latex-custom-lang-environments
      '((emacs-lisp "common-lispcode")))

#+title: YOUR TITLE
#+Options: toc:nil ^:nil
#+BIND: org-latex-title-command ""
#+Latex_Class: copernicus_discussions
#+LaTeX_CLASS_OPTIONS: [acpd, hvmath, online]

# Nice code-blocks
#+BEGIN_SRC elisp :noweb no-export :exports results
  (setq org-latex-minted-options
    '(("bgcolor" "mintedbg") ("frame" "single") ("framesep" "6pt") 
      ("mathescape" "true") ("fontsize" "\\footnotesize")))
  nil
#+END_SRC

#+BEGIN_ABSTRACT
Abstract
#+END_ABSTRACT
#+TOC: headlines 2

#+Latex: \runningtitle{SHORT TITLE}
#+Latex: \runningauthor{SHORT AUTHOR}
#+Latex: \correspondence{AUTHOR NAME\\ EMAIL}
#+Latex: \affil{YOUR UNIVERSITY}
#+Latex: \author[2,*]{SECOND AUTHOR}
#+Latex: \author[1]{THIRD AUTHOR SAME INSTITUTE}
#+Latex: \affil[2]{SECOND UNIVERSITY}
#+Latex: \affil[*]{now at: THIRD UNIVERSITY}

#+Latex: \received{}
#+Latex: \pubdiscuss{}
#+Latex: \revised{}
#+Latex: \accepted{}
#+Latex: \published{}
#+Latex: %% These dates will be inserted by ACPD
#+Latex: \firstpage{1}

#+Latex: \maketitle

#+Latex: \introduction
# * Introduction

* Second section

* Discussion

#+Latex: \conclusions
# * Conclusions

#+Latex: \appendix

# use acknowledgements for multiple
#+BEGIN_acknowledgement
Foo Bar Baz.
#+END_acknowledgement

#+Latex: \bibliographystyle{copernicus}
#+Latex: \bibliography{ABSOLUTE_PATH_TO_YOUR_BIBTEX_FILE_WITHOUT_.bib_SUFFIX}{}

# Local Variables:
# org-confirm-babel-evaluate: nil
# org-export-allow-bind-keywords: t
# End:

#+Latex_Class: copernicus_discussions
#+LaTeX_CLASS_OPTIONS: [acpd, hvmath, online]

#+BEGIN_SRC elisp :noweb no-export :exports results
  (setq org-latex-minted-options
    '(("bgcolor" "mintedbg") ("frame" "single") ("framesep" "6pt") 
      ("mathescape" "true") ("fontsize" "\\footnotesize")))
  nil
#+END_SRC

#+Latex: \appendix

# Local Variables:
# org-confirm-babel-evaluate: nil
# org-export-allow-bind-keywords: t
# End:

Unicode char \u8:χ not set up for use with LaTeX: Solution (made easy with Emacs)

For years I regularly stumbled over LaTeX-Errors in the form of Unicode char \u8:χ not set up for use with LaTeX. I always took the chickens path and replaced the unicode characters with the tex-escapes in the file. That was easy, but it made my files needlessly unreadable. Today I decided to FIX the problem once and for all. And it worked. Easily.

Firstoff: The problem I’m facing is that my keyboard layout makes it effortless for me to input characters like ℂ Σ and χ. But LaTeX cannot cope with them out-of-the-box. Org-mode already catches most of these problems, so I can write things like x² instead of x^2, but occasionally it stumbles.

The solution to that is actually pretty simple: I only need to declare the escapes-sequences LaTeX should use when it sees one of the characters (to be used before \begin{document}!):

\DeclareUnicodeCharacter{03C7}{\chi}

Or in org-mode:

#+LaTeX_HEADER: \DeclareUnicodeCharacter{03C7}{\chi}

To do this more easily, you can use the uniinput.ins and uniinput.dtx from the neo-layout project. Run latex uniinput.ins to generate uniinput.sty which you can put next to your latex files and use with \usepackage{uniinput} (instructions in German).

Thanks go to Wikibooks:LaTeX for this. Their solution suggests then to read several Unicode definition documents for tracking down the codepoint of the character. But we can make that easier with Emacs (almost everything is easier with Emacs ☺).

Instead of browsing huge documents manually, we simply rely on the unicode-definitions in Emacs: Move the cursor over the char and execute M-x describe-char.

When used with χ, this shows the following output:

             position: 672 of 35513 (2%), column: 0
            character: χ (displayed as χ) (codepoint 967, #o1707, #x3c7)
    preferred charset: unicode-bmp (Unicode Basic Multilingual Plane (U+0000..U+FFFF))
code point in charset: 0x03C7
… (and a bit more) …

What we need is code point in charset: Just leave out the 0x and you have the codepoint.

For the document I currently write, I now use the following definitions:

#+LaTeX_HEADER: \DeclareUnicodeCharacter{03C7}{\chi}
#+LaTeX_HEADER: \DeclareUnicodeCharacter{B2}{^{2}}

And that makes χ² work.

Happy Hacking - and have fun with Emacs Org-Mode!

Unicode-Characters for TODO-States in Emacs Orgmode

By default Emacs Orgmode uses uppercase words for todo keywords. But having tens of entries marked with TODO and DONE in my file looked horribly cluttered to me. So I searched for alternatives. After a few months of experimentation, I decided on the following scheme. It served me well ever since:

• ❢ To do
• ☯ In progress
  • ⚙ A program is running (optional detail)
  • ✍ I’m writing (optional detail)
• ⧖ Waiting
• ☺ To report
• ✔ Done
• ⌚ Maybe do this at some later time
• ✘ Won’t do

To set this in org-mode, just add the following to the header (and reopen the document, for example with C-x C-v):

#+SEQ_TODO: ❢ ☯ ⧖ | ☺ ✔ ⌚ ✘

or for the complex case (with details on what I do)

#+SEQ_TODO: ❢ ☯ ⚙ ✍ ⧖ | ☺ ✔ ⌚ ✘

Then use C-c C-t or SHIFT-→ (shift + right arrow) to switch to the next state or SHIFT-← (shift + left arrow) to switch to the previous state.

Anything before the | in the SEQ_TODO is shown in red (not yet done), anything after the | is show in green (done). Things which get triggered when something is done (like storing the time of a scheduled entry) happen when the state crosses the |.

And with that, my orgmode documents are not only very useful but also look pretty lean. Just as good as having a GUI with images, but additionally I can access them over SSH and edit the todo state with any tool - because it’s just text.

Use the source, Luke! — Emacs org-mode beamer export with images in figure

I just needed to tweak my Emacs org-mode to beamer-latex export to embed images into a figure environment (not wrapfigure!). After lots of googling and documentation reading I decided to bite the bullet and just read the source. Which proved to be much easier than I had expected.

This tutorial requires at least org-mode 8.0 (before that you had to use hacks to get figure without a caption). It is only tested for org-mode 8.0.2: The code you see when you read the source might look different in other versions.

#+BEGIN_SRC sh :exports results :results output raw
echo '[[./image.png]]'
#+END_SRC

\includegraphics[width=.9\linewidth]{./image.png}

\begin{figure}[htb]
\centering
\includegraphics[width=.9\linewidth]{./image.png}
\end{figure}

C-h f org-latex-export-as-latex
C-x C-o
C-s .el C-b ENTER
C-s figure C-s C-s C-s ...

(float (let ((float (plist-get attr :float)))
     (cond ((string= float "wrap") 'wrap)
       ((string= float "multicolumn") 'multicolumn)
       ((or (string= float "figure")
            (org-element-property :caption parent))
        'figure))))

#+attr_latex: :float figure

#+BEGIN_SRC sh :exports results :results output raw
echo '#+attr_latex: :float figure'
echo '[[./image.png]]'
#+END_SRC

\begin{figure}[htb]
\centering
\includegraphics[width=.9\linewidth]{./image.png}
\end{figure}

C-h f org-latex-export-as-latex

C-x C-o

C-s .el C-b ENTER

C-s figure C-s C-s C-s ...

Using Macros to avoid tedious tasks (screencast)

Because I am lazy,¹ and that makes me fast.

Screencast

(download (ogg theora video))

Using Macros to avoid tedious tasks

Plan

• [X] Show the task
• [X] Record Macro
• [X] Use Macro

Explanation

I record a macro to find ~, then activate the mark and find a space.

C-s ~, C-SPACE, C-s SPACE

Then kill the region and type ${}

C-w ${}

That’s it.

Why??

• It is resilient: I check each change I do.
• I avoid repeating unnerving stuff.

Thank you

☺

recorded with recordmydesktop: recordmydesktop --delay 10 --width 800 --height 600 --on-the-fly-encoding

Wish: KDE with Emacs-style keyboard shortcuts

I would love to be able to use KDE with emacs-style keyboard shortcuts, because Emacs offers a huge set of already clearly defined shortcuts for many different situations. Since its users tend to do very much with the keyboard alone, even more obscure tasks are available via shortcuts.

I think that this would be useful, because Emacs is like a kind of nongraphical desktop environment itself (just look at emacspeak!). For all those who use Emacs in a KDE environment, it could be a nice timesaver to be able to just use their accustomed bindings.

It also has a mostly clean structure for the bindings:

• "C-x anything" does changes which affect things outside the content of the current buffer.
• "C-c anything" is kept for specific actions of programs. For example "C-c C-c" in an email sends the email, while "C-c C-c" in a version tracking commit message finishes the message and starts the actual commit.
• "C-anything but x or c" acts on the content you're currently editing.
• "M-x" opens a 'command-selection-dialog' (just like alt-f2). You can run commands by name.
• "M-anything but x" is a different flavor of "C-anything but x or c". For example "C-f" moves the cursor one character forward, while "M-f" moves one word forward. "C-v" moves one page forward, while "M-v" moves one page backwards.

On the backend side, this would require being able to define multistep shortcuts. Everything else is just porting the emacs shortcuts to KDE actions.

The actual porting of shortcuts would then require mapping of the Emacs commands to KDE actions.

Some examples:

• "C-s" searches in a file. Replaces C-f.
• "C-r" searches backwards.
• "C-x C-s" saves a file -> close. Replaces C-w.
• "C-x C-f" opens a file -> Open. Replaces C-o.
• "C-x C-c" closes the program -> quit. Replaces C-q.
• "C-x C-b" switches between buffers/files/tabs -> switch the open file. Replaces alt-right_arrow and a few other (to my knowledge) inconsistent bindings.
• "C-x C-2" splits a window (or part of a window) vertically. "C-x C-o" switches between the parts. "C-x C-1" undoes the split and keeps the currently selected part. "C-x C-0" undoes the split and hides the currently selected part.

Write multiple images on a single page in org-mode.

• PDF-File
• Org-File

#+latex_header: \usepackage{subfig}

import pylab as pl
import numpy as np
x = np.random.random(size=(2,1000))
pl.scatter(x[0,:], x[1,:], marker=".")
pl.savefig("test.png")
print "\label{fig:image}"
print "[[./test.png]]"

#+BEGIN_LaTeX
\begin{figure}\centering
\subfloat[A gull]{\label{fig:latex-gull} 
\includegraphics[width=0.3\textwidth]{test}
} 
\subfloat[A tiger]{\label{fig:latex-tiger} 
\includegraphics[width=0.3\textwidth]{test}
} 
\subfloat[A mouse]{\label{fig:latex-mouse} 
\includegraphics[width=0.3\textwidth]{test}
}
\caption{Multiple pictures}\label{fig:latex-animals}
\end{figure}
#+END_LaTeX

#+latex: \begin{figure}\centering
#+latex: \subfloat[A gull]{\label{fig:org-gull} 
#+attr_latex: :width 0.3\textwidth
[[./test.png]]
#+latex: }\subfloat[A tiger]{\label{fig:org-tiger} 
#+attr_latex: :width 0.3\textwidth
[[./test.png]]
#+latex: }\subfloat[A mouse]{\label{fig:org-mouse} 
#+attr_latex: :width 0.3\textwidth
[[./test.png]]
#+latex: }\caption{Multiple pictures}\label{fig:org-animals}
#+latex: \end{figure}

emacs wanderlust.el setup for reading kmail maildir

This is my wanderlust.el file to read kmail maildirs. You need to define every folder you want to read.

;; mode:-*-emacs-lisp-*-
;; wanderlust 
(setq 
  elmo-maildir-folder-path "~/.kde/share/apps/kmail/mail"
          ;; where i store my mail

  wl-stay-folder-window t                       ;; show the folder pane (left)
  wl-folder-window-width 25                     ;; toggle on/off with 'i'
  
  wl-smtp-posting-server "smtp.web.de"            ;; put the smtp server here
  wl-local-domain "draketo.de"          ;; put something here...
  wl-message-id-domain "web.de"     ;; ...

file continued:

  wl-from "Arne Babenhauserheide "                  ;; my From:

  ;; note: all below are dirs (Maildirs) under elmo-maildir-folder-path 
  ;; the '.'-prefix is for marking them as maildirs
  wl-fcc ".sent-mail"                       ;; sent msgs go to the "sent"-folder
  wl-fcc-force-as-read t               ;; mark sent messages as read 
  wl-default-folder ".inbox"           ;; my main inbox 
  wl-draft-folder ".drafts"            ;; store drafts in 'postponed'
  wl-trash-folder ".trash"             ;; put trash in 'trash'
  wl-spam-folder ".gruppiert/Spam"              ;; ...spam as well
  wl-queue-folder ".queue"             ;; we don't use this

  ;; check this folder periodically, and update modeline
  wl-biff-check-folder-list '(".todo") ;; check every 180 seconds
                                       ;; (default: wl-biff-check-interval)

  ;; hide many fields from message buffers
  wl-message-ignored-field-list '("^.*:")
  wl-message-visible-field-list
  '("^\\(To\\|Cc\\):"
    "^Subject:"
    "^\\(From\\|Reply-To\\):"
    "^Organization:"
    "^Message-Id:"
    "^\\(Posted\\|Date\\):"
    )
  wl-message-sort-field-list
  '("^From"
    "^Organization:"
    "^X-Attribution:"
     "^Subject"
     "^Date"
     "^To"
     "^Cc"))


; Encryption via GnuPG

(require 'mailcrypt)
 (load-library "mailcrypt") ; provides "mc-setversion"
(mc-setversion "gpg")    ; for PGP 2.6 (default); also "5.0" and "gpg"

(autoload 'mc-install-write-mode "mailcrypt" nil t)
(autoload 'mc-install-read-mode "mailcrypt" nil t)
(add-hook 'mail-mode-hook 'mc-install-write-mode)

(add-hook 'wl-summary-mode-hook 'mc-install-read-mode)
(add-hook 'wl-mail-setup-hook 'mc-install-write-mode)

;(setq mc-pgp-keydir "~/.gnupg")
;(setq mc-pgp-path "gpg")
(setq mc-encrypt-for-me t)
(setq mc-pgp-user-id "FE96C404")

(defun mc-wl-verify-signature ()
  (interactive)
  (save-window-excursion
    (wl-summary-jump-to-current-message)
    (mc-verify)))

(defun mc-wl-decrypt-message ()
  (interactive)
  (save-window-excursion
    (wl-summary-jump-to-current-message)
    (let ((inhibit-read-only t))
      (mc-decrypt))))

(eval-after-load "mailcrypt"
  '(setq mc-modes-alist
       (append
        (quote
         ((wl-draft-mode (encrypt . mc-encrypt-message)
            (sign . mc-sign-message))
          (wl-summary-mode (decrypt . mc-wl-decrypt-message)
            (verify . mc-wl-verify-signature))))
        mc-modes-alist)))


; flowed text

 ;; Reading f=f
 (autoload 'fill-flowed "flow-fill")
 (add-hook 'mime-display-text/plain-hook
          (lambda ()
            (when (string= "flowed"
                           (cdr (assoc "format"
                                       (mime-content-type-parameters
                                        (mime-entity-content-type entity)))))
              (fill-flowed))))
; writing f=f
;(mime-edit-insert-tag "text" "plain" "; format=flowed")


(provide 'private-wanderlust)

UPDATE (2012-05-07): ~/.folders

I now use a ~/.folders file, to manage my non-kmail maildir subscriptions, too. It looks like this:

.sent-mail
.~/.local/share/mail/mgl_spam   "mgl spam" 
.~/.local/share/mail/to.arne_bab    "to arne_bab"
.inbox  "inbox" 
.trash  "Trash"
..gruppiert.directory/.inbox.directory/Freunde  "Freunde"
.drafts "Drafts"
..gruppiert.directory/.alt.directory/Posteingang-2011-09-18 "2011-09-18"
.outbox

The mail in ~/.local/share/mail is fetched via fetchmail and procmail to have a really reliable mail fetching system which does not rely on a non-broken database or free space on the disk to keep working…

keep auto-complete from competing with org-mode structure-templates

For a long time it bothered me that auto-complete made it necessary for me to abort completion before being able to use org-mode templates.

I typed <s and auto-complete showed stuff like <string, forcing me to hit C-g before I could use TAB to complete the template with org-mode.

I fixed this for me by adding all the org-mode structure templates as stop-words:

;; avoid competing with org-mode templates.
(add-hook 'org-mode-hook
          (lambda ()
            (make-local-variable 'ac-stop-words)
            (loop for template in org-structure-template-alist do
                  (add-to-list 'ac-stop-words 
                               (concat "<" (car template))))))

Note, that with this snippet you will have to reopen a file if you add an org-mode template and want it recognized as stop-word in that file.

PS: I added this as bug-report to auto-complete, so with some luck you might not have to bother with this, if you’re willing to simply wait for the next release ☺

wisp: Whitespace to Lisp

New version: draketo.de/software/wisp

display "Hello World!" ↦ (display "Hello World!")

define : factorial n     (define (factorial n)            
    if : zero? n       ↦     (if (zero? n)                
       . 1                      1                      
       * n : factorial {n - 1}  (* n (factorial {n - 1}))))

Table of Contents

display "Hello World!"      ↦      (display "Hello World!")

display "Hello World!"      ↦      (display "Hello World!")
display "Hello Again!"      ↦      (display "Hello Again!")

display                              ↦    (display
  string-append "Hello " "World!"    ↦      (string-append "Hello " "World!"))

string-append "Hello"        ↦    (string-append "Hello"
  string-append " " "World"  ↦      (string-append " " "World")
  . "!"                      ↦      "!")

= a        ↦    (= a
  . a      ↦      (. a))

. a b c    ↦    a b c

let                       ↦    (let
  :                       ↦      ((msg "Hello World!"))
    msg "Hello World!"    ↦      (display msg))
  display msg             ↦      

define : hello who                            ↦    (define (hello who)
  display : string-append "Hello " who "!"    ↦      (display (string-append "Hello " who "!")))

let                       ↦    (let
  : msg "Hello World!"    ↦      ((msg "Hello World!"))
  display msg             ↦      (display msg))

let : : msg "Hello World!"     ↦    (let ((msg "Hello World!"))
  display msg                  ↦      (display msg))

define : hello who                    ↦    (define (hello who)
_ display                             ↦      (display 
___ string-append "Hello " who "!"    ↦        (string-append "Hello " who "!")))

PS: Wisp is linked in the comparisions of SRFI-110.

say Yes, I do!

Yes, I do!

wget https://bitbucket.org/ArneBab/wisp/downloads/wisp-0.8.6.tar.gz
tar xf wi*z; cd wi*/; ./c*e; make check; guile -L . --language=wisp

display "First Witch
    When shall we three meet again
    In thunder, lightning, or in rain?\n"

define-syntax-rule : First_Witch a ...
  format #t "~A\n" 
    string-join 
      map : lambda (x) (string-join (map symbol->string x))
            quote : a ...
      . "\n"

First_Witch
    When shall we three meet again
    In thunder, lightning, or in rain?

SCENE I. A desert place.

    Thunder and lightning.

    Enter : First Witch
            Second Witch
            Third Witch

First Witch
    When shall ,(emphasized we three) meet again
    In thunder, lightning, or in rain?

Second Witch
    When the hurlyburly's done,
    When the battle's lost and won.

; ...

ALL
    Fair is foul, and foul is fair:
    Hover through the fog and filthy air.

action
    Exeunt

Going from Python to Guile Scheme - a natural progression

Python is the first language I loved. I dreamt in Python, I planned in Python, I thought I would never need anything else.

 - Free: html | pdf
 - Softcover: 14.95 €
   with pdf, epub, mobi
 - Source: download
   free licensed under GPL

I will show you why I love Python

Python is a language where I can teach a handful of APIs and cause people to learn most of the language as a whole. — Raymond Hettinger (2011-06-20)

• Pseudocode which runs
• One way to do it
• Hackable
• Batteries and Bindings
• Scales up

Where I hit its limits

Why, I feel all thin, sort of stretched if you know what I mean: like butter that has been scraped over too much bread. — Bilbo Baggins in “The Lord of the Rings”

• Dual Syntax: What we teach new users is no longer what we use
• Ceremony crops in
• Complexity is on the rise

And how I lost its shackles

You must unlearn what you have learned. — Yoda in “The Empire Strikes Back“

• Guile Scheme compared to the strengths of Python
• Guile Scheme shortcomings
• Guile beyond Python

Guile Scheme is the official GNU extension language, used for example in GNU Cash and GNU Guix and the awesome Lilypond.

Accompany me on a path beyond Python

Every sufficiently complex appli­ca­tion/langu­age/tool will either have to use Lisp or reinvent it the hard way. — Greenspuns 10th rule

• Buy the softcover: 14.95 € (includes the ebook)
• Read for free: html | pdf
• Download the source: from Sourcehut

As free cultural work, py2guile is licensed under the GPLv3 or later. You are free to share, change, remix and even to resell it as long as you say that it’s from me (attribution) and provide the whole corresponding source under the GPL (sharealike).

For instructions on building the ebook yourself, see the README in the source.

Happy Hacking!

— Arne Babenhauserheide

Gratis py2guile from Freenet

py2guile is a book I wrote about Python and Guile Scheme. It’s selling at 14.95 €](https://www.epubli.de/shop/buch/47692) for the printed softcover.

To fight the new german data retention laws, you can get the ebook gratis: Just install Freenet, then the following links work:

• py2guile.epub
• py2guile.mobi
• py2guile.pdf
• py2guile-0.9.0-fn.tar.gz

Escape total surveillance and get an ebook about the official GNU extension language for free today!

Python chooses Github, therefore I’m releasing the py2guile PDF for free

Python is the first language I loved. I dreamt in Python, I planned in Python, I thought I would never need anything else.

  Download “Python to Guile” (pdf)

You can read more about this on the Mercurial mailing list.

 - Free: html | pdf
   preview edition
   (complete)

Yes, this means that with Guile I will contribute to a language developed via Git, but it won’t be using a proprietary platform.

If you like py2guile, please consider buying the book:

 - Softcover: 14.95 €
   with digital companion
 - Source: download
   free licensed under GPL

More information: draketo.de/py2guile

Commentary on Python and Github

Subjective popularity contest without robust data

I was curious why this happened so I read through PEP 0481. It's interesting that Git was chosen to replace Mercurial due to Git's greater popularity, yet a technical comparison was deemed as subjective. In fact, no actual comparison (of any kind) was discussed. What a shame. — Emmanuel Rosa on G+

yes. And the popularity contest wasn’t done in any robust way — they present values between 3x as popular and 18x as popular. That is a crazy margin of error — especially for a value on which to base a very disrupting decision. — my answer

No more dogfooding

Yesterday Python maintainers chose to move to GitHub and Git. Python is now developed using a C-based tool on a Ruby-based, unfree platform. And that changed my view on what’s happening in the community. Python no longer fosters its children and it even stopped dogfooding where its tools are as good as or better than other tools. I don’t think it will die. But I don’t bet on it for the future anymore. — EDIT to my answer on Quora “is Python a dying language?” which originally stated “it’s not dying, it’s maturing”.

Github invades your workflows

The PEP for github hedges somewhat by using github for code but not bug tracker. Not ideal considering BitKeeper, but a full on coup for GitHub. — Martin Owens

that’s something like basic self-defense, but my experience with projects who moved to GitHub is that GitHub soon starts to invade your workflows, changing your cooperation habits. At some point people realize that they can’t work well without GitHub anymore.

Not becoming dependent on GitHub while using it requires constant vigilance. Seeing how Python already switched to Git and GitHub because existing infrastructure wasn’t maintained does not sound like they will be able or willing to put in the work to keep independent. — my answer on G+

Foreboding since 2014

I was already pretty disappointed when I heard that Python is moving to Git. Seeing it choose the proprietary platform is an even sadder choice from my perspective. Two indicators for a breakage in the culture of the project.

For me that’s a reason to leave Python. Though it’s not like I did not get a foreboding of that. It’s why I started learning Guile Scheme in 2013 — and wrote about the experience.

I will still use Python for many practical tasks — it acquired the momentum for that, especially in science (I was a champion for Python in the institute, which is now replacing Matlab and IDL for many people here, and I will be teaching Python starting two weeks from now). I think it will stay strong for many years; a good language to start and a robust base for existing programs. But with what I learned the past years, Python is no longer where I place my bets. — slightly adjusted version of my post on the Mercurial mailing list.

Popularity without robust data instead of quality

(this is taken from a message I wrote to Brett, so I don’t have to say later that I stayed silent while Python went down. I got a nice answer, and despite the disagreement we said a friendly good bye)

Back when I saw that Python might move to git, I silently resigned and stopped caring to some degree. I have seen a few projects move to Git in the past years (and in every project problems remained even years after the switch), so when it came to cPython, the quarrel with git-fans just didn’t feel worthwhile anymore.

Seeing Python choose GitHub with the notion of “git is 3x to 18x more popular than Mercurial and free solutions aren’t better than GitHub” makes me lose my trust in the core development community, though.

PEP 481 states, that it is about the quality of the tooling, but it names the popularity numbers quite prominently: python.org/dev/peps/pep-0481/

If they are not relevant, they shouldn’t be included, but they are included, so they seem to be relevant to the decision. And “the best tooling” is mostly subjective, too — which is shown in the PEP itself which mostly talks about popularity, not quality. It even goes into length about how to avoid many of the features of GitHub.

I’ve seen quite a few projects try to avoid lock-in to GitHub. None succeeded. Not even in one where two of about six active developers were deeply annoyed by GitHub. This is exactly what the scipy part of the PEP describes: lock-in due to group effects.

Finally, using hg-git is by far not seamless. I use it for several projects, and when the repositories become big (as cPython’s is), the overhead of the conversion becomes a major hassle. It works, but native Mercurial would be much more efficient. When pushing takes minutes, you start to think twice about whether you’ll just do the quick fix right now. Not to forget that at some point people start to demand signing of commits in git-style (not possible with hg-git, you can only sign commits mercurial-style) as well as other gitologisms (which have an analogue in Mercurial but aren’t converted by hg-git).

Despite my disappointment, I wish you all the best. Python is a really cool language. It’s the first one I loved and will always stay dear to me, so I’m happy that you work on it — and I hope you keep it up.

So, I think this is goodbye. A bit melancholic, but that’s how that goes.

Good luck to you in your endeavors,
Arne Babenhauserheide

Enough negativity

And that’s enough negativity from me.

Thank you, Brett, for reminding me that even though we might disagree, it’s important to remember that people in the project are hit by negativity much harder than it feels for the one who writes.

For my readers: If that also happened to you one time or the other, please read his article:

How I stay happy making open source software

Thank you, Brett. Despite everything I wrote here, I still think that Python is a great project, and it got many things right — some of which are things which are at least as important as code but much less visible, like having a large, friendly community.

I’m happy that Python exists, and I hope that it keeps going. And where I use programming to make a living, I’m glad when I can do it in Python. Despite all my criticism, I consider Python as the best choice for many tasks, and this is also written in py2guile: almost the the first half of the book talks about the strengths of Python. Essentially I could not criticize Python as strongly as I’m doing it here if I did not like it so much. Keep that in mind when you think about what you read.

Also Brett now published an article where he details his decision to move to GitHub. It is a good read: The history behind the decision to move Python to GitHub — Or, why it took over a year for me to make a decision

For me, Gentoo is about *convenient* choice

It's often said, that Gentoo is all about choice, but that doesn't quite fit what it is for me.

After all, the highest ability to choose is Linux from scratch and I can have any amount of choice in every distribution by just going deep enough (and investing enough time).

What really distinguishes Gentoo for me is that it makes it convenient to choose.

Since we all have a limited time budget, many of us only have real freedom to choose, because we use Gentoo which makes it possible to choose with the distribution-tools. Therefore only calling it “choice” doesn't ring true in general - it misses the reason, why we can choose.

So what Gentoo gives me is not just choice, but convenient choice.

Some examples to illustrate the point:

KDE 4 without qt3

I recently rebuilt my system after deciding to switch my disk layout (away from reiserfs towards a simple ext3 with reiser4 for the portage tree). When doing so I decided to try to use a "pure" KDE 4 - that means, a KDE 4 without any remains from KDE3 or qt3.

To use kde without any qt3 applications, I just had to put "-qt3" and "-qt3support" into my useflags in /etc/make.conf and "emerge -uDN world" (and solve any arising conflicts).

Imagine doing the same with a (K)Ubuntu...

Emacs support

Similarly to enable emacs support on my GentooXO (for all programs which can have emacs support), I just had to add the "emacs" useflag and "emerge -uDN world".

Selecting which licenses to use

Just add

ACCEPT_LICENSE="-* @FSF-APPROVED @FSF-APPROVED-OTHER"

to your /etc/make.conf to make sure you only get software under licenses which are approved by the FSF.

For only free licenses (regardless of the approved state) you can use:

ACCEPT_LICENSE="-* @FREE"

All others get marked as masked by license. Default (no ACCEPT_LICENSE in /etc/make.conf) is “* -@EULA”: No unfree software. You can check your setting via emerge --info | grep ACCEPT_LICENSE. More information…

One program (suite) in testing, but the main system rock stable

Another part where choosing is made convenient in Gentoo are testing and unstable programs.

I remember my pain with a Kubuntu, where I wanted to use the most recent version of Amarok. I either had to add a dedicated Amarok-only testing repository (which I'd need for every single testing program), or I had to switch my whole system into testing. I did the latter and my graphical package manager ceased to work. Just imagine how quickly I ran back to Gentoo.

And then have a look at the ease of deciding to take one package into testing in Gentoo:

• emerge --autounmask-write =cathegory/package-version
• etc-update
• emerge =cathegory/package-version

EDIT: Once I had a note here “It would be nice to be able to just add the missing dependencies with one call”. This is now possible with --autounmask-write.

And for some special parts (like KDE 4) I can easily say something like

• ln -s /usr/portage/local/layman/kde-testing/Documentation/package.keywords/kde-4.3.keywords /etc/portage/package.keywords/kde-4.3.keywords

(I don't have the kde-testing overlay on my GentooXO, where I write this post, so the exact command might vary slightly)

Closing remarks

So to finish this post: For me, Gentoo is not only about choice. It is about convenient choice.

And that means: Gentoo gives everybody the power to choose.

I hope you enjoy it as I do!

• more about Gentoo

pmerge @glsa

\#! /bin/sh

\### Update the portage tree and the glsa packages via pkgcore

\# spew a status message
echo $(date) "start to update GLSA" >> /tmp/cron-update.log

\# Sync only portage
pmaint sync /usr/portage

\# security relevant programs
pmerge -uDN @glsa > /tmp/cron-update-pkgcore-last.log || cat \
    /tmp/cron-update-pkgcore-last.log >> /tmp/cron-update.log  

\# And keep everything working
revdep-rebuild

\# Finally update all configs which can be updated automatically
cfg-update -au

echo $(date) "finished updating GLSA" >> /tmp/cron-update.log

\#!/bin/sh                                                     

\### Update my computer using pgkcore, 
\### since that also works if some dependencies couldn't be resolved.

\# Sync all overlays
eix-sync

\## First use pkgcore
\# security relevant programs (with build-time dependencies (-B))
pmerge -BuD @glsa

\# system, world and all the rest
pmerge -BuD @system
pmerge -BuD @world
pmerge -BuD @installed

\# Then use portage for packages pkgcore misses (inlcuding overlays) 
\# and for *EMERGE_DEFAULT_OPTS="--keep-going"* in make.conf 
emerge -uD @security
emerge -uD @system
emerge -uD @world
emerge -uD @installed

\# And keep everything working
emerge @preserved-rebuild
revdep-rebuild

\# Finally update all configs which can be updated automatically
cfg-update -au

alias pix='pquery --raw -nv --attr=keywords'

$ time pmerge -p @glsa
 * Resolving...
Nothing to merge.

real    0m1.863s
user    0m1.463s
sys     0m0.100s

No, it ain’t “forever” (GNU Hurd code_swarm from 1991 to 2010)

If the video doesn’t show, you can also download it as Ogg Theora & Vorbis “.ogv” or find it on youtube.

This video shows the activity of the Hurd coders and answers some common questions about the Hurd, including “How stagnated is Hurd compared to Duke Nukem Forever?”. It is created directly from commits to Hurd repositories, processed by community codeswarm.

Every shimmering dot is a change to a file. These dots align around the coder who did the change. The questions and answers are quotes from todays IRC discussions (2010-07-13) in #hurd at irc.freenode.net.

You can clearly see the influx of developers in 2003/2004 and then again a strenthening of the development in 2008 with less participants but higher activity than 2003 (though a part of that change likely comes from the switch to git with generally more but smaller commits).

I hope you enjoyed the high-level look on the activity of the Hurd project!

PS: The last part is only the information title with music to honor Sean Wright for allowing everyone to use and adapt his Album Enchanted.

Some technical advantages of the Hurd

→ An answer to just accept it, truth hurds, where Flameeyes told his reasons for not liking the Hurd and asked for technical advantages (and claimed, that the Hurd does not offer a concept which got incorporated into other free software, contributing to other projects). Note: These are the points I see. Very likely there are more technical advantages which I don’t see well enough to explain them.

The translator system in the Hurd is a simple concept that makes many tasks easy, which are complex with Linux (like init, network transparency, new filesystems, …). Additionally there are capabilities (give programs only the access they need - adjusted at runtime), subhurds and (academic) memory management.

Information for potential testers: The Hurd is already usable, but it is not yet in production state. It progressed a lot during the recent years, though. Have a look at the status report if you want to see if it’s already interesting for you. See running the Hurd for testing it yourself.

Table of Contents:

• Influence on other systems: FUSE and BSD
• Advantages for users
  • Translator-based filesystem
  • Network transparency on the filesystem level
  • Unionmount as user
  • Persistent services, activated when needed
  • Add permissions at runtime
• Advantages for administrators and developers
  • Lightweight virtualization
  • More accessible lowlevel hacking
  • Subdividing memory management
• Summary

Influence on other systems: FUSE in Linux and limited translators in NetBSD

Firstoff: FUSE is essentially an implementation of parts of the translator system (which is the main building block of the Hurd) to Linux, and NetBSD recently got a port of the translators system of the Hurd. That’s the main contribution to other projects that I see.

As an update in 2015: A pretty interesting development in the past few years is that the systemd developers have been bolting features onto Linux which the Hurd already provided 15 years ago. Examples: socket-activation provides on-demand startup like passive translators, but as crude hack piggybacked on dbus which can only be used by dbus-aware programs while passive translators can be used by any program which can access the filesystem, calling priviledged programs via systemd provides jailed priviledge escalation like adding capabilities at runtime, but as crude hack piggybacked on dbus and specialized services.

That means, there is a need for the features of the Hurd, but instead of just using the Hurd, where they are cleanly integrated, these features are bolted onto a system where they do not fit and suffer from bad performance due to requiring lots of unnecessary cruft to circumvent limitations of the base system. The clean solution would be to just set 2-3 full-time developers onto the task of resolving the last few blockers (mainly sound and USB) and then just using the Hurd.

translator-based filesystem

On the bare technical side, the translator-based filesystem stands out: The filesystem allows for making arbitrary programs responsible for displaying a given node (which can also be a directory tree) and to start these programs on demand. To make them persistent over reboots, you only need to add them to the filesystem node (for which you need the right to change that node). Also you can start translators on any node without having to change the node itself, but then they are not persistent and only affect your view of the filesystem without affecting other users. These translators are called active, and you don’t need write permissions on a node to add them.

network transparency on the filesystem level

The filesystem implements stuff like Gnome VFS (gvfs) and KDE network transparency on the filesystem level, so those are available for all programs. And you can add a new filesystem as simple user, just as if you’d write into a file “instead of this node, show the filesystem you get by interpreting file X with filesystem Y” (this is what you actually do when setting a translator but not yet starting it (passive translator)).

One practical advantage of this is that the following works:

settrans -a ftp\: /hurd/hostmux /hurd/ftpfs /
dpkg -i ftp://ftp.gnu.org/path/to/*.deb

This installs all deb-packages in the folder path/to on the FTP server. The shell sees normal directories (beginning with the directory “ftp:”), so shell expressions just work.

You could even define a Gentoo mirror translator (settrans mirror\: /hurd/gentoo-mirror), so every program could just access mirror://gentoo/portage-2.2.0_alpha31.tar.bz2 and get the data from a mirror automatically: wget mirror://gentoo/portage-2.2.0_alpha31.tar.bz2

unionmount as user

Or you could add a unionmount translator to root which makes writes happen at another place. Every user is able to make a readonly system readwrite by just specifying where the writes should go. But the writes only affect his view of the filesystem.

persistent translators, started when needed

Starting a network process is done by a translator, too: The first time something accesses the network card, the network translator starts up and actually provides the device. This replaces most initscripts in the Hurd: Just add a translator to a node, and the service will persist over restarts.

It’s a surprisingly simple concept, which reduces the complexity of many basic tasks needed for desktop systems.

And at its most basic level, Hurd is a set of protocols for messages which allow using the filesystem to coordinate and connect processes (along with helper libraries to make that easy).

add permissions at runtime (capabilities)

Also it adds POSIX compatibility to Mach while still providing access to the capabilities-based access rights underneath, if you need them: You can give a process permissions at runtime and take them away at will. For example you can start all programs without permission to use the network (or write to any file) and add the permissions when you need them.

Different from Linux, you do not need to start privileged and drop permissions you do not need (goverened by the program which is run), but you start as unprivileged process and add the permissions you need (governed by an external process):

groups # → root
addauth -p $(ps -L) -g mail
groups # → root mail 

lightweight virtualization

And then there are subhurds (essentially lightweight virtualization which allows cutting off processes from other processes without the overhead of creating a virtual machine for each process). But that’s an entire post of its own…

Easy to test lowlevel hacking

And the fact that a translator is just a simple standalone program means that these can be shared and tested much more easily, opening up completely new options for lowlevel hacking, because it massively lowers the barrier of entry.

For example the current Hurd can use the Linux network device drivers and run them in userspace (via DDE), so you can simply restart them and a crashing driver won’t bring down your system.

subdividing memory management

And then there is the possibility of subdividing memory management and using different microkernels (by porting the Hurd layer, as partly done in the NetBSD port), but that is purely academic right now (search for Viengoos to see what its about).

Summary

So in short:

The translator system in the Hurd is a simple concept that makes many tasks easy, which are complex with Linux (like init, network transparency, new filesystems, …). Additionally there are capabilities (give programs only the access they need - adjusted at runtime), subhurds and (academic) memory management.

Best wishes,
Arne

PS: I decided to read flameeyes’ post as “please give me technical reasons to dispell my emotional impression”.

PPS: If you liked this post, it would be cool if you’d flattr it:

PPPS: Additional information can be found in Gaël Le Mignot’s talk notes, in niches for the Hurd and the GNU Hurd documentation pages.

P4S: This post is also available in the Hurd Staging Wiki.

(A)GPL as hack on a Python-powered copyright system

AGPL is a hack on copyright, so it has to use copyright, else it would not compile/run.

All the GPL licenses are a hack on copyright. They insert a piece of legal code into copyright law to force it to turn around on itself.

You run that on the copyright system, and it gives you code which can’t be made unfree.

To be able to do that, it has to be written in copyright language (else it could not be interpreted).

my_code = "<your code>"

def AGPL ( code ): 
    """
    >>> is_free ( AGPL ( code ) )
    True
    """
    return eval (
        transform_to_free ( code ) )

copyright ( AGPL ( my_code ) )

You pass “AGPL ( code )” to the copyright system, and it ensures the freedom of the code.

The transformation means that I am allowed to change your code, as long as I keep the transformation, because copyright law sees only the version transformed by AGPL, and that stays valid.

Naturally both AGPL definition and the code transformed to free © must be ©-compatible. And that means: All rights reserved. Else I could go in and say: I just redefine AGPL and make your code unfree without ever touching the code itself (which is initially owned by you by the laws of ©):

def AGPL ( code ): 
    """ 
    >>> is_free ( AGPL ( code ) )
    False
    """
    return eval (
        transform_to_mine ( code ) )

In this Python-powered copyright-system, I could just define this after your definition but before your call to copyright(), and all calls to APGL ( code ) would suddenly return code owned by me.

Or you would have to include another way of defining which exact AGPL you mean. Something like “AGPL, but only the versions with the sha1 hashes AAAA BBBB and AABA”. cc tries to use links for that, but what do you do if someone changes the DNS resolution to point creativecommons.org to allmine.com? Whose DNS server is right, then - legally speaking?

In short: AGPL is a hack on copyright, so it has to use copyright, else it would not compile/run.

Are there 10x programmers?

→ An answer I wrote to this question on Quora.

Software Engineering: What is the truth of 10x programmers?
Do they really exist?…

Let’s answer the other way round: I once had to take heavy anti-histamines for three weeks. My mind was horribly hazy from that, and I felt awake only about two hours per day. However I spent every day working on a matrix multiplication problem.

It was three weeks of failure, because I just could not grasp the problem. I was unable to hold it in my mind.

Then I could finally drop the anti-histamine.

On the first day I solved the problem on my way to buy groceries. On the second day I planned the implementation while walking for two hours . On the third day I finished the program.

This taught me to accept it when people don’t manage to understand things I understand: I know that the brain can actually have different paces and that complexity which feels easy to me might feel infeasible for others. It sure did feel that way to me while I took the anti-histamines.

It also taught me to be humble: There might be people to whom my current state of mind feels like taking anti-histamines felt to me. I won’t be able to even grasp the patterns they see, because they can use another level of complexity.

To get a grasp of the impact, I ask myself a question: How would an alien solve problems who can easily keep 100 things in its mind — instead of the 4 to 7 which is the rough limit for humans?

BY-SA and GPL: creativecommons closed the chasm in the sharealike/copyleft community

This is the biggest news item for free culture and free software in the past 5 years: The creativecommons attribution sharealike license is now one-way compatible to the GPL — see the message from creativecommons and from the Free Software Foundation.

• Blender rigs under cc by-sa which use libraries under GPL are safe against suing, even if a Blender code contributor goes crazy,
• 3D models like the self replicating 3D printer RepRap and libre hardware designs under cc by-sa and GPL can be combined,
• GPL-compatible Free Software can utilize all content from the Wikipedia and
• On the personal side: we can merge art from Ryzom and Battle for Wesnoth and use both in the Pirate Party RPG.

Some license compatibility legalese might sound small, but the impact of this is hard to overestimate.

(I’ll now revise some of my texts about licensing — CC BY-SA got a major boost in utility because it not longer excludes usage in copyleft documents which need the source to have a defended sharealike clause)

Communicating your project: honest marketing for free software projects

You have an awesome project, but you see people reach for inferior tools? There are people using your project, but you can’t reach the ones you care about? Read on for a way to ensure that your communication doesn’t ruin your prospects but instead helps your project to shine.

Communicating your project is an essential step for getting the users you want. Here I summarize my experience from working on several different projects including KDE (where I learned the basics of PR - yay, sebas!), the Hurd (where I could really make a difference by improving the frontpage and writing the Month of the Hurd), Mercurial (where I practiced minimally invasive PR) and 1d6 (my own free RPG where I see how much harder it is to do PR, if the project to communicate is your own).

Since voicing the claim that marketing is important often leads to discussions with people who hate marketing of any kind, I added an appendix with an example which illustrates nicely what happens when you don’t do any PR - and what happens if you do PR of the wrong kind.

If you’re pressed for time and want the really short form, just jump to the questionnaire.

• What is good marketing?
• How to comunicate your project?
  • Who are our Target Groups?
  • What could they ask?
  • Whose wishes can we fullfil?
  • Provide those answers!
  • Further points
• bab-com Q: Short questionaire
• Note: The mission statement
• Summary
• Appendix: Why communicating your project?

What is good marketing?

Before we jump directly to the guide, there is an important term to define: Good marketing. That is the kind of marketing, we want to do.

The definition I use here is this:

Good marketing ensures that the people to whom a project would be useful learn about the project.

and

Good marketing starts with the existing strengths of a project and finds people to whom these strengths are useful.

Thus good marketing does not try to interfere with the greater plan of the project, though it might identify some points where a little effort can make the project much more interesting to users. Instead it finds users to whom the project as it is can be useful - and ensures that these know about the project.

Be fair to competitors, be honest to users, put the project goals before generic marketing considerations.

As such, good marketing is an interface between the project and its (potential) users.

How to communicate your project?

This guide depends on one condition: Your project already has at least one area in which it excels over other projects. If that isn’t the case, please start by making your project useful to at least some people.

The basic way for communicating your project to its potential users always follows the same steps.

To make this text easier to follow, I’ll intersperse it with examples from the latest project where I did this analysis: GNU Guile: The GNU Ubiquitous Intelligent Language for Extensions. Guile provides a nice example, because its mission is clearly established in its name and it has lots of backing, but up until our discussion actually had a wikipedia-page which was unappealing to the point of being hostile against Guile itself.

To improve the communication of our project, we first identify our target groups.

Who are our Target Groups?

To do so, we begin by asking ourselves, who would profit from our project:

• What can we do well and how do we compare to others?
• To whom would we already be useful or interesting if people knew about our strengths?
• To whom are we already the best option?

Try to find about 3 groups of people and give them names which identify them. Those are the people we must reach to grow on the short term.

In the next step, we ask ourselves, whom we want or need as users to fullfill our mission (our long-term goal):

• Where do we want to get? What is our goal? (do we have a mission statement?)
• Whom do we need to get there?
• Whom do we want as users? Those shape us as they take part in the development - either as users or as fellow developers.

Again try to find about 3 groups of people and give them names which identify them. Those are the people we must reach to achieve our longterm goal. If while writing this down you find that one of the already identified groups which we could reach would actually detract us from our goal, mark them. If they aren’t direly needed, we would do best to avoid targeting them in our communication, because they will hinder us in our longterm progress: They could become a liability which we cannot get rid of again.

Now we have about 6 target groups: Those are the people who should know about our project, either because they would benefit from it for pursuing their goals, or because we need to reach them to achieve our own goals. We now need to find out, which kind of information they actually need or search.

Example: Target Groups for Guile

GNU Guile is called The GNU Ubiquitous Intelligent Language for Extensions. So its mission is clear: Guile wants to become the de-facto standard language for extending programs - at least within the GNU project.

For whom are we already useful or interesting? Name them as Target-Groups.

1. Schemer: Wants to see what GNU Scheme can do.
2. Extender: GNU enthusiast wants to extend an existing program with a scripting language.
3. Learner: Free Software enthusiast thinks about using Guile to learn programming
4. Project-Starter: Experienced Programmer wants to start a new project.
5. 1337: Programmer wants the coolness-factor.
6. Emacser: Emacs users want to see what the potential future of Emacs would hold.

Whom do we want as users on the long run? Name them as Target-Groups.

1. GNU-folk: All GNU developers.

What could they ask?

This part just requires thinking ourselves into the role of each of the target groups. For each of the target groups, ask yourself:

What would you want to know, if you were to read about our project?

As result of this step, we have a set of answers. Judge them on their strengths: Would these answers make you want to invest time to test our project? If not, can we find a better answer?

Example: Questions for the Target-Groups of Guile

1. Schemer: What can guile do better than other Schemes?
2. Extender: What does Guile offer my program? Why Guile and not Python/Lua?
3. Learner: How easy and how powerful is Guile Scheme? Why Guile and not Python?
4. Starter: What’s the advantage of starting my advanced project with guile?
5. 1337: Why is guile cool?
6. Emacser: What does Guile offer for Emacs?
7. GNU-folk: What does Guile offer my program? (Being a GNU package is a distinct advantage, so there is less competition by non-GNU languages)

Whose wishes can we fullfill?

If our answers for a given group are not yet strong enough, we cannot yet communicate our project convincingly to them. In that case it is best to postpone reaching out to that group, otherwise they could get a lasting weak image of our project which would make it harder to reach them when we have stronger answers at some point in the future.

Remove all groups whose wishes we cannot yet fullfill, or for whom we do not see ourselves as the best choice.

Example: Chosen Target-Groups

1. Schemer: Guile is a solid implementation of Scheme. For a comparison, see An opinionated Guide to Scheme implementations.
2. Extender: The guile manual offers a nicely detailed guide for extending a program with Guile. We’re a bit weak on the examples and existing extensions, though, especially on non-GNU-plattforms.
3. Learner: There aren’t yet tutorials for learning to program in Guile, though there are tutorials for learning to write scheme - and even one for understanding Scheme from the view of a Python-user. But our project resources cannot yet support people who cannot program at all well enough, so we have to restrict ourselves to programmers who want to learn a new language.
4. Starter: Guile has solid support for many unix-specific things, but it is not yet a complete project-publishing solution. So we have to restrict ourselves to targeting people who want to start a project which is mainly intended to be used in environments with proper package management (mostly GNU/Linux).
5. 1337: Guile is explicitely named in the GNU Coding Standards. It doesn’t get much cooler than that - at least for a certain idea of cool. We can’t get the Java-1337s, but we can get the Free Software-1337s.
6. Emacser: Guile provides foreign-function-call. If guile gets used as base for Emacs, Emacs users get direct access to all scheme functions, too - as well as real threading. And that’s pretty strong. Also Geiser provides solid Guile Scheme support in Emacs.
7. GNU-folk: They are either extenders or project starters or learners, but additionally they want to know in which GNU projects they can use Guile.

Provide those answers!

Now we have answers for the target groups. When we now talk or write about our project, we should keep those target groups in mind.

You can make that arbitrarily complex, for example by trying to find out which of our target groups use which medium. But lets keep it simple:

Ensure that our website (and potentially existing wikipedia page) includes the information which matters to our target groups. Just take all the answers for all the target groups we can already reach and check whether the basic information contained in them is given on the front page of our website.

And if not, find ways to add it.

As next steps, we can make sure that the questions we found for the target groups not only get answered, but directly lead the target groups to actions: For example to start using our project.

Example: The new Wikipedia-Page of Guile

For Guile, we used this analysis to fix the Wikipedia-Page. The old-version mainly talked about history and weaknesses (to the point of sounding hostile towards Guile), and aside from the latest release number, it was horribly outdated. And it did not provide the information our target groups required.

The current Wikipedia-Page of GNU Guile works much better - for the project as well as for the readers of the page. Just compare them directly and you’ll see quite a difference. But aside from sounding nicer, the new site also addresses the questions of our target groups. To check that, we now ask: Did we include information for all the potential user-groups?

1. Schemers: Yepp (it’s scheme and there’s a section on Guile Scheme
2. Extenders: Yepp (libguile)
3. Learners: Not yet. We might need a syntax-section with some examples. But wikipedians do not like Howto-Like sections. Also the interpreter should get a notice.
4. Project-Starters: Partly in the “core idea”-part in the section Guile Scheme. It might need one more paragraph showing advantages of Guile which make it especially suited for that.
5. 1337s: It is the preferred extension system for the GNU Project. If you’re not that kind of 1337: The Macro-System is hygienic (no surprising side-effects).
6. Emacs users: They got their own section.
7. GNU-Folk: They have a section on Guile in make. We should add a list of GNU projects with Guile support.

So there you go: Not perfect, but most of the groups are covered. And this also ensures that the Wikipedia-page is more interesting to its readers: A clear win-win.

Further points

Additional points which we should keep in mind:

• On the website, do all of our target groups quickly find their way to advanced information about their questions? This is essential to keep the ones interested who aren’t completely taken by the short answers.
• What is a common negative misconception about our project? We need to ensure that we do not write anything which strengthens this misconception. Is there an existing strength, which we can show to counter the negative misconception?
• Where do we want to go? Do we have a mission statement?

bab-com q: Arne Babenhauserheide’s Project Communication Questionaire

• For whom are we already useful or interesting? Name them as Target-Groups.

  • (1)
  • (2)
  • (3)

• Whom do we want as users on the long run? Name them as Target-Groups.

  • (4)
  • (5)
  • (6)
• What could the Target-Groups ask? What are their needs? Formulate them as questions.
  • (1)
  • (2)
  • (3)
  • (4)
  • (5)
  • (6)
• Answer their questions.
  
  • (1)
  • (2)
  • (3)
  • (4)
  • (5)
  • (6)
• Whose needs can we already fulfill well? For whom do we see ourselves as the best choice?
  
  • (1)
  • (2)
  • (3)
  • (4)
• Ensure that our communication includes the answers to these questions (i.e. website, wikipedia page, talks, …), at least for the groups who are likely to use the medium on which we communicate!

Use bab-com to avoid bad-com ☺ - yes, I know this phrase is horrible, but it is catchy and that fits this article: you need catchy things

Note: The mission statement

The mission statement is a short paragraph in which a project defines its goal.

A good example is:

Our mission is to create a general-purpose kernel suitable for the GNU operating system, which is viable for everyday use, and gives users and programs as much control over their computing environment as possible. → GNU Hurd mission explained

Another example again comes from Guile:

Guile was conceived by the GNU Project following the fantastic success of Emacs Lisp as an extension language within Emacs. Just as Emacs Lisp allowed complete and unanticipated applications to be written within the Emacs environment, the idea was that Guile should do the same for other GNU Project applications. This remains true today. → Guile and the GNU project

Closely tied to the mission statement is the slogan: A catch-phrase which helps anchoring the gist of your project in your readers mind. Guile does not have that, yet, but judging from its strengths, the following could work quite well for Guile 2.0 - though it falls short of Guile in general:

GNU Guile scripting: Use Guile Scheme, reuse anything.

Summary

We saw why it is essential to communicate the project to the outside, and we discussed a simple structure to check whether our way of communication actually fits our projects strengths and goals.

Finding the communication strategy actually boils down to 3 steps:

• Target those who would profit from our project or whom we need.
• Check what they need to know.
• Answer that.

Also a clear mission statement, slogan and project description help to make the project more tangible for readers. In this context, good marketing means to ensure that the right people learn about the real strengths of the project.

With that I’ll conclude this guide. Have fun and happy hacking!
— Arne Babenhauserheide

Appendix: Why communicating your project?

In free software we often think that quality is a guarantee for success. But in just the 10 years I’ve been using free software nowadays, I saw my share of technically great projects succumbing to inferior projects which simply reached more people and used that to build a dynamic which greatly outpaced the technically better product.

One example for that are pkgcore and paludis. When portage, the package manager of Gentoo, grew too slow because it did ever more extensive tests, two teams set out to build a replacement.

One of the teams decided that the fault of the low performance lay in Python, the language used by portage. That team built a package manager in C++ and had --wonderfully-long-command-options without shortcuts (have fun typing), and you actually had to run it twice: Once to see what would get installed and then again to actually install it (while portage had had an --ask option for ages, with -a as shortcut). And it forgot all the work it had done in the previous run, so you could wait twice as long for the result. They also had wonderful latin names, and they managed the feat of being even slower than portage, despite being written in C++. So their claim that C++ would be magically faster than python was simply wrong (because they skipped analyzing the real performance bottlenecks). They called their program paludis.

Note: Nowadays paludis has a completely new commandline interface which actually supports short command options. That interface is called cave and looks sane.

The other team did a performance analysis and realized that the low performance actually lay with the filesystem: The portage tree, which holds the required information, contains about 30,000 ebuilds and almost 200,000 files in total, and portage accessed far more of those files than actually needed for resolving the dependencies needed to install the package. They picked python as their language - just like portage. They used almost the same commandline options as portage, except for the places where functionality differed. And they actually got orders of magnitude faster than portage - so fast that their search command often finished after less than a second while. portage took over 10 seconds. They called their program pkgcore.

Both had more exact resolution of packages and could break cyclic dependencies and so on.

So, judging from my account of the quality, which project would you expect to succeed?

I sure expected pkgcore to replace portage within a few months. But this is not what happened. And as I see it in hindsight, the difference lay purely in PR.

The paludis team with their slow and hard-to-use program went all over the Gentoo forums claiming that Python is a horrible language and that a C program will kick portage any time. On their website they repeated their attacks against python and claimed superiority at every step. And they gathered quite a few zealots. While actually being slower than portage. Eventually they rebranded paludis as just better and more correct, not faster. And they created their own distribution (exherbo) as direct rival of Gentoo. With a new, portage-incompatible package format. As if they had read the book, how not to be a friendly competitor.

The pkgcore team on the other hand focussed on good technology. They created the snakeoil library for high-performance python code, but they were friendly about it and actually contributed back to portage where code could be shared. But their website was out of date, often not noting the newest release and you actually had to run pmerge --help to see the most current commandline options (though you could simply guess them if you knew portage). And they got attacked by paludis zealots so much, that this year the main developer finally sacked the project: He told me on IRC that he had taken so much vitriol over the years that it simply wasn’t worth the cost anymore.

Update: About a year later someone else took over. Good code often survives the loss of its creator.

So, what can we learn from this? Technical superiority does not gain you anything, if you fail to convince people to actually use your project.

If you don't communicate your project, you don't get users. If you don’t get users, your chances of losing motivation are orders of magnitude higher than if you get users who support you.

And aggressive marketing works, even if you cannot actually deliver on your promises. Today they have a better user-interface and even short option-names. But even to date, exherbo has much fewer packages in its repositories than Gentoo. If the number of files is any measure, the 10,000 files in their special repositories are just about 5% of the almost 200,000 files portage holds. But they managed quite well to fraction the Gentoo users - at least for some time. And their repeated pushes for new standards in the portage tree (EAPIs) created a constant pressure on pkgcore to adapt, which had the effect that nowadays pkgcore cannot install from the portage tree anymore (the search still works, though, and I still use it - I will curse mightily on the day they manage to also break that).

Update: Someone else took over and now pkgcore can install again.

So aggressive marketing and doing everything in the book of unfriendly competition might have allowed the paludis devs to gather some users and destroy the momentum of pkgcore, but it did not allow them to actually become a replacement of portage within Gentoo. Their behaviour alienated far too many people for that. So aggressive and unfriendly marketing is better than no marketing, but it has severe drawbacks which you will likely want to avoid.

If you use overly aggressive, unfriendly or dishonest communication tactics, you get some users, but if your users know their stuff, you won’t win the mindshare you need to actually make a difference.

If on the other hand you want to see communication done right, just take a look at KDE and Gnome nowadays. They cooperate quite well, and they compete on features and by improving their project so users can take an informed choice about the project they choose.

And their number of contributors steadily keeps growing.

So what do they do? Besides being technically great, it boils down to good marketing.

Conveniently merge a NEWS file without conflicts

echo "
[merge-patterns]
# avoid bogus conflicts in NEWS files
NEWS = internal:union
" >> .hg/hgrc

echo "/NEWS merge=union" >> .gitattributes
git add .gitattributes
git commit -m "union-merge NEWS" .gitattributes

Download one page from a website with all its prerequisites

Often I want to simply backup a single page from a website. Until now I always had half-working solutions, but today I found one solution using wget which works really well, and I decided to document it here. That way I won’t have to search it again, and you, dear readers, can benefit from it, too ☺

Update 2020: You can also use the copyweb-script from pyFreenet: copyweb -d TARGET_FOLDER URL
Install via pip3 install --user pyFreenet3.

In short: This is the command:

wget --no-parent --timestamping --convert-links --page-requisites --no-directories --no-host-directories --span-hosts --adjust-extension --no-check-certificate -e robots=off -U 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.1.6) Gecko/20070802 SeaMonkey/1.1.4' [URL]

Optionally add --directory-prefix=[target-folder-name]

(see the meaning of the options and getting wget for some explanation)

That’s it! Have fun copying single sites! (but before passing them on, ensure that you have the right to do it)

Does this really work?

As a test, how about running this:

wget -np -N -k -p -nd -nH -H -E --no-check-certificate -e robots=off -U 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.1.6) Gecko/20070802 SeaMonkey/1.1.4' --directory-prefix=download-web-site http://draketo.de/english/download-web-page-with-all-prerequisites

(this command uses the short forms of the options)

Then test the downloaded page with firefox:

firefox download-web-site/download-web-page-all-prerequisites.html

Getting wget

If you run GNU/Linux, you likely already have it - and if not, then your package manager has it. GNU wget is one of the standard tools available everywhere.

Some information in the (sadly) typically terse style can be found on the wget website from the GNU project: gnu.org/s/wget.

In case you run Windows, have a look at Wget for Windows from the gnuwin32 project or at GNU Wgetw for Windows from eternallybored.

Alternatively you can get a graphical interface via WinWGet from cybershade.

Or you can get serious about having good tools and install MSYS or Cygwin - the latter gets you some of the functionality of a unix working environment on windows, including wget.

If you run MacOSX, either get wget via fink, homebrew or MacPorts or follow the guide from osxdaily or the german guide from dirk (likely there are more guides - these two were just the first hits in google).

The meaning of the options (and why you need them):

• --no-parent: Only get this file, not other articles higher up in the filesystem hierarchy.
• --timestamping: Only get newer files (don’t redownload files).
• --page-requisites: Get all files needed to display this page.
• --convert-links: Change files to point to the local files you downloaded.
• --no-directories: Do not create directories: Put all files into one folder.
• --no-host-directories: Do not create separate directories per web host: Really put all files in one folder.
• --span-hosts: Get files from any host, not just the one with which you reached the website.
• --adjust-extension: Add a .html extension to the file.
• --no-check-certificate: Do not check SSL certificates. This is necessary if you’re missing one of the host certificates one of the hosts uses. Just use this. If people with enough power to snoop on your browsing would want to serve you a changed website, they could simply use one of the fake certifications authorities they control.
• -e robots=off: Ignore robots.txt files which tell you to not spider and save this website. You are no robot, but wget does not know that, so you have to tell it.
• -U 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.1.6) Gecko/20070802 SeaMonkey/1.1.4': Fake being an old Firefox to avoid blocking based on being wget.
• --directory-prefix=[target-folder-name]: Save the files into a subfolder to avoid having to create the folder first. Without that options, all files are created in the folder in which your shell is at the moment. Equivalent to mkdir [target-folder-name]; cd [target-folder-name]; [wget without --directory-prefix]

Conclusion

If you know the required options, mirroring single pages from websites with wget is fast and easy.

Note that if you want to get the whole website, you can just replace --no-parent with --mirror.

Happy Hacking!

Elegant commandline argument parsing on the shell

Parsing command line arguments on the shell is often done in an ad-hoc fashion, growing unwieldy as time goes by, but there are tools to make that elegant. Here’s a complete example.

I use this in the conf project (easy setup of autotools projects). It builds on the great solution by Adam Katz.

# outer loop to allow processing option arguments at the end
while test ! $# -eq 0; do
    # getopts loop, here you define the short options: 
    # h for -h, l: for -l <lang>. -: provides support for long-options.
    while getopts -- hl:-: arg "$@"; do
        case $arg in
            h ) ARG_HELP=true ;;
            l ) ARG_LANG="$OPTARG" ;;
            - ) LONG_OPTARG="${OPTARG#*=}"
                case "$OPTARG" in
                    help    ) ARG_HELP=true;;
                    lang=?* ) ARG_LANG="$LONG_OPTARG" ;;
                    # FIXME: using the same option twice (either both
                    # after the argument or both before it) gives the
                    # first, not the second value
                    lang*   ) ARG_LANG="${@:$OPTIND:1}" ; OPTIND=$(($OPTIND + 1));;
                    vcs=?*  ) ARG_VCS="$LONG_OPTARG" ;;
                    vcs*    ) ARG_VCS="${@:$OPTIND:1}" ; OPTIND=$(($OPTIND + 1));;
                    '' )      break ;; # "--" terminates argument
                                       # processing to allow giving
                                       # options for autogen.sh after
                                       # --
                    * )       echo "Illegal option --$OPTARG" >&2; exit 2;;
                esac;;
            \? ) exit 2 ;;  # getopts already reported the illegal
                            # option
        esac
    done
    shift $((OPTIND-1)) # remove parsed options and args from $@ list
    # reinitialize OPTIND to allow parsing again
    OPTIND=1
    # provide help output.
    if test x"${ARG_HELP}" = x"true"; then
        echo "${PROG} new [-h | --help] [-l | --lang <LANGUAGE>] [--vcs <VCS>] PROJECT_NAME"
        exit 0
    fi
    # get the argument
    if test x"${1}" = x"--"; then
        if test x"${PROJ}" = x""; then
            echo "Missing project name." >&2; exit 2
        else
            # nothing more to parse.
            # Remove -- from the remaining arguments
            shift 1
            break
        fi
    fi
    if test ! x"${1}" = x""; then
        PROJ="${1%/}" # without trailing slash
    fi
    # remove the argument, then continue the loop to allow putting
    # the options after the argument
    shift 1
done

Additional explanation for this is available from Adam Katz (2015). I’m allowed to include it here, because every answer on Stackoverflow is licensed under creativecommons attribution sharealike (cc by-sa) and because cc by-sa is upwards compatible to GPLv3.

# From Adam Katz, 2015: http://stackoverflow.com/users/519360/adam-katz
# Available at http://stackoverflow.com/a/28466267/7666
# License: cc by-sa: https://creativecommons.org/licenses/by-sa/3.0/
while getopts ab:c-: arg; do
  case $arg in
    a )  ARG_A=true ;;
    b )  ARG_B="$OPTARG" ;;
    c )  ARG_C=true ;;
    - )  LONG_OPTARG="${OPTARG#*=}"
         case $OPTARG in
           alpha    )  ARG_A=true ;;
           bravo=?* )  ARG_B="$LONG_OPTARG" ;;
           bravo*   )  echo "No arg for --$OPTARG option" >&2; exit 2 ;;
           charlie  )  ARG_C=true ;;
           alpha* | charlie* )
                       echo "No arg allowed for --$OPTARG option" >&2; exit 2 ;;
           '' )        break ;; # "--" terminates argument processing
           * )         echo "Illegal option --$OPTARG" >&2; exit 2 ;;
         esac ;;
    \? ) exit 2 ;;  # getopts already reported the illegal option
  esac
done
shift $((OPTIND-1)) # remove parsed options and args from $@ list

With this and with the practical usage at the top you should be able to implement clean commandline parsing with ease.

Happy Hacking!

GNU Guix in 5 minutes

So you get excited when you hear about surviving a power-outage during updates without a hitch and you want to give Guix a try — but woes, you only have 5 minutes of time?

Fear not, that’s enough to get it up and running — all the way to per-user environments and package install as non-priviledged user!

The instructions here are from the official docs, specialized for a GNU Linux host and cut to what I need in a working system.

as user:

$ cd /tmp
$ wget ftp://alpha.gnu.org/gnu/guix/guix-binary-0.8.3.x86_64-linux.tar.xz

become root

$ sudo screen

unpack install and setup Guix

# tar xf guix-binary-0.8.3.x86_64-linux.tar.xz 
# mv var/guix /var/ && mv gnu /
# ln -sf /var/guix/profiles/per-user/root/guix-profile ~root/.guix-profile

Create the build users as per Build-Environment-Setup:

# groupadd --system guixbuild
# for i in `seq -w 1 10`;
   do
      useradd -g guixbuild -G guixbuild           \
              -d /var/empty -s `which nologin`    \
              -c "Guix build user $i" --system    \
              guixbuilder$i;
   done

Run the daemon:

# ~root/.guix-profile/bin/guix-daemon --build-users-group=guixbuild

Switch to a second root window with CTRL-a c to adjust the PATH, allow substitutes from the Hydra build server, and to install and set locales (required since we’re installing an overlay, not a full distro).

# echo 'PATH="$HOME/.guix-profile/bin:$HOME/.guix-profile/sbin:${PATH}"' >> $HOME/.bashrc
# echo 'export LOCPATH=$HOME/.guix-profile/lib/locale'  >> $HOME/.bashrc
# source $HOME/.bashrc
# guix archive --authorize < ~root/.guix-profile/share/guix/hydra.gnu.org.pub
# guix package -i glibc-utf8-locales

Allow all users to use the guix command (as long as guix-daemon is running):

# mkdir -p /usr/local/bin
# cd /usr/local/bin
# ln -s /var/guix/profiles/per-user/root/guix-profile/bin/guix

Switch back to your regular user and provide the guix profile. Also install the locales (remember that the installation is really per-user, though the users share packages if they install them both). The per-user profile will be generated the first time you run guix package.

$ ln -sf /var/guix/profiles/per-user/$(whoami)/guix-profile ~/.guix-profile
$ echo 'export PATH="$HOME/.guix-profile/bin:$HOME/.guix-profile/sbin:${PATH}"' >> $HOME/.bashrc
$ echo 'export LOCPATH=$HOME/.guix-profile/lib/locale'  >> $HOME/.bashrc
$ source $HOME/.bashrc
$ guix package -i glibc-utf8-locales

And now:

$ guix package -i guile-emacs --fallback
$ ~/.guix-profile/bin/emacs -Q

So you believed that only a pipe-dream, just like power-loss-resistant updates and functional packaging using the official GNU extension language? I was glad to be proven wrong, and I hope you’ll feel the same ☺ (though guile-emacs is still experimental it already allows calling elisp functions directly from scheme)

Happy Hacking!

GnuPG/PGP signature, short explanation

»What is the .asc file?« This explanation is intended to be copied as-is into emails when someone asks about your signature.

The .asc file is a signature which can be used to verify that the email was really sent by me and wasn’t tampered with.[1] It can be verified with standard email security tools like Enigmail[2], Gpg4win[3] or MacGPG[4] - and others tools supporting OpenPGP[5].

Best wishes,
Arne

[1]: For further information on signatures see
    https://www.gnupg.org/gph/en/manual/x135.html

[2]: Enigmail enables secure communication in Thunderbird:
    https://addons.mozilla.org/de/thunderbird/addon/enigmail/

[3]: GPG4win provides secure encryption for Windows:
    http://gpg4win.org/download.html

[4]: MacGPG provides encryption for MacOSX:
    https://gpgtools.org/

[5]: Encryption for other systems is available from the GnuPG website:
    https://www.gnupg.org/download/

Going from a simple Makefile to Autotools

Table of Contents

Intro

I recently started looking into Autotools, to make it easier to run my code on multiple platforms.

Naturally you can use cmake or scons or waf or ninja or tup, all of which are interesting in there own respect. But none of them has seen the amount of testing which went into autotools, and none of them have the amount of tweaks needed to support about every system under the sun. And I recently found pyconfigure which allows using autotools with python and offers detection of library features.

Warning 2016: Contains some cargo-cult-programming — my current setup is cleaner thanks to using AC_CONFIG_LINKS in configure.ac.

I had already used Makefiles for easily storing the build information of anything from python projects (python setup.py build) to my PhD thesis with all the required graphs.

I also had used scons for those same tasks.

But I wanted to test, what autotools have to offer. And I found no simple guide which showed me how to migrate from a Makefile to autotools - and what I could gain through that.

So I decided to write one.

# comments start with #
thing : required source files # separated by spaces
    build command
    second build command
# ^ this is a TAB.

all: doktorarbeit.pdf sink.pdf

sink.pdf : sink.tex images/comp-t3-s07-tem-boas.png images/comp-t3-s07-tem-bona.png images/bona-marble.png images/boas-marble.png
    pdflatex sink.tex
    rm -f  *_flymake* flymake* *.log *.out *.toc *.aux *.snm *.nav *.vrb # kill litter

comp-t3-s07-tem-boas.png comp-t3-s07-tem-bona.png : nee-comp.pyx nee-comp.txt
    pyxplot nee-comp.pyx

doktorarbeit.pdf : doktorarbeit.org
    emacs --batch --visit "doktorarbeit.org" --funcall org-export-as-pdf  

dnl run `autoreconf -i` to generate a configure script. 
dnl Then run ./configure to generate a Makefile.
dnl Finally run make to generate the project.

AC_INIT([Doktorarbeit Inverse GHG], [0.1], [arne.babenhauserheide@kit.edu])
dnl we use the build type foreign here instead of gnu because I do not have a NEWS file and similar, yet.
AM_INIT_AUTOMAKE([foreign])
AC_CONFIG_FILES([Makefile])
AC_OUTPUT

pdf_DATA = sink.pdf doktorarbeit.pdf

sink = sink.tex
pkgdata_DATA = images/comp-t3-s07-tem-boas.png images/comp-t3-s07-tem-bona.png
dist_pkgdata_DATA = images/bona-marble.png images/boas-marble.png

plotdir = .
dist_plot_DATA = nee-comp.pyx nee-comp.txt

doktorarbeit = doktorarbeit.org

EXTRA_DIST = ${sink} ${dist_pkgdata_DATA} ${doktorarbeit}

MOSTLYCLEANFILES = \#* *~ *.bak # kill editor backups
CLEANFILES = ${pdf_DATA}
DISTCLEANFILES = ${pkgdata_DATA}

sink.pdf : ${sink} ${pkgdata_DATA} ${dist_pkgdata_DATA}
    TEXINPUTS=${TEXINPUTS}:$(srcdir)/:$(srcdir)/images// pdflatex $<
    rm -f  *_flymake* flymake* *.log *.out *.toc *.aux *.snm *.nav *.vrb # kill litter

${pkgdata_DATA} : ${dist_plot_DATA}
    $(foreach i,$^,if test "$(i)" != "$(notdir $(i))"; then cp -u "$(i)" "$(notdir $(i))"; fi;)
    ${MKDIR_P} images
    pyxplot $<
    $(foreach i,$^,if test "$(i)" != "$(notdir $(i))"; then rm -f "$(notdir $(i))"; fi;)

doktorarbeit.pdf : ${doktorarbeit}
    if test "$<" != "$(notdir $<)"; then cp -u "$<" "$(notdir $<)"; fi
    emacs --batch --visit "$(notdir $<)" --funcall org-export-as-pdf
    if test "$<" != "$(notdir $<)"; then rm -f "$(notdir $<)"; rm -f $(basename $(notdir $<)).tex $(basename $(notdir $<)).tex~; else rm -f $(basename $<).tex $(basename $<).tex~; fi

dnl run `autoreconf -i` to generate a configure script. 
dnl Then run ./configure to generate a Makefile.
dnl Finally run make to generate the project.

AC_INIT([Doktorarbeit Inverse GHG], [0.1], [arne.babenhauserheide@kit.edu])
# Check for programs I need for my build
AC_CANONICAL_TARGET
AC_ARG_VAR([emacs], [How to call Emacs.])
AC_CHECK_TARGET_TOOL([emacs], [emacs], [no])
AC_ARG_VAR([pyxplot], [How to call the Pyxplot plotting tool.])
AC_CHECK_TARGET_TOOL([pyxplot], [pyxplot], [no])
AC_ARG_VAR([pdflatex], [How to call pdflatex.])
AC_CHECK_TARGET_TOOL([pdflatex], [pdflatex], [no])
AS_IF([test "x$pdflatex" = "xno"], [AC_MSG_ERROR([cannot find pdflatex.])])
AS_IF([test "x$emacs" = "xno"], [AC_MSG_ERROR([cannot find Emacs.])])
AS_IF([test "x$pyxplot" = "xno"], [AC_MSG_ERROR([cannot find pyxplot.])])
# Run automake
AM_INIT_AUTOMAKE([foreign])
AM_MAINTAINER_MODE([enable])
AC_CONFIG_FILES([Makefile])
AC_OUTPUT

pdf_DATA = sink.pdf doktorarbeit.pdf

sink = sink.tex
pkgdata_DATA = images/comp-t3-s07-tem-boas.png images/comp-t3-s07-tem-bona.png
dist_pkgdata_DATA = images/bona-marble.png images/boas-marble.png

plotdir = .
dist_plot_DATA = nee-comp.pyx nee-comp.txt

doktorarbeit = doktorarbeit.org

EXTRA_DIST = ${sink} ${dist_pkgdata_DATA} ${doktorarbeit}

MOSTLYCLEANFILES = \#* *~ *.bak # kill editor backups
CLEANFILES = ${pdf_DATA}
DISTCLEANFILES = ${pkgdata_DATA}

sink.pdf : ${sink} ${pkgdata_DATA} ${dist_pkgdata_DATA}
    TEXINPUTS=${TEXINPUTS}:$(srcdir)/:$(srcdir)/images// @pdflatex@ $<
    rm -f  *_flymake* flymake* *.log *.out *.toc *.aux *.snm *.nav *.vrb # kill litter

${pkgdata_DATA} : ${dist_plot_DATA}
    $(foreach i,$^,if test "$(i)" != "$(notdir $(i))"; then cp -u "$(i)" "$(notdir $(i))"; fi;)
    ${MKDIR_P} images
    @pyxplot@ $<
    $(foreach i,$^,if test "$(i)" != "$(notdir $(i))"; then rm -f "$(notdir $(i))"; fi;)

doktorarbeit.pdf : ${doktorarbeit}
    if test "$<" != "$(notdir $<)"; then cp -u "$<" "$(notdir $<)"; fi
    @emacs@ --batch --visit "$(notdir $<)" --funcall org-export-as-pdf
    if test "$<" != "$(notdir $<)"; then rm -f "$(notdir $<)"; rm -f $(basename $(notdir $<)).tex $(basename $(notdir $<)).tex~; else rm -f $(basename $<).tex $(basename $<).tex~; fi  

#!/usr/bin/env python
## I need a couple of special builders for my projects
# the $SOURCE replacement only uses the first source file. $SOURCES gives all.
# specifying all source files makes it possible to rerun the build if a single source file changed.
orgexportpdf = 'emacs --batch --visit "$SOURCE" --funcall org-export-as-pdf'
pyxplot = 'pyxplot $SOURCE'
# pdflatex is quite dirty. I directly clean up after it with rm.
pdflatex = 'pdflatex $SOURCE -o $TARGET; rm -f  *_flymake* flymake* *.log *.out *.toc *.aux *.snm *.nav *.vrb'

# build the PhD thesis from emacs org-mode.
Command("doktorarbeit.pdf", "doktorarbeit.org",
        orgexportpdf)

# create plots
Command(["images/comp-t3-s07-tem-boas.png", 
         "images/comp-t3-s07-tem-bona.png"], 
        ["nee-comp.pyx", 
         "nee-comp.txt"],
        pyxplot)

# build my sink.pdf
Command("sink.pdf", 
        ["sink.tex", 
         "images/comp-t3-s07-tem-boas.png", 
         "images/comp-t3-s07-tem-bona.png", 
         "images/bona-marble.png", 
         "images/boas-marble.png"],
        pdflatex)

# My editors leave tempfiles around. I want them gone after a build clean. This is not yet supported!
tempfiles = Glob('*~') + Glob('#*#') + Glob('*.bak')
# using this here would run the cleaning on every run.
#Command("clean", [], Delete(tempfiles))

sconsrun : scons
    python scons/bootstrap.py -Q

scons : 
    hg clone https://bitbucket.org/ArneBab/scons

#+BEGIN_LaTeX
%%% Local Variables:
%%% TeX-master: t
%%% End:
#+END_LaTeX

How to fix a bug, using the example of Quod Libet empty panes on Gentoo GNU/Linux (bug solving process)

Traceback (most recent call last):
  File "/home/arne/.quodlibet/plugins/songsmenu/albumart.py", line 51, in <module>
    from gi.repository import Gtk, Pango, GLib, Gdk, GdkPixbuf
  File "/usr/lib64/python2.7/site-packages/gi/__init__.py", line 27, in <module>
    from ._gi import _API, Repository
ImportError: cannot import name _API

equery belongs /usr/lib64/python2.7/site-packages/gi/__init__.py

 * Searching for /usr/lib64/python2.7/site-packages/gi/__init__.py ... 
dev-python/pygobject-3.8.3 (/usr/lib64/python2.7/site-packages/gi/__init__.py)

qlop -l pygobject

...
Thu Dec  5 00:26:27 2013 >>> dev-python/pygobject-3.8.3

echo =dev-python/pygobject-3.8.3 >> /usr/portage/package.mask
emerge -u pygobject

LANG=C quodlibet

/usr/lib64/python2.7/site-packages/quodlibet/qltk/songlist.py:44: GtkWarning: Unable to locate theme engine in module_path: "clearlooks",
  _label = gtk.Label().create_pango_layout("")

cd /usr/portage/media-sound/
# create the category in my local portage overlay, defined as
# PORTAGE_OVERLAY=/usr/local/portage in /etc/make.conf
mkdir -p /usr/local/portage/media-sound
# copy over the quodlibet directory, keeping the permissiong with -p
cp -rp quodlibet /usr/local/portage/media-sound
# most times it is enough to simply rename the ebuild to the new version
cd /usr/local/portage/media-sound/quodlibet
mv quodlibet-2.5.ebuild quodlibet-2.5.1.ebuild
# now prepare all the metadata portage needs - this requires
# app-portage/gentoolkit
ebuild quodlibet-2.5.1.ebuild digest compile 
# now it's prepared for the package manager. Just update it as usual:
emerge -u quodlibet

How to run your own GNU Hurd (in 140 letters)

Don’t want to rely on other’s opinions about the Hurd? How to run your own GNU Hurd, in 140 letters:

wget http://people.debian.org/~sthibault/hurd-i386/debian-hurd.img.tar.gz; tar xf de*hu*gz; qemu-system-x86_64 -hda de*hu*g -m 1G

For additional convenience and performance, setup ssh access and enable kvm:

wget http://people.debian.org/~sthibault/hurd-i386/debian-hurd.img.tar.gz; tar xf de*hu*gz; qemu-system-x86_64 -enable-kvm -net user,hostfwd=tcp:127.0.0.1:2222-:22 -net nic -m 1G -drive cache=writeback,file=$(ls de*hu*g)

⇒ login: root, no pw needed. Set a password for user demo:

passwd demo

⇒ log into your Hurd via ssh:

ssh demo@localhost -p 2222

That’s it: You run the Hurd. You you would want to do that? See cat translator_intro — and much more.

Additional information:

• README
• hurd/running/qemu
• Technical Advantages of the Hurd
• Instant Hurd Development Environment
• QEMU

Huge datafiles in free culture projects under GPL

4 ways how large raw artwork files are treated in free culture projects to provide the editable source.¹

In the discussion about license compatibility of the creativecommons sharealike license towards the GPL, Anthony asked how the source-requirement is solved for artwork which often has huge raw files. These are the 4 basic ways I described in my answer.

1. The Wesnoth Way

“The Source is what we have”

The project just asks artists for full resolution PNG image files (without all the layering information) - and only uses these to develop the art. This was spearheaded by the GPL-licensed strategy game Battle for Wesnoth.

This is a viable strategy and also allows developing art, though a bit less convenient than with the layered sources. For example the illustrator who created many of the images in the RPG I work on used our PNG instead of her photoshop file to extract a die from the cover she created for us. She took the chance to also touch up the colors a bit - she had learned some new tricks to improve her paintings.

This clearly complies with the GPL, because the GPL just requires providing the file used for editing published file. If the released file is what you actually use to change published files, then the published file is the source.

2. The External Storage

“Use the FTP, Luke”

Here, files which are too big to be versioned effectively or which most people don’t need when working with the project get version-numbers and are put into an external storage - like an FTP server.

I do that for gimp-files: I put these into our public release-listing via FTP. For example I used that for a multi-layer cover which gets baked into our PDF.

3. The Elegant Way

“Make it so!”

Here huge files are simply versioned alongside other files and the versions to be used are created directly from the multi-layered files. The usual way to do that is a Makefile in which scripts explicitly define how the derived file can be extracted.

This is most elegant, because it has no duplication of information, the source is always trivial to find, it’s always clear that the derived file really originated from the source and it is easy to avoid quality loss or even reduce it later.

The disadvantage is that it can be very cumbersome to force new developers to get all huge files and then create them before being able to really start developing.

The common way to do this is a Makefile - for example the one I use for building my PhD thesis.

4. Pragmatic Elegance

“Hybrids win”

All the ways above can be combined: Huge files are put in version control, but the derived files are included, too, to make it easier for new people to get in. Maybe the huge files are only included on request - for example they could be stubs with which the version control system can retrieve the full files when the user wants them. This can partially be done with the largefiles extension in Mercurial by just not getting the large files.

Also you can just keep separate raw files and derived files. This is also used in Battle for Wesnoth: Optimized files of the right size for the game are stored in one folder while the bigger full resolution files are stored separately.

If you want to include free art in a GPL-covered work, I hope this article gave you some inspiration!

Immutable function arguments and variables

1. Dev A: “Fortran is totally outdated.”
2. Dev B: “I wish we could declare objects in function arguments or variable values as immutable in Java and Javascript.”

Fortran developer silently weeps:

! immutable 2D array as argument in Fortran
  integer, intent(in) :: arg(:,:)
! constant value
  character(len=10), parameter :: numbers = "0123456789"

See parameter vs. intent(in).

(yes, I’m currently reading a Javascript book)

If you now want to see more of Fortran:

• Fortran Surprises
• Writing a commandline tool in Fortran

Installing GNU Guix 0.6, easily

mkdir guix && cd guix
wget http://alpha.gnu.org/gnu/guix/guix-0.6.tar.gz
wget http://alpha.gnu.org/gnu/guix/guix-0.6.tar.gz.sig
gpg --verify guix-0.?.tar.gz.sig

tar xf guix-0.?.tar.gz
cd guix-0.?
./configure && make -j16
sudo make install

sudo screen
groupadd guix-builder
for i in `seq 1 10`;
  do
    useradd -g guix-builder -G guix-builder           \
            -d /var/empty -s `which nologin`          \
            -c "Guix build user $i" --system          \
            guix-builder$i;
  done
exit

sudo mkdir -p /usr/local/var/nix/profiles/per-user/$USER
sudo chown -R $USER:$USER /usr/local/var/nix/profiles/per-user/$USER

chgrp 1002 /nix/store; chmod 1775 /nix/store

sudo screen
echo "#\!/bin/sh" >> /etc/local.d/guix-daemon.start
echo "guix-daemon --build-users-group=guix-builder &" >> /etc/local.d/guix-daemon.start
echo "#\!/bin/sh" >> /etc/local.d/guix-daemon.stop
echo "pkill guix-daemon" >> /etc/local.d/guix-daemon.stop
chmod +x /etc/local.d/guix-daemon.start
chmod +x /etc/local.d/guix-daemon.stop
exit

sudo /etc/local.d/guix-daemon.start

echo "export PATH=$PATH:~/.guix-profile/bin" >> ~/.bashrc
. ~/.bashrc

$ guix --help
Usage: guix COMMAND ARGS...
Run COMMAND with ARGS.

COMMAND must be one of the sub-commands listed below:

   build
   download
   gc
   hash
   import
   package
   pull
   refresh
   substitute-binary

Installing Scipy and PyNIO on a Bare Cluster with the Intel Compiler

export CC='gcc-4.8.1 -Wall -Werror'                                                          
echo $CC | grep \ - | sed 's/.* -/-/'                                                                     
-Werror

#!/bin/bash

# Untar

for i in *.tar* *.tgz; do
  tar xvf $i || exit
done

# Install

PREFIX=/home/ws/babenhau/
PYPREFIX=/home/ws/babenhau/python/

# Blas

cd BLAS
cp ../blas-make.inc make.inc || exit
#make -j9 clean
F77=ifort make -j9 || exit
#make -j9 install --prefix=$PREFIX
# OR for Intel compiler:
ifort -fPIC -FI -w90 -w95 -cm -O3 -xHost -unroll -c *.f || exit
#Continue below irrespective of compiler:
ar r libfblas.a *.o || exit
ranlib libfblas.a || exit
cd ..
ln -s BLAS blas

## Lapack

cd lapack-3.3.1
ln -s ../blas
# this has a hardcoded absolute path to blas in it: replace is with the appropriate one for you.
cp ../lapack-make.inc make.inc || exit
make -j9 clean  || exit
make -j9
make -j9 || exit
cp lapack_LINUX.a libflapack.a || exit
#make -j9 install --prefix=$PREFIX
cd ..

# C interface

patch -p0 < lapacke-ifort.diff

cd lapacke
# patch for lapack 3.3.1 and blas
for i in gnu inc intel ; do 
    sed -i s/lapack-3\.2\.1\\/lapack\.a/lapack-3\.3\.1\\/lapack_LINUX.a/ make.$i; 
    sed -i s/lapack-3\.2\.1\\/blas\.a/blas\\/blas_LINUX.a/ make.$i; 
done

make -j9 clean || exit
#make -j9
LINKER=ifort LDFLAGS=-nofor-main make -j9 # || exit
#LINKER=ifort LDFLAGS=-nofor-main make -j9 install
cd ..

## ATLAS

cd ATLAS
cp ../Make.Linux_HC3 . || exit
echo "ATLAS needs manual intervention. Run make by hand first."
#echo "just say yes. It makes some stuff we need later."
#make
#mv bin/Linux_UNKNOWNSSE2_8 bin/Linux_HC3
#for i in bin/Linux_HC3/*; do sed -i s/UNKNOWNSSE2_8/HC3/ $i ; done
#rm bin/Linux_HC3/Make.inc
#cd bin/Linux_HC3/
#ln -s ../../Make.Linux_HC3 Make.inc
#cd -

make -j9 install arch=Linux_HC3 || exit
cd lib
for i in Linux_HC3/* ; do ln -s $i ; done
cd ../bin
for i in Linux_HC3/* ; do ln -s $i ; done
cd ../include
for i in Linux_HC3/* ; do ln -s $i ; done
cd ..
cd ..

# Numpy and SciPy with intel compilers

# Read this: http://marklodato.github.com/2009/08/30/numpy-scipy-and-intel.html

# patching

patch -p0 < SuiteSparse.diff  || exit
patch -p0 < SuiteSparse-umfpack.diff  || exit

rm numpy
ln -s numpy-*.*.*/ numpy
patch -p0 < numpy-icc.diff  || exit
patch -p0 < numpy-icpc.diff || exit
patch -p0 <<EOF
--- numpy/numpy/distutils/fcompiler/intel.py      2009-03-29 07:24:21.000000000 -0400
+++ numpy/numpy/distutils/fcompiler/intel.py  2009-08-06 23:08:59.000000000 -0400
@@ -47,6 +47,7 @@
     module_include_switch = '-I'

     def get_flags(self):
+        return ['-fPIC', '-cm']
         v = self.get_version()
         if v >= '10.0':
             # Use -fPIC instead of -KPIC.
@@ -63,6 +64,7 @@
         return ['-O3','-unroll']

     def get_flags_arch(self):
+        return ['-xHost']
         v = self.get_version()
         opt = []
         if cpu.has_fdiv_bug():
EOF
# include -fPIC in the fcompiler.
sed -i "s/w90/w90\", \"-fPIC/" numpy/numpy/distutils/fcompiler/intel.py
# and more of that
patch -p0 < numpy-ifort.diff

rm scipy
ln -s scipy-*.*.*/ scipy

patch -p0 < scipy-qhull-icc.diff || exit
patch -p0 < scipy-qhull-icc2.diff || exit

# # unnecessary!
# patch -p0 <<EOF
# --- scipy/scipy/special/cephes/const.c    2009-08-07 01:56:43.000000000 -0400
# +++ scipy/scipy/special/cephes/const.c        2009-08-07 01:57:08.000000000 -0400
# @@ -91,12 +91,12 @@
# double THPIO4 =  2.35619449019234492885;       /* 3*pi/4 */
# double TWOOPI =  6.36619772367581343075535E-1; /* 2/pi */
# #ifdef INFINITIES
# -double INFINITY = 1.0/0.0;  /* 99e999; */
# +double INFINITY = __builtin_inff();
# #else
# double INFINITY =  1.79769313486231570815E308;    /* 2**1024*(1-MACHEP) */
# #endif
# #ifdef NANS
# -double NAN = 1.0/0.0 - 1.0/0.0;
# +double NAN = __builtin_nanf("");
# #else
# double NAN = 0.0;
# #endif
# EOF


# building

# TODO: try again later

cd SuiteSparse

make -j9 -C AMD || exit
make -j9 -C UMFPACK || exit

cd ..

# TODO: build numpy again and make sure it has blas and lapack (and ATLAS?)

cd numpy
python setup.py -v build_src config --compiler=intel build_clib \
    --compiler=intel build_ext --compiler=intel || exit
python setup.py install --prefix=$PYPREFIX || exit
cd ..

# scons and numscons
cd scons-2.0.1
python setup.py -v install --prefix=/home/ws/babenhau/python/ || exit
cd ..

git clone git://github.com/cournape/numscons.git
cd numscons 
python setup.py -v install --prefix=/home/ws/babenhau/python/  || exit
cd ..

# adapt /home/ws/babenhau/python/lib/python2.7/site-packages/numpy/distutils/fcompiler/intel.py by hand to include fPIC for intelem

cd scipy

PYTHONPATH=/home/ws/babenhau/python//lib/scons-2.0.1/ ATLAS=../ATLAS/ \
    LAPACK=../lapack-3.3.1/libflapack.a LAPACK_SRC=../lapack-3.3.1 BLAS=../BLAS/libfblas.a \
    F77=ifort f77_opt=ifort python setup.py -v config --compiler=intel --fcompiler=intelem build_clib \
    --compiler=intel --fcompiler=intelem build_ext --compiler=intel --fcompiler=intelem \
    -I../SuiteSparse/UFconfig # no exit, because we do the linking by hand later on.

# one file is C++ :(
icpc -fPIC -I/home/ws/babenhau/python/include/python2.7 -I/home/ws/babenhau/python/lib/python2.7/site-packages/numpy/core/include -I/home/ws/babenhau/python/lib/python2.7/site-packages/numpy/core/include -c scipy/spatial/qhull/src/user.c -o build/temp.linux-x86_64-2.7/scipy/spatial/qhull/src/user.o || exit

# linking by hand

# for x in csr csc coo bsr dia; do
#    icpc -xHost -O3 -fPIC -shared \
#        build/temp.linux-x86_64-2.7/scipy/sparse/sparsetools/${x}_wrap.o \
#        -o build/lib.linux-x86_64-2.7/scipy/sparse/sparsetools/_${x}.so || exit
# done
#icpc -xHost -O3 -fPIC -openmp -shared \
#   build/temp.linux-x86_64-2.7/scipy/interpolate/src/_interpolate.o \
#   -o build/lib.linux-x86_64-2.7/scipy/interpolate/_interpolate.so || exit

# build again with the C++ file already compiled

PYTHONPATH=/home/ws/babenhau/python//lib/scons-2.0.1/ ATLAS=../ATLAS/ \
    LAPACK=../lapack-3.3.1/libflapack.a LAPACK_SRC=../lapack-3.3.1 BLAS=../BLAS/libfblas.a \
    F77=ifort f77_opt=ifort python setup.py config --compiler=intel --fcompiler=intelem build_clib \
    --compiler=intel --fcompiler=intelem build_ext --compiler=intel --fcompiler=intelem \
    -I../SuiteSparse/UFconfig || exit

# make sure we have cephes
cd scipy/special
PYTHONPATH=/home/ws/babenhau/python//lib/scons-2.0.1/ ATLAS=../../../ATLAS/ \
    LAPACK=../../../lapack-3.3.1/libflapack.a LAPACK_SRC=../lapack-3.3.1 BLAS=../../../BLAS/libfblas.a \
    F77=ifort f77_opt=ifort python setup.py -v config --compiler=intel --fcompiler=intelem build_clib \
    --compiler=intel --fcompiler=intelem build_ext --compiler=intel --fcompiler=intelem \
    -I../../../SuiteSparse/UFconfig
cd ../..

# install
PYTHONPATH=/home/ws/babenhau/python//lib/scons-2.0.1/ ATLAS=../ATLAS/ \
    LAPACK=../lapack-3.3.1/libflapack.a LAPACK_SRC=../lapack-3.3.1 BLAS=../BLAS/libfblas.a \
    F77=ifort f77_opt=ifort python setup.py config --compiler=intel --fcompiler=intelem build_clib \
    --compiler=intel --fcompiler=intelem install --prefix=$PYPREFIX || exit

cd ..

# PyNIO

# netcdf-4

patch -p0 < netcdf-patch1.diff || exit
patch -p0 < netcdf-patch2.diff || exit

cd netcdf-4.1.3

CPPFLAGS="-I/home/ws/babenhau/libbutz/hdf5-1.8.7/include -I/home/ws/babenhau/include" LDFLAGS="-L/home/ws/babenhau/libbutz/hdf5-1.8.7/lib/ -L/home/ws/babenhau/lib -lsz -L/home/ws/babenhau/libbutz/szip-2.1/lib -L/opt/intel/Compiler/11.1/080/lib/intel64/libifcore.a -lifcore" ./configure --prefix=/home/ws/babenhau/ --enable-netcdf-4 --enable-shared || exit

make -j9; make check install -j9 || exit

cd ..

# NetCDF4
cd netCDF4-0.9.7
HAS_SZIP=1 SZIP_PREFIX=/home/ws/babenhau/libbutz/szip-2.1/ HAS_HDF5=1 HDF5_DIR=/home/ws/babenhau/libbutz/hdf5-1.8.7 HDF5_PREFIX=/home/ws/babenhau/libbutz/hdf5-1.8.7 HDF5_includedir=/home/ws/babenhau/libbutz/hdf5-1.8.7/include HDF5_libdir=/home/ws/babenhau/libbutz/hdf5-1.8.7/lib HAS_NETCDF4=1 NETCDF4_PREFIX=/home/ws/babenhau/ python setup.py build_ext --compiler="intel" --fcompiler="intel -fPIC" install --prefix $PYPREFIX
cd ..

# parallel netcdf and hdf5: ~/libbutz/

patch -p0 < pynio-fix-no-grib.diff || exit

cd PyNIO-1.4.1
HAS_SZIP=1 SZIP_PREFIX=/home/ws/babenhau/libbutz/szip-2.1/ HAS_HDF5=1 HDF5_DIR=/home/ws/babenhau/libbutz/hdf5-1.8.7 HDF5_PREFIX=/home/ws/babenhau/libbutz/hdf5-1.8.7 HDF5_includedir=/home/ws/babenhau/libbutz/hdf5-1.8.7/include HDF5_libdir=/home/ws/babenhau/libbutz/hdf5-1.8.7/lib HAS_NETCDF4=1 NETCDF4_PREFIX=/home/ws/babenhau/ python setup.py install --prefix=$PYPREFIX || exit
# TODO: Make sure that the install goes to /home/ws/.., not home/ws/...
cd ..

# satexp_utils.so

f2py -c -m satexp_utils --f77exec=ifort --f90exec=ifort interpolate_levels.F90 || exit

## pyhdf

# recompile hdf with fPIC - grr!
cd hdf-4*/
# Fix configure for compilers with - in the name.
patch -p0 < ../hdf-fix-configure.ac.diff
autoconf
FFLAGS="-ip -O3 -xHost -fPIC -r8" CFLAGS="-ip -O3 -xHost -fPIC" CXXFLAGS="$CFLAGS -I/usr/include/rpc  -DBIG_LONGS -DSWAP" F77=ifort ./configure --prefix=/home/ws/babenhau/ --disable-netcdf --with-szlib=/home/ws/babenhau/libbutz/szip-2.1 # --with-zlib=/home/ws/babenhau/libbutz/zlib-1.2.5 --with-jpeg=/home/ws/babenhau/libbutz/jpeg-8c
# finds zlib and jpeg due to LD_LIBRARY_PATH (hack but works…)
make
make install
cd ..

# build pyhdf
cd pyhdf-0.8.3/
INCLUDE_DIRS="/home/ws/babenhau/include:/home/ws/babenhau/libbutz/szip-2.1/include" LIBRARY_DIRS="/home/ws/babenhau/lib:/home/ws/babenhau/libbutz/szip-2.1/lib" python setup.py build -c intel --fcompiler ifort install --prefix=/home/ws/babenhau/python 
cd ..

## matplotlib

cd matplotlib-1.1.0
patch -p0 < ../matplotlib-add-icc-support.diff
python setup.py build -c intel install --prefix=/home/ws/babenhau/python
cd ..

# GEOS → http://download.osgeo.org/geos/geos-3.3.2.tar.bz2

cd geos*/ 
./configure --prefix=/home/ws/babenhau/
make
make check
make install 
cd ..

# basemap

easy_install --prefix /home/ws/babenhau/python basemap
# fails but should now have all dependencies.

cd basemap-*/

python setup.py build -c intel install --prefix=/home/ws/babenhau/python

cd ..

--- SuiteSparse/UMFPACK/Lib/GNUmakefile 2009-11-11 21:09:54.000000000 +0100
+++ SuiteSparse/UMFPACK/Lib/GNUmakefile 2011-09-09 14:18:57.000000000 +0200
@@ -9,7 +9,7 @@
 C = $(CC) $(CFLAGS) $(UMFPACK_CONFIG) \
     -I../Include -I../Source -I../../AMD/Include -I../../UFconfig \
     -I../../CCOLAMD/Include -I../../CAMD/Include -I../../CHOLMOD/Include \
-    -I../../metis-4.0/Lib -I../../COLAMD/Include
+    -I../../COLAMD/Include

 #-------------------------------------------------------------------------------
 # source files

--- SuiteSparse/UFconfig/UFconfig.mk    2011-09-09 13:14:03.000000000 +0200
+++ SuiteSparse/UFconfig/UFconfig.mk    2011-09-09 13:15:03.000000000 +0200
@@ -33,11 +33,11 @@
 # C compiler and compiler flags:  These will normally not give you optimal
 # performance.  You should select the optimization parameters that are best
 # for your system.  On Linux, use "CFLAGS = -O3 -fexceptions" for example.
-CC = cc
-CFLAGS = -O3 -fexceptions
+CC = icc
+CFLAGS = -O3 -xHost -fPIC -openmp -vec_report=0

 # C++ compiler (also uses CFLAGS)
-CPLUSPLUS = g++
+CPLUSPLUS = icpc

 # ranlib, and ar, for generating libraries
 RANLIB = ranlib
@@ -49,8 +49,8 @@
 MV = mv -f

 # Fortran compiler (not normally required)
-F77 = f77
-F77FLAGS = -O
+F77 = ifort
+F77FLAGS = -O3 -xHost
 F77LIB =

 # C and Fortran libraries
@@ -132,13 +132,13 @@
 # The path is relative to where it is used, in CHOLMOD/Lib, CHOLMOD/MATLAB, etc.
 # You may wish to use an absolute path.  METIS is optional.  Compile
 # CHOLMOD with -DNPARTITION if you do not wish to use METIS.
-METIS_PATH = ../../metis-4.0
-METIS = ../../metis-4.0/libmetis.a
+# METIS_PATH = ../../metis-4.0
+# METIS = ../../metis-4.0/libmetis.a

 # If you use CHOLMOD_CONFIG = -DNPARTITION then you must use the following
 # options:
-# METIS_PATH =
-# METIS =
+METIS_PATH =
+METIS =

 #------------------------------------------------------------------------------
 # UMFPACK configuration:
@@ -194,7 +194,7 @@
 # -DNSUNPERF       for Solaris only.  If defined, do not use the Sun
 #          Performance Library

-CHOLMOD_CONFIG =
+CHOLMOD_CONFIG = -DNPARTITION

 #------------------------------------------------------------------------------
 # SuiteSparseQR configuration:

--- configure.ac    2012-03-01 15:00:28.000000000 +0100
+++ configure.ac    2012-03-01 15:00:40.000000000 +0100
@@ -815,7 +815,7 @@
 dnl Report anything stripped as a flag in CFLAGS and 
 dnl only the compiler in CC_VERSION.
 CC_NOFLAGS=`echo $CC | sed 's/ -.*//'`
-CFLAGS_TO_ADD=`echo $CC | grep - | sed 's/.* -/-/'`
+CFLAGS_TO_ADD=`echo $CC | grep \ - | sed 's/.* -/-/'`
 if test -n $CFLAGS_TO_ADD; then
   CFLAGS="$CFLAGS_TO_ADD$CFLAGS"
 fi

--- lapacke/make.intel.old  2011-10-05 13:24:14.000000000 +0200
+++ lapacke/make.intel  2011-10-05 16:17:00.000000000 +0200
@@ -56,7 +56,7 @@
 # Ensure that the libraries have the same data model (LP64/ILP64).
 #
 LAPACKE = lapacke.a
-LIBS = ../../../lapack-3.3.1/lapack_LINUX.a ../../../blas/blas_LINUX.a -lm
+LIBS = /opt/intel/Compiler/11.1/080/lib/intel64/libifcore.a ../../../lapack-3.2.1/lapack.a ../../../lapack-3.2.1/blas.a -lm -ifcore
 #
 #  The archiver and the flag(s) to use when building archive (library)
 #  If your system has no ranlib, set RANLIB = echo.

diff -r 38c2a32c56ae matplotlib-1.1.0/setup.py
--- a/matplotlib-1.1.0/setup.py Fri Mar 02 12:29:47 2012 +0100
+++ b/matplotlib-1.1.0/setup.py Fri Mar 02 12:30:39 2012 +0100
@@ -31,6 +31,13 @@
 if major==2 and minor1<4 or major<2:
     raise SystemExit("""matplotlib requires Python 2.4 or later.""")

+if "intel" in sys.argv or "icc" in sys.argv:
+    try: # make it compile with the intel compiler
+        from numpy.distutils import intelccompiler
+    except ImportError:
+        print "Compiling with the intel compiler requires numpy."
+        raise
+
 import glob
 from distutils.core import setup
 from setupext import build_agg, build_gtkagg, build_tkagg,\

--- netcdf-4.1.3/fortran/ncfortran.h    2011-07-01 01:22:22.000000000 +0200
+++ netcdf-4.1.3/fortran/ncfortran.h    2011-09-14 14:56:03.000000000 +0200
@@ -658,7 +658,7 @@
  * The following is for f2c-support only.
  */

-#if defined(f2cFortran) && !defined(pgiFortran) && !defined(gFortran)
+#if defined(f2cFortran) && !defined(pgiFortran) && !defined(gFortran) &&!defined(__INTEL_COMPILER)

 /*
  * The f2c(1) utility on BSD/OS and Linux systems adds an additional

--- netcdf-4.1.3/nf_test/fortlib.c  2011-09-14 14:58:47.000000000 +0200
+++ netcdf-4.1.3/nf_test/fortlib.c  2011-09-14 14:58:38.000000000 +0200
@@ -14,7 +14,7 @@
 #include "../fortran/ncfortran.h"


-#if defined(f2cFortran) && !defined(pgiFortran) && !defined(gFortran)
+#if defined(f2cFortran) && !defined(pgiFortran) && !defined(gFortran) &&!defined(__INTEL_COMPILER)
 /*
  * The f2c(1) utility on BSD/OS and Linux systems adds an additional
  * underscore suffix (besides the usual one) to global names that have

--- numpy/numpy/distutils/intelccompiler.py 2011-09-08 14:14:03.000000000 +0200
+++ numpy/numpy/distutils/intelccompiler.py 2011-09-08 14:20:37.000000000 +0200
@@ -30,11 +30,11 @@
     """ A modified Intel x86_64 compiler compatible with a 64bit gcc built Python.
     """
     compiler_type = 'intelem'
-    cc_exe = 'icc -m64 -fPIC'
+    cc_exe = 'icc -m64 -fPIC -xHost -O3'
     cc_args = "-fPIC"
     def __init__ (self, verbose=0, dry_run=0, force=0):
         UnixCCompiler.__init__ (self, verbose,dry_run, force)
-        self.cc_exe = 'icc -m64 -fPIC'
+        self.cc_exe = 'icc -m64 -fPIC -xHost -O3'
         compiler = self.cc_exe
         self.set_executables(compiler=compiler,
                              compiler_so=compiler,