What else is this like?
Destroy nothing; Destroy the most important thing
This commit is contained in:
parent
36956db9fa
commit
546af83dca
2
.gitignore
vendored
Normal file
2
.gitignore
vendored
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
notes/*.tex
|
||||
/notes/*.tex
|
||||
13
etherpad.el
13
etherpad.el
|
|
@ -44,11 +44,13 @@
|
|||
;; - not much in the way of error checking or recovery
|
||||
;; - etc
|
||||
|
||||
|
||||
;;; Code:
|
||||
|
||||
(require 'request)
|
||||
(add-to-list 'load-path ".")
|
||||
|
||||
(require 'let-alist)
|
||||
(require 'ethersync)
|
||||
(require 'request)
|
||||
(require 'cl-lib)
|
||||
|
||||
(defgroup etherpad nil
|
||||
|
|
@ -78,6 +80,12 @@
|
|||
(defvar etherpad--local-pad-revision ""
|
||||
"Buffer local pad details.")
|
||||
|
||||
;; minor mode
|
||||
(define-minor-mode etherpad-mode
|
||||
"Minor mode to sync changes with etherpad." nil " etherpad" nil
|
||||
(if etherpad-mode
|
||||
(ethersync--add-change-hooks)
|
||||
(ethersync--remove-change-hooks)))
|
||||
|
||||
;; API functions
|
||||
|
||||
|
|
@ -265,6 +273,5 @@ should be specific to minor mode and buffer local."
|
|||
(interactive)
|
||||
(remove-hook 'auto-save-hook #'etherpad-save))
|
||||
|
||||
|
||||
(provide 'etherpad)
|
||||
;;; etherpad.el ends here
|
||||
|
|
|
|||
501
ethersync.el
Normal file
501
ethersync.el
Normal file
|
|
@ -0,0 +1,501 @@
|
|||
;;; ethersync.el ---Etherpad easysync protocol -*- coding: utf-8; lexical-binding: t -*-
|
||||
|
||||
;; Copyright 2020 FoAM
|
||||
;;
|
||||
;; Author: nik gaffney <nik@fo.am>
|
||||
;; Created: 2020-12-12
|
||||
;; Version: 0.1
|
||||
;; Package-Requires: ((emacs "26.1") (request "0.3") (let-alist "0.0"))
|
||||
;; Keywords: comm, etherpad, collaborative editing
|
||||
;; URL: https://github.com/zzkt/ethermacs
|
||||
|
||||
;; This file is not part of GNU Emacs.
|
||||
|
||||
;; This program is free software; you can redistribute it and/or modify
|
||||
;; it under the terms of the GNU General Public License as published by
|
||||
;; the Free Software Foundation, either version 3 of the License, or
|
||||
;; (at your option) any later version.
|
||||
|
||||
;; This program is distributed in the hope that it will be useful,
|
||||
;; but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
;; GNU General Public License for more details.
|
||||
|
||||
;; You should have received a copy of the GNU General Public License
|
||||
;; along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
|
||||
;;; Commentary:
|
||||
|
||||
;; Etherpad is a highly customizable Open Source online editor providing
|
||||
;; collaborative editing in really real-time.
|
||||
;;
|
||||
;; The easysync protocol is used for communication of edits, changesets
|
||||
;; and metadata between etherpad server and clients. It uses websockets
|
||||
;; for the transport layer
|
||||
;;
|
||||
;; details -> https://etherpad.org/doc/v1.8.5/#index_http_api
|
||||
|
||||
;; current issues 2020-12-15 00:52:32
|
||||
;; - on ws per buffer. buffer local. shared.
|
||||
;; - incorrect newline counts when sending changesets
|
||||
;; - problems w. deleting text and/or changing buffer size & change-hooks
|
||||
;; - see "Additional Constraints" of easysync
|
||||
;; - potential race conditions sending changesets on buffer changes
|
||||
;; - doesn't apply attributes or changes from the apool
|
||||
;; - general lack of error checking
|
||||
|
||||
;;; Code:
|
||||
|
||||
(require 'websocket)
|
||||
(require 'let-alist)
|
||||
(require 'calc-bin)
|
||||
(require 'parsec)
|
||||
(require '0xc)
|
||||
(require 's)
|
||||
|
||||
;; debug details
|
||||
(setq websocket-debug t)
|
||||
|
||||
;; local and buffer local variables
|
||||
(defvar-local ep--pre-buffer-length 0)
|
||||
(defvar-local ep--new-buffer-length 0)
|
||||
(defvar-local ep--change-length 0)
|
||||
(defvar-local ep--change-string "")
|
||||
|
||||
(defvar-local ep--local-rev 0)
|
||||
(defvar-local ep--current-pad "")
|
||||
(defvar-local ep--local-author "") ;; e.g. "a.touCZaixjPgKDSiN"
|
||||
|
||||
(defvar-local ep--hearbeat-timer nil)
|
||||
(defvar-local ep--current-socket nil)
|
||||
|
||||
;; enable/disable keep alive message to the server
|
||||
(defun ethersync-heartbeat-start ()
|
||||
"Maintain connection to server with periodic pings."
|
||||
(message "heartbeat started: %s" (current-buffer))
|
||||
(setq ep--hearbeat-timer
|
||||
(run-with-timer 5 15 #'wss-send "2")))
|
||||
|
||||
(defun ethersync-heartbeat-stop ()
|
||||
"Stop sending keep-alive messages."
|
||||
(cancel-timer ep--hearbeat-timer))
|
||||
|
||||
;; buffering
|
||||
(defvar *etherpad-buffer* (generate-new-buffer "*etherpad*"))
|
||||
|
||||
(defun ethersync-current-socket (&optional socket)
|
||||
"Return currently active socket or set SOCKET as current."
|
||||
(message "socket in buffer: %s" (current-buffer))
|
||||
(if socket
|
||||
(setq ep--current-socket socket)
|
||||
ep--current-socket))
|
||||
|
||||
;; setters
|
||||
(defun ethersync--set-local-rev (n)
|
||||
"Set the local revision."
|
||||
(message "current rev: %s" ep--local-rev)
|
||||
(setq ep--local-rev n)
|
||||
(message "updated rev: %s" ep--local-rev)
|
||||
(with-current-buffer *etherpad-buffer*
|
||||
(rename-buffer (format "etherpad:%s:%s"
|
||||
ep--current-pad
|
||||
ep--local-rev))))
|
||||
|
||||
;; see also -> inhibit-modification-hooks
|
||||
|
||||
(defun ethersync--add-change-hooks ()
|
||||
(interactive)
|
||||
(message "setting up buffer change hooks")
|
||||
(with-current-buffer *etherpad-buffer*
|
||||
(add-hook 'before-change-functions
|
||||
'ethersync--before-buffer-changes nil t)
|
||||
;; ordering is important...
|
||||
(add-hook 'after-change-functions
|
||||
'ethersync--after-buffer-changes 22 t)
|
||||
(add-hook 'after-change-functions
|
||||
'ethersync--send-changes 23 t)))
|
||||
|
||||
(defun ethersync--remove-change-hooks ()
|
||||
(interactive)
|
||||
(message "removing buffer change hooks")
|
||||
(with-current-buffer *etherpad-buffer*
|
||||
(remove-hook 'before-change-functions
|
||||
'ethersync--before-buffer-changes t)
|
||||
(remove-hook 'after-change-functions
|
||||
'ethersync--after-buffer-changes t)
|
||||
(remove-hook 'after-change-functions
|
||||
'ethersync--send-changes t)))
|
||||
|
||||
|
||||
(defun ethersync--before-buffer-changes (begin end)
|
||||
;; (message "before -> b:%s e:%s" begin end)
|
||||
(setq-local ep--pre-buffer-length (length (buffer-string))
|
||||
ep--change-length (- end begin)))
|
||||
|
||||
(defun ethersync--after-buffer-changes (begin end pre)
|
||||
;; (message "after -> b:%s e:%s p:%s" begin end pre)
|
||||
(setq-local ep--new-buffer-length (length (buffer-string))
|
||||
ep--change-string (buffer-substring begin end)
|
||||
ep--change-length (- end begin)))
|
||||
|
||||
;; (message "eabc: pre: %s post: %s changed: %s chars to: %s at: %s(%s)"
|
||||
;; ep--pre-buffer-length
|
||||
;; ep--new-buffer-length
|
||||
;; ep--change-length
|
||||
;; ep--change-string
|
||||
;; (point)
|
||||
;; (n-36 (point))))
|
||||
|
||||
|
||||
;; emacs -> etherpad changes
|
||||
|
||||
(defun ethersync--send-changes (_b _e _p)
|
||||
"Create and encode a changeset."
|
||||
(let* ((b0 ep--pre-buffer-length)
|
||||
(b1 ep--new-buffer-length)
|
||||
(ops (if (< b0 b1)
|
||||
(format "+%s" (- b1 b0))
|
||||
(format "-%s" (- b0 b1))))
|
||||
(changeset
|
||||
(ethersync--encode-changeset
|
||||
b0
|
||||
(- b1 b0)
|
||||
ops
|
||||
ep--change-string)))
|
||||
(message "changeset: %s" changeset)
|
||||
(ethersync--send-user-changes changeset)))
|
||||
|
||||
|
||||
(defun ethersync--encode-changeset (length change-size ops chars)
|
||||
"Create a changeset from some buffer activty."
|
||||
(message "encoding: o:%s (%s) cs:%s op:%s ch:%s" length (n-36 length) change-size ops chars)
|
||||
(let* ((change (cond ((= 0 change-size) "=0")
|
||||
((> 0 change-size)
|
||||
(format "<%s" (n-36 (abs change-size))))
|
||||
((< 0 change-size)
|
||||
(format ">%s" (n-36 change-size)))))
|
||||
(newline-count (s-count-matches "\n" (buffer-substring
|
||||
(point-min)
|
||||
(point))))
|
||||
(offset (- (point) (length chars) 1))
|
||||
;; offset is distance from point-min to point w.out inserted chars and w. newlines
|
||||
(pos-op (if (< 0 newline-count)
|
||||
(format "|%s=%s=%s"
|
||||
;; 2 steps reqd. newline insert, then from beginning of line?
|
||||
(n-36 newline-count) (n-36 offset)
|
||||
(n-36 (caar
|
||||
(reverse
|
||||
(s-matched-positions-all
|
||||
"\n" (buffer-substring
|
||||
(point-min) (point)))))))
|
||||
(format "=%s" (n-36 offset)))))
|
||||
(format "Z:%s%s%s%s$%s" (n-36 length) change pos-op ops chars)))
|
||||
|
||||
|
||||
(defun ethersync--send-user-changes (cs)
|
||||
"Send a 'USER_CHANGES' message with changeset CS."
|
||||
(let* ((author ep--local-author)
|
||||
(rev ep--local-rev)
|
||||
(changeset cs)
|
||||
(payload (format "42[\"message\",{\"type\":\"COLLABROOM\",\"component\":\"pad\",\"data\":{\"type\":\"USER_CHANGES\",\"baseRev\":%s,\"changeset\":\"%s\",\"apool\":{\"numToAttrib\":{},\"nextNum\":1}}}]" rev changeset author)))
|
||||
|
||||
(message "send this -> %s" payload)
|
||||
(wss-send payload)
|
||||
))
|
||||
|
||||
;; parsec info https://github.com/cute-jumper/parsec.el
|
||||
|
||||
(defun ethersync-parse-changeset (cs)
|
||||
"Parse a changeset CS.
|
||||
|
||||
:N : Source text has length N (must be first op)
|
||||
>N : Final text is N (positive) characters longer than source text (must be second op)
|
||||
<N : Final text is N (positive) characters shorter than source text (must be second op)
|
||||
>0 : Final text is same length as source text
|
||||
+N : Insert N characters from the bank, none of them newlines
|
||||
-N : Skip over (delete) N characters from the source text, none of them newlines
|
||||
=N : Keep N characters from the source text, none of them newlines
|
||||
|L+N : Insert N characters from the source text, containing L newlines. The last
|
||||
character inserted MUST be a newline, but not the (new) document's final newline.
|
||||
|L-N : Delete N characters from the source text, containing L newlines. The last
|
||||
character inserted MUST be a newline, but not the (old) document's final newline.
|
||||
|L=N : Keep N characters from the source text, containing L newlines. The last character
|
||||
kept MUST be a newline, and the final newline of the document is allowed.
|
||||
*I : Apply attribute I from the pool to the following +, =, |+, or |= command.
|
||||
In other words, any number of * ops can come before a +, =, or | but not
|
||||
between a | and the corresponding + or =.
|
||||
If +, text is inserted having this attribute. If =, text is kept but with
|
||||
the attribute applied as an attribute addition or removal.
|
||||
Consecutive attributes must be sorted lexically by (key,value) with key
|
||||
and value taken as strings. It's illegal to have duplicate keys
|
||||
for (key,value) pairs that apply to the same text. It's illegal to
|
||||
have an empty value for a key in the case of an insertion (+), the
|
||||
pair should just be omitted."
|
||||
|
||||
(let* ((changes
|
||||
(parsec-with-input cs
|
||||
;; a letter Z (the "magic character" and format version identifier)
|
||||
(parsec-str "Z")
|
||||
(parsec-collect*
|
||||
;; source text length
|
||||
(parsec-re ":[0-9a-z]+")
|
||||
;; change in text length
|
||||
(parsec-re "[>=<][0-9a-z]+")
|
||||
;; insertion & deletion operations
|
||||
(parsec-many
|
||||
(parsec-or
|
||||
(parsec-re "|[0-9a-z]+[+-=][0-9a-z]+")
|
||||
(parsec-re "[><+-=*][0-9a-z]+")))
|
||||
;; separator
|
||||
(parsec-str "$")
|
||||
;; a string of characters used by insertion operations (the "char bank")
|
||||
(parsec-many-s
|
||||
(parsec-any-ch))))))
|
||||
|
||||
(let* ((old-length
|
||||
(0xc-string-to-number (substring (car changes) 1) 36))
|
||||
(change-sign
|
||||
(if (s-equals? ">" (substring (nth 1 changes) 0 1)) 1 -1))
|
||||
(change-size
|
||||
(0xc-string-to-number (substring (nth 1 changes) 1) 36))
|
||||
(new-length
|
||||
(+ old-length (* change-sign change-size)))
|
||||
(ops
|
||||
(nth 2 changes))
|
||||
(chars
|
||||
(car (last changes))))
|
||||
;;(message "old length: %s new length: %s ops: %s chars: %s" old-length new-length ops chars)
|
||||
(list old-length ops chars)
|
||||
)))
|
||||
|
||||
|
||||
;; radix reductions
|
||||
|
||||
(defun s-36 (string)
|
||||
"Convert a base-36 number STRING to decimal."
|
||||
(0xc-string-to-number string 36))
|
||||
|
||||
(defun n-36 (number)
|
||||
"Convert a decimal NUMBER to base-36 as string."
|
||||
(let ((calc-number-radix 36))
|
||||
(downcase
|
||||
(math-format-radix number))))
|
||||
|
||||
|
||||
;; operations -> buffer changes
|
||||
|
||||
(defun ethersync-apply-ops (ops chars)
|
||||
"Apply a series of insert/delete operations.
|
||||
Numeric offsets are calculated from the beginning of the buffer."
|
||||
(with-current-buffer *etherpad-buffer*
|
||||
(save-mark-and-excursion
|
||||
(goto-char (point-min))
|
||||
(let ((char-bank chars))
|
||||
(mapcar
|
||||
(lambda (s)
|
||||
(let* ((o1 (s-left 1 s))
|
||||
(p1 (substring s 1)))
|
||||
(message "op: %s val: %s" o1 p1)
|
||||
(pcase o1
|
||||
("+" (ethersync-insert (s-left (s-36 p1) char-bank))
|
||||
(setq char-bank (s-right (s-36 p1) char-bank)))
|
||||
("-" (ethersync-delete (s-36 p1)))
|
||||
("=" (ethersync-keep (s-36 p1)))
|
||||
("|" (let* ((p2 (s-split "[+=-]" (s-36 p1)))
|
||||
(l1 (s-36 (car p2)))
|
||||
(n1 (s-36 (cadr p2))))
|
||||
;; doesn't insert or delete newlines correctly (yet)
|
||||
(message "ops: l1:%s n1:%s" l1 n1)
|
||||
(pcase p1
|
||||
((pred (s-matches? "+"))
|
||||
(ethersync-insert char-bank)
|
||||
(setq char-bank (s-right n1 char-bank)))
|
||||
((pred (s-matches? "-")) (ethersync-delete n1))
|
||||
((pred (s-matches? "=")) (ethersync-keep n1)))))
|
||||
("*" t)
|
||||
(_ nil)
|
||||
)))
|
||||
ops)))))
|
||||
|
||||
|
||||
;; character operations for remote->local sync
|
||||
;; which should not trigger change hooks
|
||||
|
||||
(defun ethersync-insert (chars)
|
||||
"Insert CHARS into the source text."
|
||||
(let ((inhibit-modification-hooks t))
|
||||
(insert chars)))
|
||||
|
||||
(defun ethersync-delete (n)
|
||||
"Delete (skip over) N chars from the source text."
|
||||
(let ((inhibit-modification-hooks t))
|
||||
(delete-char n)))
|
||||
|
||||
(defun ethersync-keep (n)
|
||||
"Keep N chars from the source text."
|
||||
(let ((inhibit-modification-hooks t))
|
||||
(forward-char n)))
|
||||
|
||||
;; start with current pad text
|
||||
|
||||
(defun ethersync-init-text (chars)
|
||||
"Seeds a buffer with CHARS from a remote pad"
|
||||
(with-current-buffer *etherpad-buffer*
|
||||
(let ((inhibit-modification-hooks t))
|
||||
(erase-buffer)
|
||||
(goto-char (point-min))
|
||||
(insert chars))))
|
||||
|
||||
(defun ethersync-try-changeset (cs)
|
||||
(let* ((changes
|
||||
(ethersync-parse-changeset cs))
|
||||
(len (nth 0 changes))
|
||||
(ops (nth 1 changes))
|
||||
(chars (nth 2 changes)))
|
||||
(ethersync--check-length len)
|
||||
(ethersync-apply-ops ops chars)))
|
||||
|
||||
(defun ethersync--check-length (size)
|
||||
"Check the changeset and buffer SIZE are consistent."
|
||||
(when (not (= size (length (buffer-string))))
|
||||
(message "changeset and buffer length are inconsistent.")))
|
||||
|
||||
;; various stanzas
|
||||
|
||||
(defun ethersync--request-client-ready (padId)
|
||||
"Ethersync: send CLIENT_READY for PADID."
|
||||
(format "42[\"message\",{\"component\":\"pad\",\"type\":\"CLIENT_READY\",\"padId\":\"%s\",\"token\":\"%s\",\"protocolVersion\":2}]" padId *session-token*))
|
||||
|
||||
(defun ethersync--request-get-comments (padId)
|
||||
"Ethersync: request comments on PADID."
|
||||
(format "42/comment,0[\"getComments\",{\"padId\":\"%s\"}]" padId))
|
||||
|
||||
(defun ethersync--request-get-comment-replies (padId)
|
||||
"Ethersync: request comment replies on PADID."
|
||||
(format "42/comment,1[\"getCommentReplies\",{\"padId\":\"%s\"}]" padId))
|
||||
|
||||
|
||||
;; sending via websockets
|
||||
|
||||
(defalias 'wss-send #'ethersync-wss-send)
|
||||
|
||||
(defun ethersync-wss-send (msg)
|
||||
"Send MSG to a websocket."
|
||||
(when (stringp msg)
|
||||
(websocket-send-text
|
||||
(ethersync-current-socket) msg)))
|
||||
|
||||
|
||||
;; parsing & dispatch of incoming frames
|
||||
|
||||
(defun ethersync-parse-wsframe (_websocket frame)
|
||||
"Parse & dispatch incoming FRAME.
|
||||
Parsing occurs with-current-buffer for constancy with buffer-local variables
|
||||
use let bindings for multiple connections."
|
||||
;(message "parsing: %s" frame)
|
||||
(with-current-buffer *etherpad-buffer*
|
||||
(let* ((fr0 (websocket-frame-text frame))
|
||||
(frp (parsec-with-input
|
||||
fr0
|
||||
(parsec-collect* (parsec-re "[0-9]+")
|
||||
(parsec-many-s (parsec-any-ch))))))
|
||||
(message "frame: %s" (length fr0))
|
||||
(pcase (car frp)
|
||||
("0" (ethersync--parse-0 frp))
|
||||
("2" (ethersync--parse-2 frp))
|
||||
("3" (message "3: keep-alive"))
|
||||
("42" (ethersync--parse-42 frp))
|
||||
))))
|
||||
|
||||
;; parse various incoming message types
|
||||
|
||||
(defun ethersync--parse-0 (p0)
|
||||
"Parse messages beginning with 0 from P0.
|
||||
set sid, upgrades, pingInterval and pingTimeout for session."
|
||||
(when (listp p0)
|
||||
(pcase (length p0)
|
||||
(0 nil)
|
||||
(1 (car p0))
|
||||
(_ (let* ((p1 (json-parse-string (nth 1 p0) :object-type 'alist))
|
||||
(sid (alist-get 'sid p1)))
|
||||
(message "sid %s" sid)
|
||||
)))))
|
||||
|
||||
|
||||
(defun ethersync--parse-2 (p0)
|
||||
"Parse messages beginning with 2 from P0.
|
||||
set revisions etc."
|
||||
(when (listp p0)
|
||||
(pcase (length p0)
|
||||
(0 nil)
|
||||
(1 (car p0))
|
||||
(_ (let* ((p1 (json-parse-string (nth 1 p0) :object-type 'alist)))
|
||||
(let-alist (aref p1 1)
|
||||
(pcase .type
|
||||
("COLLABROOM"
|
||||
(pcase .data.type
|
||||
("ACCEPT_COMMIT"
|
||||
(message "accepted changes: rev:%s and %s (by %s)"
|
||||
.data.newRev)
|
||||
(ethersync--set-local-rev .data.newRev)))))))))))
|
||||
|
||||
|
||||
(defun ethersync--parse-42 (p0)
|
||||
"Parse messages beginning with 42 from P0.
|
||||
most of the COLLABROOM and update stuff..."
|
||||
(when (listp p0)
|
||||
(pcase (length p0)
|
||||
(0 nil)
|
||||
(1 (car p0))
|
||||
(_ (let* ((p1 (json-parse-string (nth 1 p0) :object-type 'alist)))
|
||||
(let-alist (aref p1 1)
|
||||
(pcase .type
|
||||
("COLLABROOM"
|
||||
(pcase .data.type
|
||||
|
||||
("USER_NEWINFO"
|
||||
(message "42: new user %s (color %s)"
|
||||
.data.userInfo.userId
|
||||
.data.userInfo.colorId))
|
||||
|
||||
("NEW_CHANGES"
|
||||
(message "42: new_changes rev:%s changeset:%s (by %s)"
|
||||
.data.newRev
|
||||
.data.changeset
|
||||
.data.author)
|
||||
(ethersync--set-local-rev .data.newRev)
|
||||
(ethersync-try-changeset .data.changeset))
|
||||
|
||||
("USER_CHANGES"
|
||||
(message "42: user_changes rev:%s changeset:%s (by %s)"
|
||||
.data.baseRev
|
||||
.data.changeset
|
||||
.data.apool.author))
|
||||
|
||||
("ACCEPT_COMMIT"
|
||||
(message "42: accept-commit rev:%s" .data.newRev)
|
||||
(ethersync--set-local-rev .data.newRev)
|
||||
)))
|
||||
|
||||
("CLIENT_READY"
|
||||
(message "42: ready -> %s and %s" .padId .token))
|
||||
|
||||
("CLIENT_VARS"
|
||||
(message "42: client_vars (%s) rev:%s -> %s"
|
||||
.data.padId
|
||||
.data.collab_client_vars.rev
|
||||
.data.collab_client_vars.initialAttributedText.text)
|
||||
(ethersync--set-local-rev
|
||||
.data.collab_client_vars.rev)
|
||||
(ethersync-init-text
|
||||
.data.collab_client_vars.initialAttributedText.text)))
|
||||
|
||||
(pcase .disconnect
|
||||
("badChangeset"
|
||||
(message "42: disconnect (%s)" .disconnect)))
|
||||
))))))
|
||||
|
||||
|
||||
(provide 'ethersync)
|
||||
;;; ethersync.el ends here
|
||||
474
notes/etherpad-protocol.org
Normal file
474
notes/etherpad-protocol.org
Normal file
|
|
@ -0,0 +1,474 @@
|
|||
# -*- mode: org; coding: utf-8; -*-
|
||||
#+LaTeX_CLASS: zzkt-article
|
||||
#+LateX_Header: \setcounter{secnumdepth}{0}
|
||||
#+OPTIONS: toc:2
|
||||
#+author: nik gaffney
|
||||
#+title: etherpad protocol notes
|
||||
|
||||
#+begin_export latex
|
||||
\newpage
|
||||
#+end_export
|
||||
* etherpad & the easysync protocol
|
||||
|
||||
- [[https://github.com/ether/etherpad-lite/tree/develop/doc/easysync][easysync protocol description & notes]]
|
||||
- [[http://geekdirt.com/blog/how-etherpad-works/][How etherpad-lite, a real time collaborative editor, works?]]
|
||||
|
||||
possibly relevant parts of the etherpad code
|
||||
- https://github.com/ether/etherpad-lite/blob/develop/src/node/handler/PadMessageHandler.js
|
||||
- https://github.com/payload/ethersync/blob/master/src/ethersync.coffee
|
||||
- code for a [[https://github.com/JohnMcLear/etherpad-cli-client/blob/master/lib/index.js][cli-client]] and the [[https://github.com/ether/etherpad-lite/tree/develop/src/static/js][javasctipt client]]
|
||||
|
||||
* websockets & socket.io
|
||||
|
||||
etherpad uses socket.io for realtime communication between server and clients. socket.io uses ws and wss as transport protocols, but not directly compatible. appears to work as expected by adding headers and/or prefixes to json data sent via wss (“Socket.IO is NOT a WebSocket implementation” according to the [[https://socket.io/docs/][socket.io docs]])
|
||||
|
||||
* websockets in emacs
|
||||
|
||||
see https://blog.abrochard.com/websockets.html and [[https://github.com/ahyatt/emacs-websocket][emacs-websocket]] for details
|
||||
|
||||
* protocol, probes & partials
|
||||
|
||||
browser client sends url with pad name to server (e.g. https://etherpad.wikimedia.org/p/test ) establishes session (sid) and receives token (in cookie data). updates, changes & pad metadata are sent via wss connection. (e.g. wss://etherpad.wikimedia.org/socket.io/?EIO=3&transport=websocket&sid=Ap47gBZD98dHcW38AoqY )
|
||||
|
||||
** overview
|
||||
|
||||
#+BEGIN_SRC plantuml :exports none :file proto-x1.png
|
||||
!include https://raw.githubusercontent.com/bschwarz/puml-themes/master/themes/cerulean/puml-theme-cerulean.puml
|
||||
|
||||
== init ==
|
||||
client -> ep_server: wss://example.org//socket.io/?EIO=3&transport=websocket
|
||||
ep_server --> client: 0 sid, upgrades, etc
|
||||
client -> ep_server: 2 CLIENT_READY padId, token, etc
|
||||
ep_server --> client: 42 CLIENT_VARS pad text, lots of junk about server, colours, authors, etc
|
||||
ep_server --> client: 42 USER_NEWINFO (if other active clients)
|
||||
|
||||
== local edits ==
|
||||
client -> ep_server: 42 USER_CHANGES baseRev, changeset
|
||||
ep_server --> client: 42 ACCEPT_COMMIT newRev
|
||||
note right: COLLABROOM
|
||||
|
||||
== edits from elsewhere ==
|
||||
ep_server --> client: 42 USER_NEWINFO
|
||||
ep_server --> client: 42 NEW_CHANGES newrev, changeset, author, etc
|
||||
ep_server --> client: 42 USER_LEAVE
|
||||
note right: COLLABROOM
|
||||
|
||||
== keep-alive ==
|
||||
client -> ep_server: 2
|
||||
ep_server --> client: 3
|
||||
|
||||
#+END_SRC
|
||||
|
||||
#+RESULTS:
|
||||
[[file:proto-x1.png]]
|
||||
|
||||
client -> ep_server:
|
||||
ep_server --> client:
|
||||
|
||||
|
||||
#+CAPTION: overview of etherpad/easysync protocol
|
||||
#+ATTR_ORG: :width 400
|
||||
#+ATTR_LaTeX: :height 15cm :placement [!h]
|
||||
#+RESULTS:
|
||||
[[file:proto-x1.png]]
|
||||
|
||||
** comment plugin
|
||||
|
||||
#+BEGIN_SRC plantuml :exports none :file proto-x2.png
|
||||
!include https://raw.githubusercontent.com/bschwarz/puml-themes/master/themes/cerulean/puml-theme-cerulean.puml
|
||||
|
||||
title comments
|
||||
|
||||
== comments ==
|
||||
client -> ep_server: 40/comment,
|
||||
ep_server --> client: 40/comment,
|
||||
client -> ep_server: 42/comment getComments, padId
|
||||
client -> ep_server: 42/comment getCommentReplies, padId
|
||||
ep_server --> client: 43/comment comments
|
||||
ep_server --> client: 43/comment comment replies
|
||||
|
||||
== updates (new) ==
|
||||
ep_server --> client: 42/comment pushAddCommentReply, commentId, text, etc
|
||||
client -> ep_server: 42/comment getCommentReplies, padId
|
||||
ep_server --> client: 43/comment replies, etc
|
||||
|
||||
== updates (changes) ==
|
||||
ep_server --> client: 42/comment, textCommentUpdated
|
||||
|
||||
== updates (deletion) ==
|
||||
ep_server --> client: 42/comment, commentDeleted
|
||||
ep_server --> client: 42 NEW_CHANGES
|
||||
|
||||
#+END_SRC
|
||||
|
||||
#+CAPTION: comments
|
||||
#+ATTR_ORG: :width 400
|
||||
#+ATTR_LaTeX: :height 15cm :placement [!h]
|
||||
#+RESULTS:
|
||||
[[file:proto-x2.png]]
|
||||
|
||||
** example messages
|
||||
|
||||
*init/request*
|
||||
#+BEGIN_SRC text
|
||||
40/comment,
|
||||
|
||||
42/comment,0["getComments",{"padId":"test"}]
|
||||
42/comment,1["getCommentReplies",{"padId":"test"}]
|
||||
43/comment,0[{"comments":{"c-4U2BW8J2Lp0r68ZL":{"author":"a.0iRJZx7jiOAxVNMP","name":"zzkt","text":"yes","timestamp":1607769834917}}}]
|
||||
|
||||
43/comment,1[{"replies":{}}]
|
||||
#+END_SRC
|
||||
|
||||
*updates (new)*
|
||||
#+BEGIN_SRC text
|
||||
42/comment,["pushAddCommentReply","c-reply-vMSgWSY4bFhaCCLR",{"commentId":"c-4U2BW8J2Lp0r68ZL","text":"no","changeTo":null,"changeFrom":null,"author":"a.0iRJZx7jiOAxVNMP","name":"zzkt","timestamp":1607770300230,"replyId":"c-reply-vMSgWSY4bFhaCCLR"}]
|
||||
|
||||
42/comment,2["getCommentReplies",{"padId":"test"}]
|
||||
|
||||
43/comment,2[{"replies":{"c-reply-vMSgWSY4bFhaCCLR":{"commentId":"c-4U2BW8J2Lp0r68ZL","text":"no","changeTo":null,"changeFrom":null,"author":"a.0iRJZx7jiOAxVNMP","name":"zzkt","timestamp":1607770300230}}}]
|
||||
#+END_SRC
|
||||
|
||||
*updates (changes)*
|
||||
#+BEGIN_SRC text
|
||||
42/comment,["textCommentUpdated","c-reply-vMSgWSY4bFhaCCLR","not yet"]
|
||||
#+END_SRC
|
||||
|
||||
*updates (deletion)*
|
||||
#+BEGIN_SRC text
|
||||
42/comment,["commentDeleted","c-4U2BW8J2Lp0r68ZL"]
|
||||
42["message",{"type":"COLLABROOM","data":{"type":"NEW_CHANGES","newRev":234,"changeset":"Z:e>0=7*0=4$","apool":{"numToAttrib":{"0":["comment","comment-deleted"]},"attribToNum":{"comment,comment-deleted":0},"nextNum":1},"author":"a.0iRJZx7jiOAxVNMP","currentTime":1607770511397,"timeDelta":null}}]
|
||||
#+END_SRC
|
||||
|
||||
|
||||
** changesets
|
||||
|
||||
via https://github.com/ether/etherpad-lite/
|
||||
…and [[https://raw.githubusercontent.com/ether/etherpad-lite/develop/doc/easysync/easysync-notes.txt][easysync notes]]
|
||||
|
||||
An "attribute" is a (key,value) pair such as (author,abc123456) or (bold,true). Sometimes an attribute is treated as an instruction to add that attribute, in which case an empty value means to remove it. So (bold,) removes the "bold" attribute. Attributes are interned and given numeric IDs, so the number "6" could represent "(bold,true)", for example. This mapping is stored in an attribute "pool" which may be shared by multiple changesets.
|
||||
|
||||
Entries in the pool must be unique, so that attributes can be compared by their IDs. Attribute names cannot contain commas.
|
||||
|
||||
A changeset looks something like the following:
|
||||
|
||||
=Z:5g>1|5=2p=v*4*5+1$x=
|
||||
|
||||
With the corresponding pool containing these entries:
|
||||
...
|
||||
4 -> (author,1059348573)
|
||||
5 -> (bold,true)
|
||||
...
|
||||
This changeset, together with the pool, represents inserting
|
||||
a bold letter "x" into the middle of a line. The string consists of:
|
||||
|
||||
- a letter Z (the "magic character" and format version identifier)
|
||||
- a series of opcodes (punctuation) and numeric values in base 36 (the
|
||||
alphanumerics)
|
||||
- a dollar sign ($)
|
||||
- a string of characters used by insertion operations (the "char bank")
|
||||
|
||||
If we separate out the operations and convert the numbers to base 10, we get:
|
||||
|
||||
=Z :196 >1 |5=97 =31 *4 *5 +1 $"x"=
|
||||
|
||||
Here are descriptions of the operations, where capital letters are variables:
|
||||
|
||||
#+BEGIN_SRC text
|
||||
":N" : Source text has length N (must be first op)
|
||||
">N" : Final text is N (positive) characters longer than source text (must be second op)
|
||||
"<N" : Final text is N (positive) characters shorter than source text (must be second op)
|
||||
">0" : Final text is same length as source text
|
||||
"+N" : Insert N characters from the bank, none of them newlines
|
||||
"-N" : Skip over (delete) N characters from the source text, none of them newlines
|
||||
"=N" : Keep N characters from the source text, none of them newlines
|
||||
"|L+N" : Insert N characters from the source text, containing L newlines. The last
|
||||
character inserted MUST be a newline, but not the (new) document's final newline.
|
||||
"|L-N" : Delete N characters from the source text, containing L newlines. The last
|
||||
character inserted MUST be a newline, but not the (old) document's final newline.
|
||||
"|L=N" : Keep N characters from the source text, containing L newlines. The last character
|
||||
kept MUST be a newline, and the final newline of the document is allowed.
|
||||
"*I" : Apply attribute I from the pool to the following +, =, |+, or |= command.
|
||||
In other words, any number of * ops can come before a +, =, or | but not
|
||||
between a | and the corresponding + or =.
|
||||
If +, text is inserted having this attribute. If =, text is kept but with
|
||||
the attribute applied as an attribute addition or removal.
|
||||
Consecutive attributes must be sorted lexically by (key,value) with key
|
||||
and value taken as strings. It's illegal to have duplicate keys
|
||||
for (key,value) pairs that apply to the same text. It's illegal to
|
||||
have an empty value for a key in the case of an insertion (+), the
|
||||
pair should just be omitted.
|
||||
#+END_SRC
|
||||
|
||||
Characters from the source text that aren't accounted for are assumed to be kept with the same attributes.
|
||||
|
||||
*Additional Constraints:*
|
||||
|
||||
- Consecutive +, -, and = ops of the same type that could be combined are not allowed. Whether combination is possible depends on the attributes of the ops and whether each is multiline or not. For example, two multiline deletions can never be consecutive, nor can any insertion come after a non-multiline insertion with the same attributes.
|
||||
|
||||
- "No-op" ops are not allowed, such as deleting 0 characters. However, attribute applications that don't have any effect are allowed.
|
||||
|
||||
- Characters at the end of the source text cannot be explicitly kept with no changes; if the change doesn't affect the last N characters, those "keep" ops must be left off.
|
||||
|
||||
- In any consecutive sequence of insertions (+) and deletions (-) with no keeps (=), the deletions must come before the insertions.
|
||||
|
||||
- The document text before and after will always end with a newline. This policy avoids a lot of special-casing of the end of the document. If a final newline is always added when importing text and removed when exporting text, then the changeset representation can be used to process text files that may or may not have a final newline.
|
||||
|
||||
*Attribution string:*
|
||||
|
||||
An "attribution string" is a series of inserts with no deletions or keeps. For example, "*3+8|1+5" describes the attributes of a string of length 13, where the first 8 chars have attribute 3 and the next 5 chars have no attributes, with the last of these 5 chars being a newline. Constraints apply similar to those affecting changesets, but the restriction about the final newline of the new document being added doesn't apply.
|
||||
|
||||
Attributes in an attribution string cannot be empty, like "(bold,)", they should instead be absent.
|
||||
|
||||
|
||||
** attributes, colours, authors, etc
|
||||
|
||||
the “apool”
|
||||
#+BEGIN_SRC text
|
||||
"apool":{"numToAttrib":{"0":["author","a.touCZaixjPgKDSiN"]},"nextNum":1}
|
||||
#+END_SRC
|
||||
|
||||
author ids, names & colour mapping
|
||||
|
||||
** CLIENT_VARS
|
||||
|
||||
#+BEGIN_SRC text
|
||||
42["message",{"type":"CLIENT_VARS","data":{… [etc]
|
||||
#+END_SRC
|
||||
|
||||
most directly useful
|
||||
- pad name - =[1]["data"]["padId"]= (and also =[1]["data"]["collab_client_vars"]["padId"]=)
|
||||
- revision - =[1]["data"]["collab_client_vars"]["rev"]=
|
||||
- pad text - =[1]["data"]["collab_client_vars"]["initialAttributedText"]["text"]=
|
||||
- text attributes (as changset )- =[1]["data"]["collab_client_vars"]["initialAttributedText"]["attribs"]=
|
||||
|
||||
authors
|
||||
- author list - =[1]["data"]["collab_client_vars"]["historicalAuthorData"]=
|
||||
- e.g. ="a.ltSpoKLpHyziPkDn": {"name": "someone", "colorId": 46)}=
|
||||
|
||||
colo[u]rs
|
||||
- array of hex values - =[1]["data"]["colorPalette"]=
|
||||
- map authors -> colour - e.g. =[1]["data"]["collab_client_vars"]["historicalAuthorData"]["a.TcyaduN34UmzJIxa"]["colorId"]=
|
||||
|
||||
plugins available
|
||||
- listed in =[1]["data"]["plugins"]=
|
||||
- e.g. =[1]["data"]["plugins"]["plugins"]["ep_comments_page"]=
|
||||
- =["data"]["plugins"]["plugins"]["ep_etherpad-lite"]["package"]["description"]=
|
||||
- =["data"]["plugins"]["plugins"]["ep_etherpad-lite"]["package"]["version"]=
|
||||
|
||||
example/reduced
|
||||
|
||||
#+BEGIN_SRC js
|
||||
[
|
||||
"message",
|
||||
{
|
||||
"type": "CLIENT_VARS",
|
||||
"data": {
|
||||
"skinName": "colibris",
|
||||
"skinVariants": "super-dark-toolbar super-dark-background dark-editor",
|
||||
"randomVersionString": "0ec6de15",
|
||||
"accountPrivs": {
|
||||
"maxRevisions": 100
|
||||
},
|
||||
"automaticReconnectionTimeout": 5,
|
||||
"initialRevisionList": [],
|
||||
"initialOptions": {
|
||||
"guestPolicy": "deny"
|
||||
},
|
||||
"savedRevisions": [],
|
||||
"collab_client_vars": {
|
||||
"initialAttributedText": {
|
||||
"text": "ethereal\n",
|
||||
"attribs": "*0+8|1+1"
|
||||
},
|
||||
"clientIp": "127.0.0.1",
|
||||
"padId": "test2",
|
||||
"historicalAuthorData": {
|
||||
"a.ltSpoKLpHyziPkDn": {
|
||||
"name": null,
|
||||
"colorId": 46
|
||||
},
|
||||
"a.touCZaixjPgKDSiN": {
|
||||
"name": null,
|
||||
"colorId": 7
|
||||
},
|
||||
"a.TcyaduN34UmzJIxa": {
|
||||
"name": null,
|
||||
"colorId": 31
|
||||
}
|
||||
},
|
||||
"apool": {
|
||||
"numToAttrib": {
|
||||
"0": [
|
||||
"author",
|
||||
"a.touCZaixjPgKDSiN"
|
||||
]
|
||||
},
|
||||
"nextNum": 1
|
||||
},
|
||||
"rev": 174,
|
||||
"time": 1607568522484
|
||||
},
|
||||
"colorPalette": [
|
||||
"#ffc7c7",
|
||||
"#fff1c7",
|
||||
"#e3ffc7",
|
||||
"#c7ffd5",
|
||||
"#c7ffff",
|
||||
"#c7d5ff",
|
||||
"#e3c7ff",
|
||||
"#ffc7f1",
|
||||
"#ffa8a8",
|
||||
"#ffe699",
|
||||
"#cfff9e",
|
||||
"#99ffb3",
|
||||
"#a3ffff",
|
||||
"#99b3ff",
|
||||
"#cc99ff",
|
||||
"#ff99e5",
|
||||
"#e7b1b1",
|
||||
"#e9dcAf",
|
||||
"#cde9af",
|
||||
"#bfedcc",
|
||||
"#b1e7e7",
|
||||
"#c3cdee",
|
||||
"#d2b8ea",
|
||||
"#eec3e6",
|
||||
"#e9cece",
|
||||
"#e7e0ca",
|
||||
"#d3e5c7",
|
||||
"#bce1c5",
|
||||
"#c1e2e2",
|
||||
"#c1c9e2",
|
||||
"#cfc1e2",
|
||||
"#e0bdd9",
|
||||
"#baded3",
|
||||
"#a0f8eb",
|
||||
"#b1e7e0",
|
||||
"#c3c8e4",
|
||||
"#cec5e2",
|
||||
"#b1d5e7",
|
||||
"#cda8f0",
|
||||
"#f0f0a8",
|
||||
"#f2f2a6",
|
||||
"#f5a8eb",
|
||||
"#c5f9a9",
|
||||
"#ececbb",
|
||||
"#e7c4bc",
|
||||
"#daf0b2",
|
||||
"#b0a0fd",
|
||||
"#bce2e7",
|
||||
"#cce2bb",
|
||||
"#ec9afe",
|
||||
"#edabbd",
|
||||
"#aeaeea",
|
||||
"#c4e7b1",
|
||||
"#d722bb",
|
||||
"#f3a5e7",
|
||||
"#ffa8a8",
|
||||
"#d8c0c5",
|
||||
"#eaaedd",
|
||||
"#adc6eb",
|
||||
"#bedad1",
|
||||
"#dee9af",
|
||||
"#e9afc2",
|
||||
"#f8d2a0",
|
||||
"#b3b3e6"
|
||||
],
|
||||
"clientIp": "127.0.0.1",
|
||||
"userIsGuest": true,
|
||||
"userColor": 7,
|
||||
"padId": "test2",
|
||||
"padOptions": {
|
||||
"noColors": false,
|
||||
"showControls": true,
|
||||
"showChat": false,
|
||||
"showLineNumbers": false,
|
||||
"useMonospaceFont": false,
|
||||
"userName": true,
|
||||
"userColor": true,
|
||||
"alwaysShowChat": false,
|
||||
"chatAndUsers": false,
|
||||
"ShowComments": true,
|
||||
"lang": "en-gb",
|
||||
"rtl": false
|
||||
},
|
||||
"padShortcutEnabled": {
|
||||
"altF9": true,
|
||||
"altC": true,
|
||||
"cmdShift2": true,
|
||||
"delete": true,
|
||||
"return": true,
|
||||
"esc": true,
|
||||
"cmdS": true,
|
||||
"tab": true,
|
||||
"cmdZ": true,
|
||||
"cmdY": true,
|
||||
"cmdI": true,
|
||||
"cmdB": true,
|
||||
"cmdU": true,
|
||||
"cmd5": true,
|
||||
"cmdShiftL": true,
|
||||
"cmdShiftN": true,
|
||||
"cmdShift1": true,
|
||||
"cmdShiftC": true,
|
||||
"cmdH": true,
|
||||
"ctrlHome": true,
|
||||
"pageUp": true,
|
||||
"pageDown": true
|
||||
},
|
||||
"initialTitle": "Pad: test2"
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
#+END_SRC
|
||||
|
||||
|
||||
* various tools & accessories
|
||||
|
||||
- Firefox/Chrome/Safari -> network/ws/messages/console/log etc
|
||||
- =git clone https://github.com/guyzmo/PyEtherpadLite=
|
||||
- wscat
|
||||
- netcat
|
||||
|
||||
|
||||
* testing & tracing
|
||||
|
||||
#+BEGIN_SRC emacs-lisp
|
||||
(defun ethertest-loop ()
|
||||
(interactive)
|
||||
(with-current-buffer *etherpad-buffer*
|
||||
(let ((server-url "wss://example.org/socket.io/?EIO=3&transport=websocket")
|
||||
(pad "test"))
|
||||
(text-mode)
|
||||
(etherpad-mode)
|
||||
(ethersync-current-socket
|
||||
(websocket-open subtest
|
||||
:on-message #'ethersync-parse-wsframe
|
||||
:on-error (lambda (_websocket type err)
|
||||
(message "ws error: %s %s" type err))
|
||||
:on-close (lambda (_websocket)
|
||||
(message "websocket closed"))))
|
||||
(let* ((*subtest-socket* (ethersync-current-socket)))
|
||||
(message "protocols: %s" (websocket-negotiated-protocols *subtest-socket*))
|
||||
(message "extensions: %s" (websocket-negotiated-extensions *subtest-socket*))
|
||||
(message "cookies? %s" url-cookie-storage)
|
||||
;; init & keep alive
|
||||
(ethersync-heartbeat-start)
|
||||
(sleep-for 1)
|
||||
;; request data
|
||||
(wss-send (ethersync--request-client-ready pad))
|
||||
;; etcn
|
||||
))))
|
||||
#+END_SRC
|
||||
|
||||
|
||||
|
||||
#+BEGIN_SRC shell
|
||||
❯ wscat -c "wss://example.org/socket.io/?EIO=3&transport=websocket"
|
||||
Connected (press CTRL+C to quit)
|
||||
< 0{"sid":"6_TVij3sJug26KFLAAGc","upgrades":[],"pingInterval":25000,"pingTimeout":5000}
|
||||
< 40
|
||||
>
|
||||
#+END_SRC
|
||||
BIN
notes/proto-x1.png
Normal file
BIN
notes/proto-x1.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 61 KiB |
BIN
notes/proto-x2.png
Normal file
BIN
notes/proto-x2.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 59 KiB |
Loading…
Reference in a new issue