zzkt/ethermacs
1
0
Fork 0
Code Issues 4 Pull requests Projects Releases Packages Wiki Activity

Compare commits

base: zzkt:endless
Branches Tags
zzkt:endless
zzkt:ethersync
..
compare: zzkt:ethersync
Branches Tags
zzkt:endless
zzkt:ethersync

5 commits
endless ... ethersync

Author SHA1 Message Date
nik gaffney
4717de97ab Astro 2459239 2021-01-24 12:07:33 +01:00
nik gaffney
a3db8b58f5 Argyrodes insectus, The Red Raiders & Estadio El Sardinero (1913) 2020-12-19 23:18:29 +01:00
nik gaffney
51e1555aef Merge branch 'endless' into ethersync 2020-12-19 16:02:10 +01:00
nik gaffney
52f27136ac Destroy nothing; Destroy the most important thing 2020-12-19 15:44:28 +01:00
nik gaffney
97bfa6961e What else is this like? 
Destroy nothing; Destroy the most important thing
 2020-12-19 15:42:15 +01:00
6 changed files with 527 additions and 683 deletions
Show stats Expand all files Collapse all files

30
.github/workflows/melpazoid-etherpad.yml vendored
View file

@ -1,30 +0,0 @@
# melpazoid <https://github.com/riscy/melpazoid> build checks.
# for etherpad package
name: melpazoid-etherpad
on: [push, pull_request]
jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Set up Python 3.10
uses: actions/setup-python@v4
with:
python-version: '3.10'
- name: Install
run: |
python -m pip install --upgrade pip
sudo apt-get install emacs && emacs --version
git clone https://github.com/riscy/melpazoid.git ~/melpazoid
pip install ~/melpazoid
- name: Run
env:
LOCAL_REPO: ${{ github.workspace }}
# RECIPE is your recipe as written for MELPA:
RECIPE: (etherpad :repo "zzkt/ethermacs" :fetcher github)
# set this to false (or remove it) if the package isn't on MELPA:
EXIST_OK: true
run: echo $GITHUB_REF && make -C ~/melpazoid

542
etherpad-esync.el
View file

@ -1,542 +0,0 @@
;;; etherpad-esync.el --- Etherpad easysync protocol -*- coding: utf-8; lexical-binding: t -*-
;; Copyright 2020 FoAM
;;
;; Author: nik gaffney <nik@fo.am>
;; Created: 2020-12-12
;; Version: 0.1
;; Keywords: comm, etherpad, collaborative editing
;; URL: https://github.com/zzkt/ethermacs
;; This file is not part of GNU Emacs.
;; This program is free software; you can redistribute it and/or modify
;; it under the terms of the GNU General Public License as published by
;; the Free Software Foundation, either version 3 of the License, or
;; (at your option) any later version.
;; This program is distributed in the hope that it will be useful,
;; but WITHOUT ANY WARRANTY; without even the implied warranty of
;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
;; GNU General Public License for more details.
;; You should have received a copy of the GNU General Public License
;; along with this program. If not, see <https://www.gnu.org/licenses/>.
;;; Commentary:
;; Etherpad is a highly customizable Open Source online editor providing
;; collaborative editing in really real-time.
;;
;; The easysync protocol is used for communication of edits, changesets
;; and metadata between etherpad server and clients. It uses websockets
;; for the transport layer
;;
;; details -> https://etherpad.org/doc/v1.8.5/#index_http_api
;; current issues 2020-12-15 00:52:32
;; - one ws per buffer. buffer local. shared.
;; - incorrect newline counts when sending changesets
;; - problems w. deleting text and/or changing buffer size & change-hooks
;; - see "Additional Constraints" of easysync
;; - potential race conditions sending changesets on buffer changes
;; - doesn't apply attributes or changes from the apool
;; - general lack of error checking
;;; Code:
(require 'websocket)
(require 'let-alist)
(require 'calc-bin)
(require 'parsec)
(require '0xc)
(require 's)
;; debug details
;; (setq websocket-debug t)
;; local and buffer local variables
(defvar-local etherpad-esync--pre-buffer-length 0)
(defvar-local etherpad-esync--new-buffer-length 0)
(defvar-local etherpad-esync--change-length 0)
(defvar-local etherpad-esync--change-string "")
(defvar-local etherpad-esync--local-rev 0)
(defvar-local etherpad-esync--current-pad "")
(defvar-local etherpad-esync--local-author "") ;; e.g. "a.touCZaixjPgKDSiN"
(defvar-local etherpad-esync--hearbeat-timer nil)
(defvar-local etherpad-esync--current-socket nil)
;; session
(defvar etherpad-esync-session-token "") ;; see also *session-token*
;; buffering
(defvar etherpad-esync-buffer (generate-new-buffer "*etherpad (easysync)*"))
;; keep-alive message & heartbeat timers
(defun etherpad-esync-heartbeat-send ()
"Send a keep-alive message."
;; only send if there is a current socket
;; and delete an active timer when there isn't a socket open
(message "heartbeat?")
(when (etherpad-esync-current-socket)
(etherpad-esync-wss-send "2")
(message "heartbeat sent: %s" etherpad-esync-buffer))
(when (not (etherpad-esync-current-socket))
(etherpad-esync-heartbeat-stop)
(message "heartbeat stopped: %s" etherpad-esync--hearbeat-timer)))
(defun etherpad-esync-heartbeat-start ()
"Maintain connection to server with periodic pings."
(message "heartbeat started: %s" etherpad-esync-buffer)
(setq etherpad-esync--hearbeat-timer
(run-with-timer 5 15 #'etherpad-esync-heartbeat-send))
(etherpad-esync-current-socket))
(defun etherpad-esync-heartbeat-stop ()
"Stop sending keep-alive messages."
(when etherpad-esync--hearbeat-timer
(cancel-timer etherpad-esync--hearbeat-timer))
(setq etherpad-esync--hearbeat-timer nil)
(message "heartbeat stopped: %s" etherpad-esync-buffer))
;; sockets
(defun etherpad-esync-current-socket (&optional socket)
"Return currently active socket or set SOCKET as current."
(when socket
(setq etherpad-esync--current-socket socket))
(message "current socket: set")
etherpad-esync--current-socket)
;; setters
(defun etherpad-esync--set-local-rev (n)
"Set the local revision to N."
(message "current rev: %s" etherpad-esync--local-rev)
(setq etherpad-esync--local-rev n)
(message "updated rev: %s" etherpad-esync--local-rev)
(with-current-buffer etherpad-esync-buffer
(rename-buffer (format "etherpad:%s:%s"
etherpad-esync--current-pad
etherpad-esync--local-rev))))
;; see also -> inhibit-modification-hooks
(defun etherpad-esync--add-change-hooks ()
"Add predefined change hooks."
(interactive)
(message "setting up buffer change hooks")
(with-current-buffer etherpad-esync-buffer
(add-hook 'before-change-functions
#'etherpad-esync--before-buffer-changes nil t)
;; ordering is important...
(add-hook 'after-change-functions
#'etherpad-esync--after-buffer-changes 22 t)
(add-hook 'after-change-functions
#'etherpad-esync--send-changes 23 t)))
(defun etherpad-esync--remove-change-hooks ()
"Remove predefined change hooks."
(interactive)
(message "removing buffer change hooks")
(with-current-buffer etherpad-esync-buffer
(remove-hook 'before-change-functions
#'etherpad-esync--before-buffer-changes t)
(remove-hook 'after-change-functions
#'etherpad-esync--after-buffer-changes t)
(remove-hook 'after-change-functions
#'etherpad-esync--send-changes t)))
(defun etherpad-esync--before-buffer-changes (begin end)
"Length before buffer is synced. BEGIN END."
;; (message "before -> b:%s e:%s" begin end)
(setq-local etherpad-esync--pre-buffer-length (length (buffer-string))
etherpad-esync--change-length (- end begin)))
(defun etherpad-esync--after-buffer-changes (begin end _pre)
"Length after buffer is synced. BEGIN END."
;; (message "after -> b:%s e:%s p:%s" begin end pre)
(setq-local etherpad-esync--new-buffer-length (length (buffer-string))
etherpad-esync--change-string (buffer-substring begin end)
etherpad-esync--change-length (- end begin)))
;; (message "eabc: pre: %s post: %s changed: %s chars to: %s at: %s(%s)"
;; etherpad-esync--pre-buffer-length
;; etherpad-esync--new-buffer-length
;; etherpad-esync--change-length
;; etherpad-esync--change-string
;; (point)
;; (n-36 (point))))
;; emacs -> etherpad changes
(defun etherpad-esync--send-changes (_b _e _p)
"Create and encode a changeset."
(let* ((b0 etherpad-esync--pre-buffer-length)
(b1 etherpad-esync--new-buffer-length)
(ops (if (< b0 b1)
(format "+%s" (- b1 b0))
(format "-%s" (- b0 b1))))
(changeset
(etherpad-esync--encode-changeset
b0
(- b1 b0)
ops
etherpad-esync--change-string)))
(message "changeset: %s" changeset)
(etherpad-esync--send-user-changes changeset)))
(defun etherpad-esync--encode-changeset (length change-size ops chars)
"Create a changeset from some buffer activity. LENGTH CHANGE-SIZE OPS CHARS."
(cl-labels ((n-36 (n)
(let ((calc-number-radix 36))
(downcase (math-format-radix n)))))
(message "encoding: o:%s (%s) cs:%s op:%s ch:%s"
length (n-36 length)
change-size ops chars)
(let* ((change (cond ((= 0 change-size) "=0")
((> 0 change-size)
(format "<%s" (n-36 (abs change-size))))
((< 0 change-size)
(format ">%s" (n-36 change-size)))))
(newline-count (s-count-matches "\n" (buffer-substring
(point-min)
(point))))
(offset (- (point) (length chars) 1))
;; offset is distance from point-min to point w.out inserted chars and w. newlines
(pos-op (if (< 0 newline-count)
(format "|%s=%s"
;; 2 steps reqd. newline insert, then from beginning of line?
(n-36 newline-count)
(let ((p1 (- offset
(caar
(reverse
(s-matched-positions-all
"\n" (buffer-substring
(point-min) (point))))))))
(message "offset: %s p1: %s" offset p1)
(if (= 1 p1)
(n-36 (+ 1 offset))
(format "%s=%s"
(n-36 (- offset p1 -1))
(n-36 (- p1 1))))))
(format "=%s" (n-36 offset)))))
(format "Z:%s%s%s%s$%s" (n-36 length) change pos-op ops chars))))
(defun etherpad-esync--send-user-changes (cs)
"Send a `USER_CHANGES' message with changeset CS."
(let* ((author etherpad-esync--local-author)
(rev etherpad-esync--local-rev)
(changeset cs)
(payload
(format "42[\"message\",{\"type\":\"COLLABROOM\",\"component\":\"pad\",\"data\":{\"type\":\"USER_CHANGES\",\"baseRev\":%s,\"changeset\":\"%s\",\"apool\":{\"numToAttrib\":{},\"nextNum\":1}}}]"
rev changeset))) ;; author?
(message "send this (as %s) -> %s" payload author)
(etherpad-esync-wss-send payload)))
;; parsec info https://github.com/cute-jumper/parsec.el
(defun etherpad-esync-parse-changeset (cs)
"Parse a changeset CS.
:N : Source text has length N (must be first op)
>N : Final text is N (positive) characters longer than source text (must be second op)
<N : Final text is N (positive) characters shorter than source text (must be second op)
>0 : Final text is same length as source text
+N : Insert N characters from the bank, none of them newlines
-N : Skip over (delete) N characters from the source text, none of them newlines
=N : Keep N characters from the source text, none of them newlines
|L+N : Insert N characters from the source text, containing L newlines. The last
character inserted MUST be a newline, but not the (new) document's final newline.
|L-N : Delete N characters from the source text, containing L newlines. The last
character inserted MUST be a newline, but not the (old) document's final newline.
|L=N : Keep N characters from the source text, containing L newlines. The last character
kept MUST be a newline, and the final newline of the document is allowed.
*I : Apply attribute I from the pool to the following +, =, |+, or |= command.
In other words, any number of * ops can come before a +, =, or | but not
between a | and the corresponding + or =.
If +, text is inserted having this attribute. If =, text is kept but with
the attribute applied as an attribute addition or removal.
Consecutive attributes must be sorted lexically by (key,value) with key
and value taken as strings. It's illegal to have duplicate keys
for (key,value) pairs that apply to the same text. It's illegal to
have an empty value for a key in the case of an insertion (+), the
pair should just be omitted."
(let* ((changes
(parsec-with-input
cs
;; a letter Z (the "magic character" and format version identifier)
(parsec-str "Z")
(parsec-collect*
;; source text length
(parsec-re ":[0-9a-z]+")
;; change in text length
(parsec-re "[>=<][0-9a-z]+")
;; insertion & deletion operations
(parsec-many
(parsec-or
(parsec-re "|[0-9a-z]+[+-=][0-9a-z]+")
(parsec-re "[><+-=*][0-9a-z]+")))
;; separator
(parsec-str "$")
;; a string of characters used by insertion operations (the "char bank")
(parsec-many-s
(parsec-any-ch))))))
(let* ((old-length
(0xc-string-to-number (substring (car changes) 1) 36))
(change-sign
(if (s-equals? ">" (substring (nth 1 changes) 0 1)) 1 -1))
(change-size
(0xc-string-to-number (substring (nth 1 changes) 1) 36))
(new-length
(+ old-length (* change-sign change-size)))
(ops
(nth 2 changes))
(chars
(car (last changes))))
(message "old length: %s new length: %s ops: %s" old-length new-length ops)
(list old-length ops chars))))
;; operations -> buffer changes
(defun etherpad-esync-apply-ops (ops chars)
"Apply a series of insert/delete OPS using CHARS.
Numeric offsets are calculated from the beginning of the buffer."
(with-current-buffer etherpad-esync-buffer
(save-mark-and-excursion
(goto-char (point-min))
(cl-flet
;; Convert a base-36 number STRING to decimal.
((s-36 (string)
(0xc-string-to-number string 36)))
(let ((char-bank chars))
(mapcar
(lambda (s)
(let* ((o1 (s-left 1 s))
(p1 (substring s 1)))
(message "op: %s val: %s" o1 p1)
(pcase o1
("+" (etherpad-esync-insert (s-left (s-36 p1) char-bank))
(setq char-bank (s-right (s-36 p1) char-bank)))
("-" (etherpad-esync-delete (s-36 p1)))
("=" (etherpad-esync-keep (s-36 p1)))
("|" (let* ((p2 (s-split "[+=-]" p1))
(l1 (s-36 (car p2)))
(n1 (s-36 (cadr p2))))
;; doesn't insert or delete newlines correctly (yet)
(message "op: | → l1: %s n1: %s" l1 n1)
(pcase p1
((pred (s-matches? "+"))
(etherpad-esync-insert (make-string n1 10)))
((pred (s-matches? "-")) (etherpad-esync-delete n1))
((pred (s-matches? "=")) (etherpad-esync-keep n1)))))
("*" t)
(_ nil))))
ops))))))
;; character operations for remote->local sync
;; which should not trigger change hooks
(defun etherpad-esync-insert (chars)
"Insert CHARS into the source text."
(let ((inhibit-modification-hooks t))
(insert chars)))
(defun etherpad-esync-delete (n)
"Delete (skip over) N chars from the source text."
(let ((inhibit-modification-hooks t))
(delete-char n)))
(defun etherpad-esync-keep (n)
"Keep N chars from the source text."
(let ((inhibit-modification-hooks t))
(forward-char n)))
;; start with current pad text
(defun etherpad-esync-init-text (chars)
"Seeds a buffer with CHARS from a remote pad."
(with-current-buffer etherpad-esync-buffer
(let ((inhibit-modification-hooks t))
(erase-buffer)
(goto-char (point-min))
(insert chars))))
(defun etherpad-esync-try-changeset (cs)
"Try changeset CS."
(let* ((changes
(etherpad-esync-parse-changeset cs))
(len (nth 0 changes))
(ops (nth 1 changes))
(chars (nth 2 changes)))
(etherpad-esync--check-length len)
(etherpad-esync-apply-ops ops chars)))
(defun etherpad-esync--check-length (size)
"Check the changeset and buffer SIZE are consistent."
(when (not (= size (length (buffer-string))))
(message "changeset and buffer length are inconsistent.")))
;; various stanzas
(defun etherpad-esync--request-client-ready (padId)
"Ethersync: send CLIENT_READY for PADID."
(format "42[\"message\",{\"component\":\"pad\",\"type\":\"CLIENT_READY\",\"padId\":\"%s\",\"token\":\"%s\",\"protocolVersion\":2}]" padId etherpad-esync-session-token))
(defun etherpad-esync--request-get-comments (padId)
"Ethersync: request comments on PADID."
(format "42/comment,0[\"getComments\",{\"padId\":\"%s\"}]" padId))
(defun etherpad-esync--request-get-comment-replies (padId)
"Ethersync: request comment replies on PADID."
(format "42/comment,1[\"getCommentReplies\",{\"padId\":\"%s\"}]" padId))
;; sending via websockets
(defun etherpad-esync-wss-send (msg)
"Send MSG to a websocket."
(if (websocket-openp (etherpad-esync-current-socket))
(when (stringp msg)
(websocket-send-text
(etherpad-esync-current-socket) msg))
(message "websocket is closed. not sending: %s" msg)))
;; parsing & dispatch of incoming frames
(defun etherpad-esync-parse-wsframe (_websocket frame)
"Parse & dispatch incoming FRAME.
Parsing occurs `with-current-buffer' for constancy with buffer-local variables
use let bindings for multiple connections."
;; (message "parsing: %s" frame)
(with-current-buffer etherpad-esync-buffer
(let* ((fr0 (websocket-frame-text frame))
(frp (parsec-with-input
fr0
(parsec-collect* (parsec-re "[0-9]+")
(parsec-many-s (parsec-any-ch))))))
(message "frame: %s" (length fr0))
(when (= 2 (length fr0))
(message "frame: %s" fr0))
(pcase (car frp)
("0" (etherpad-esync--parse-0 frp))
("2" (etherpad-esync--parse-2 frp))
("3" (message "3: keep-alive"))
("40" (etherpad-esync--parse-40 frp))
("42" (etherpad-esync--parse-42 frp))))))
;; parse various incoming message types
(defun etherpad-esync--parse-0 (p0)
"Parse messages beginning with 0 from P0.
set sid, upgrades, pingInterval and pingTimeout for session."
(when (listp p0)
(pcase (length p0)
(0 nil)
(1 (car p0))
(_ (let* ((p1 (json-parse-string (nth 1 p0) :object-type 'alist))
(sid (alist-get 'sid p1)))
(message "sid %s" sid))))))
(defun etherpad-esync--parse-2 (p0)
"Parse messages beginning with 2 from P0.
set revisions etc."
(when (listp p0)
(pcase (length p0)
(0 nil)
(1 (car p0))
(_ (let* ((p1 (json-parse-string (nth 1 p0) :object-type 'alist)))
(let-alist (aref p1 1)
(pcase .type
("COLLABROOM"
(pcase .data.type
("ACCEPT_COMMIT"
(message "accepted changes: rev:%s"
.data.newRev)
(etherpad-esync--set-local-rev .data.newRev)))))))))))
(defun etherpad-esync--parse-40 (p0)
"Parse messages beginning with 40 from P0.
comments and comment threads."
(message "40: comments: %s" p0))
(defun etherpad-esync--parse-42 (p0)
"Parse messages beginning with 42 from P0.
most of the COLLABROOM and update stuff..."
(when (listp p0)
(pcase (length p0)
(0 nil)
(1 (car p0))
(_ (let* ((p1 (json-parse-string (nth 1 p0) :object-type 'alist)))
(let-alist (aref p1 1)
(pcase .type
("COLLABROOM"
(pcase .data.type
("USER_NEWINFO"
(message "42: new user %s (color %s)"
.data.userInfo.userId
.data.userInfo.colorId))
("NEW_CHANGES"
(message "42: new_changes rev:%s changeset:%s (by %s)"
.data.newRev
.data.changeset
.data.author)
(etherpad-esync--set-local-rev .data.newRev)
(etherpad-esync-try-changeset .data.changeset))
("USER_CHANGES"
(message "42: user_changes rev:%s changeset:%s (by %s)"
.data.baseRev
.data.changeset
.data.apool.author))
("ACCEPT_COMMIT"
(message "42: accept-commit rev:%s" .data.newRev)
(etherpad-esync--set-local-rev .data.newRev))))
("CLIENT_READY"
(message "42: ready -> %s and %s" .padId .token))
("CLIENT_VARS"
(message "42: client_vars (%s) rev:%s -> %s"
.data.padId
.data.collab_client_vars.rev
.data.collab_client_vars.initialAttributedText.text)
(etherpad-esync--set-local-rev
.data.collab_client_vars.rev)
(etherpad-esync-init-text
.data.collab_client_vars.initialAttributedText.text)))
(pcase .disconnect
("badChangeset"
(message "42: disconnect (%s)" .disconnect)))))))))
(provide 'etherpad-esync)
;;; etherpad-esync.el ends here

23
etherpad.el
View file

@ -5,7 +5,7 @@
;; Author: nik gaffney <nik@fo.am>
;; Created: 2020-08-08
;; Version: 0.1
;; Package-Requires: ((emacs "27.1") (request "0.3") (let-alist "0.0") (websocket "1.12") (parsec "0.1") (0xc "0.1"))
;; Package-Requires: ((emacs "26.1") (request "0.3") (let-alist "0.0"))
;; Keywords: comm, etherpad, collaborative editing
;; URL: https://github.com/zzkt/ethermacs
@ -38,7 +38,7 @@
;; known bugs, limitations, shortcomings, etc
;; - various problems with realtime editing using easysync
;; - doesn't do realtime editing
;; - the server and api key could be buffer local to enable editing on more than one server
;; - doesn't automate API interface generation from openapi.json
;; - not much in the way of error checking or recovery
@ -48,16 +48,10 @@
(add-to-list 'load-path ".")
(require 'etherpad-esync)
(require 'let-alist)
(require 'websocket)
(require 'calc-bin)
(require 'ethersync)
(require 'request)
(require 'cl-lib)
(require 'parsec)
(require '0xc)
(require 's)
(defgroup etherpad nil
"Etherpad edits."
@ -86,15 +80,12 @@
(defvar etherpad--local-pad-revision ""
"Buffer local pad details.")
;; minor mode
(define-minor-mode etherpad-mode
"Minor mode to sync changes with etherpad."
:lighter " etherpad"
:keymap (make-sparse-keymap)
"Minor mode to sync changes with etherpad." nil " etherpad" nil
(if etherpad-mode
(etherpad-esync--add-change-hooks)
(etherpad-esync--remove-change-hooks)))
(ethersync--add-change-hooks)
(ethersync--remove-change-hooks)))
;; API functions

501
ethersync.el Normal file
View file

@ -0,0 +1,501 @@
;;; ethersync.el ---Etherpad easysync protocol -*- coding: utf-8; lexical-binding: t -*-
;; Copyright 2020 FoAM
;;
;; Author: nik gaffney <nik@fo.am>
;; Created: 2020-12-12
;; Version: 0.1
;; Package-Requires: ((emacs "26.1") (request "0.3") (let-alist "0.0"))
;; Keywords: comm, etherpad, collaborative editing
;; URL: https://github.com/zzkt/ethermacs
;; This file is not part of GNU Emacs.
;; This program is free software; you can redistribute it and/or modify
;; it under the terms of the GNU General Public License as published by
;; the Free Software Foundation, either version 3 of the License, or
;; (at your option) any later version.
;; This program is distributed in the hope that it will be useful,
;; but WITHOUT ANY WARRANTY; without even the implied warranty of
;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
;; GNU General Public License for more details.
;; You should have received a copy of the GNU General Public License
;; along with this program. If not, see <https://www.gnu.org/licenses/>.
;;; Commentary:
;; Etherpad is a highly customizable Open Source online editor providing
;; collaborative editing in really real-time.
;;
;; The easysync protocol is used for communication of edits, changesets
;; and metadata between etherpad server and clients. It uses websockets
;; for the transport layer
;;
;; details -> https://etherpad.org/doc/v1.8.5/#index_http_api
;; current issues 2020-12-15 00:52:32
;; - on ws per buffer. buffer local. shared.
;; - incorrect newline counts when sending changesets
;; - problems w. deleting text and/or changing buffer size & change-hooks
;; - see "Additional Constraints" of easysync
;; - potential race conditions sending changesets on buffer changes
;; - doesn't apply attributes or changes from the apool
;; - general lack of error checking
;;; Code:
(require 'websocket)
(require 'let-alist)
(require 'calc-bin)
(require 'parsec)
(require '0xc)
(require 's)
;; debug details
(setq websocket-debug t)
;; local and buffer local variables
(defvar-local ep--pre-buffer-length 0)
(defvar-local ep--new-buffer-length 0)
(defvar-local ep--change-length 0)
(defvar-local ep--change-string "")
(defvar-local ep--local-rev 0)
(defvar-local ep--current-pad "")
(defvar-local ep--local-author "") ;; e.g. "a.touCZaixjPgKDSiN"
(defvar-local ep--hearbeat-timer nil)
(defvar-local ep--current-socket nil)
;; enable/disable keep alive message to the server
(defun ethersync-heartbeat-start ()
"Maintain connection to server with periodic pings."
(message "heartbeat started: %s" (current-buffer))
(setq ep--hearbeat-timer
(run-with-timer 5 15 #'wss-send "2")))
(defun ethersync-heartbeat-stop ()
"Stop sending keep-alive messages."
(cancel-timer ep--hearbeat-timer))
;; buffering
(defvar *etherpad-buffer* (generate-new-buffer "*etherpad*"))
(defun ethersync-current-socket (&optional socket)
"Return currently active socket or set SOCKET as current."
(message "socket in buffer: %s" (current-buffer))
(if socket
(setq ep--current-socket socket)
ep--current-socket))
;; setters
(defun ethersync--set-local-rev (n)
"Set the local revision."
(message "current rev: %s" ep--local-rev)
(setq ep--local-rev n)
(message "updated rev: %s" ep--local-rev)
(with-current-buffer *etherpad-buffer*
(rename-buffer (format "etherpad:%s:%s"
ep--current-pad
ep--local-rev))))
;; see also -> inhibit-modification-hooks
(defun ethersync--add-change-hooks ()
(interactive)
(message "setting up buffer change hooks")
(with-current-buffer *etherpad-buffer*
(add-hook 'before-change-functions
'ethersync--before-buffer-changes nil t)
;; ordering is important...
(add-hook 'after-change-functions
'ethersync--after-buffer-changes 22 t)
(add-hook 'after-change-functions
'ethersync--send-changes 23 t)))
(defun ethersync--remove-change-hooks ()
(interactive)
(message "removing buffer change hooks")
(with-current-buffer *etherpad-buffer*
(remove-hook 'before-change-functions
'ethersync--before-buffer-changes t)
(remove-hook 'after-change-functions
'ethersync--after-buffer-changes t)
(remove-hook 'after-change-functions
'ethersync--send-changes t)))
(defun ethersync--before-buffer-changes (begin end)
;; (message "before -> b:%s e:%s" begin end)
(setq-local ep--pre-buffer-length (length (buffer-string))
ep--change-length (- end begin)))
(defun ethersync--after-buffer-changes (begin end pre)
;; (message "after -> b:%s e:%s p:%s" begin end pre)
(setq-local ep--new-buffer-length (length (buffer-string))
ep--change-string (buffer-substring begin end)
ep--change-length (- end begin)))
;; (message "eabc: pre: %s post: %s changed: %s chars to: %s at: %s(%s)"
;; ep--pre-buffer-length
;; ep--new-buffer-length
;; ep--change-length
;; ep--change-string
;; (point)
;; (n-36 (point))))
;; emacs -> etherpad changes
(defun ethersync--send-changes (_b _e _p)
"Create and encode a changeset."
(let* ((b0 ep--pre-buffer-length)
(b1 ep--new-buffer-length)
(ops (if (< b0 b1)
(format "+%s" (- b1 b0))
(format "-%s" (- b0 b1))))
(changeset
(ethersync--encode-changeset
b0
(- b1 b0)
ops
ep--change-string)))
(message "changeset: %s" changeset)
(ethersync--send-user-changes changeset)))
(defun ethersync--encode-changeset (length change-size ops chars)
"Create a changeset from some buffer activty."
(message "encoding: o:%s (%s) cs:%s op:%s ch:%s" length (n-36 length) change-size ops chars)
(let* ((change (cond ((= 0 change-size) "=0")
((> 0 change-size)
(format "<%s" (n-36 (abs change-size))))
((< 0 change-size)
(format ">%s" (n-36 change-size)))))
(newline-count (s-count-matches "\n" (buffer-substring
(point-min)
(point))))
(offset (- (point) (length chars) 1))
;; offset is distance from point-min to point w.out inserted chars and w. newlines
(pos-op (if (< 0 newline-count)
(format "|%s=%s=%s"
;; 2 steps reqd. newline insert, then from beginning of line?
(n-36 newline-count) (n-36 offset)
(n-36 (caar
(reverse
(s-matched-positions-all
"\n" (buffer-substring
(point-min) (point)))))))
(format "=%s" (n-36 offset)))))
(format "Z:%s%s%s%s$%s" (n-36 length) change pos-op ops chars)))
(defun ethersync--send-user-changes (cs)
"Send a 'USER_CHANGES' message with changeset CS."
(let* ((author ep--local-author)
(rev ep--local-rev)
(changeset cs)
(payload (format "42[\"message\",{\"type\":\"COLLABROOM\",\"component\":\"pad\",\"data\":{\"type\":\"USER_CHANGES\",\"baseRev\":%s,\"changeset\":\"%s\",\"apool\":{\"numToAttrib\":{},\"nextNum\":1}}}]" rev changeset author)))
(message "send this -> %s" payload)
(wss-send payload)
))
;; parsec info https://github.com/cute-jumper/parsec.el
(defun ethersync-parse-changeset (cs)
"Parse a changeset CS.
:N : Source text has length N (must be first op)
>N : Final text is N (positive) characters longer than source text (must be second op)
<N : Final text is N (positive) characters shorter than source text (must be second op)
>0 : Final text is same length as source text
+N : Insert N characters from the bank, none of them newlines
-N : Skip over (delete) N characters from the source text, none of them newlines
=N : Keep N characters from the source text, none of them newlines
|L+N : Insert N characters from the source text, containing L newlines. The last
character inserted MUST be a newline, but not the (new) document's final newline.
|L-N : Delete N characters from the source text, containing L newlines. The last
character inserted MUST be a newline, but not the (old) document's final newline.
|L=N : Keep N characters from the source text, containing L newlines. The last character
kept MUST be a newline, and the final newline of the document is allowed.
*I : Apply attribute I from the pool to the following +, =, |+, or |= command.
In other words, any number of * ops can come before a +, =, or | but not
between a | and the corresponding + or =.
If +, text is inserted having this attribute. If =, text is kept but with
the attribute applied as an attribute addition or removal.
Consecutive attributes must be sorted lexically by (key,value) with key
and value taken as strings. It's illegal to have duplicate keys
for (key,value) pairs that apply to the same text. It's illegal to
have an empty value for a key in the case of an insertion (+), the
pair should just be omitted."
(let* ((changes
(parsec-with-input cs
;; a letter Z (the "magic character" and format version identifier)
(parsec-str "Z")
(parsec-collect*
;; source text length
(parsec-re ":[0-9a-z]+")
;; change in text length
(parsec-re "[>=<][0-9a-z]+")
;; insertion & deletion operations
(parsec-many
(parsec-or
(parsec-re "|[0-9a-z]+[+-=][0-9a-z]+")
(parsec-re "[><+-=*][0-9a-z]+")))
;; separator
(parsec-str "$")
;; a string of characters used by insertion operations (the "char bank")
(parsec-many-s
(parsec-any-ch))))))
(let* ((old-length
(0xc-string-to-number (substring (car changes) 1) 36))
(change-sign
(if (s-equals? ">" (substring (nth 1 changes) 0 1)) 1 -1))
(change-size
(0xc-string-to-number (substring (nth 1 changes) 1) 36))
(new-length
(+ old-length (* change-sign change-size)))
(ops
(nth 2 changes))
(chars
(car (last changes))))
;;(message "old length: %s new length: %s ops: %s chars: %s" old-length new-length ops chars)
(list old-length ops chars)
)))
;; radix reductions
(defun s-36 (string)
"Convert a base-36 number STRING to decimal."
(0xc-string-to-number string 36))
(defun n-36 (number)
"Convert a decimal NUMBER to base-36 as string."
(let ((calc-number-radix 36))
(downcase
(math-format-radix number))))
;; operations -> buffer changes
(defun ethersync-apply-ops (ops chars)
"Apply a series of insert/delete operations.
Numeric offsets are calculated from the beginning of the buffer."
(with-current-buffer *etherpad-buffer*
(save-mark-and-excursion
(goto-char (point-min))
(let ((char-bank chars))
(mapcar
(lambda (s)
(let* ((o1 (s-left 1 s))
(p1 (substring s 1)))
(message "op: %s val: %s" o1 p1)
(pcase o1
("+" (ethersync-insert (s-left (s-36 p1) char-bank))
(setq char-bank (s-right (s-36 p1) char-bank)))
("-" (ethersync-delete (s-36 p1)))
("=" (ethersync-keep (s-36 p1)))
("|" (let* ((p2 (s-split "[+=-]" (s-36 p1)))
(l1 (s-36 (car p2)))
(n1 (s-36 (cadr p2))))
;; doesn't insert or delete newlines correctly (yet)
(message "ops: l1:%s n1:%s" l1 n1)
(pcase p1
((pred (s-matches? "+"))
(ethersync-insert char-bank)
(setq char-bank (s-right n1 char-bank)))
((pred (s-matches? "-")) (ethersync-delete n1))
((pred (s-matches? "=")) (ethersync-keep n1)))))
("*" t)
(_ nil)
)))
ops)))))
;; character operations for remote->local sync
;; which should not trigger change hooks
(defun ethersync-insert (chars)
"Insert CHARS into the source text."
(let ((inhibit-modification-hooks t))
(insert chars)))
(defun ethersync-delete (n)
"Delete (skip over) N chars from the source text."
(let ((inhibit-modification-hooks t))
(delete-char n)))
(defun ethersync-keep (n)
"Keep N chars from the source text."
(let ((inhibit-modification-hooks t))
(forward-char n)))
;; start with current pad text
(defun ethersync-init-text (chars)
"Seeds a buffer with CHARS from a remote pad"
(with-current-buffer *etherpad-buffer*
(let ((inhibit-modification-hooks t))
(erase-buffer)
(goto-char (point-min))
(insert chars))))
(defun ethersync-try-changeset (cs)
(let* ((changes
(ethersync-parse-changeset cs))
(len (nth 0 changes))
(ops (nth 1 changes))
(chars (nth 2 changes)))
(ethersync--check-length len)
(ethersync-apply-ops ops chars)))
(defun ethersync--check-length (size)
"Check the changeset and buffer SIZE are consistent."
(when (not (= size (length (buffer-string))))
(message "changeset and buffer length are inconsistent.")))
;; various stanzas
(defun ethersync--request-client-ready (padId)
"Ethersync: send CLIENT_READY for PADID."
(format "42[\"message\",{\"component\":\"pad\",\"type\":\"CLIENT_READY\",\"padId\":\"%s\",\"token\":\"%s\",\"protocolVersion\":2}]" padId *session-token*))
(defun ethersync--request-get-comments (padId)
"Ethersync: request comments on PADID."
(format "42/comment,0[\"getComments\",{\"padId\":\"%s\"}]" padId))
(defun ethersync--request-get-comment-replies (padId)
"Ethersync: request comment replies on PADID."
(format "42/comment,1[\"getCommentReplies\",{\"padId\":\"%s\"}]" padId))
;; sending via websockets
(defalias 'wss-send #'ethersync-wss-send)
(defun ethersync-wss-send (msg)
"Send MSG to a websocket."
(when (stringp msg)
(websocket-send-text
(ethersync-current-socket) msg)))
;; parsing & dispatch of incoming frames
(defun ethersync-parse-wsframe (_websocket frame)
"Parse & dispatch incoming FRAME.
Parsing occurs with-current-buffer for constancy with buffer-local variables
use let bindings for multiple connections."
;(message "parsing: %s" frame)
(with-current-buffer *etherpad-buffer*
(let* ((fr0 (websocket-frame-text frame))
(frp (parsec-with-input
fr0
(parsec-collect* (parsec-re "[0-9]+")
(parsec-many-s (parsec-any-ch))))))
(message "frame: %s" (length fr0))
(pcase (car frp)
("0" (ethersync--parse-0 frp))
("2" (ethersync--parse-2 frp))
("3" (message "3: keep-alive"))
("42" (ethersync--parse-42 frp))
))))
;; parse various incoming message types
(defun ethersync--parse-0 (p0)
"Parse messages beginning with 0 from P0.
set sid, upgrades, pingInterval and pingTimeout for session."
(when (listp p0)
(pcase (length p0)
(0 nil)
(1 (car p0))
(_ (let* ((p1 (json-parse-string (nth 1 p0) :object-type 'alist))
(sid (alist-get 'sid p1)))
(message "sid %s" sid)
)))))
(defun ethersync--parse-2 (p0)
"Parse messages beginning with 2 from P0.
set revisions etc."
(when (listp p0)
(pcase (length p0)
(0 nil)
(1 (car p0))
(_ (let* ((p1 (json-parse-string (nth 1 p0) :object-type 'alist)))
(let-alist (aref p1 1)
(pcase .type
("COLLABROOM"
(pcase .data.type
("ACCEPT_COMMIT"
(message "accepted changes: rev:%s and %s (by %s)"
.data.newRev)
(ethersync--set-local-rev .data.newRev)))))))))))
(defun ethersync--parse-42 (p0)
"Parse messages beginning with 42 from P0.
most of the COLLABROOM and update stuff..."
(when (listp p0)
(pcase (length p0)
(0 nil)
(1 (car p0))
(_ (let* ((p1 (json-parse-string (nth 1 p0) :object-type 'alist)))
(let-alist (aref p1 1)
(pcase .type
("COLLABROOM"
(pcase .data.type
("USER_NEWINFO"
(message "42: new user %s (color %s)"
.data.userInfo.userId
.data.userInfo.colorId))
("NEW_CHANGES"
(message "42: new_changes rev:%s changeset:%s (by %s)"
.data.newRev
.data.changeset
.data.author)
(ethersync--set-local-rev .data.newRev)
(ethersync-try-changeset .data.changeset))
("USER_CHANGES"
(message "42: user_changes rev:%s changeset:%s (by %s)"
.data.baseRev
.data.changeset
.data.apool.author))
("ACCEPT_COMMIT"
(message "42: accept-commit rev:%s" .data.newRev)
(ethersync--set-local-rev .data.newRev)
)))
("CLIENT_READY"
(message "42: ready -> %s and %s" .padId .token))
("CLIENT_VARS"
(message "42: client_vars (%s) rev:%s -> %s"
.data.padId
.data.collab_client_vars.rev
.data.collab_client_vars.initialAttributedText.text)
(ethersync--set-local-rev
.data.collab_client_vars.rev)
(ethersync-init-text
.data.collab_client_vars.initialAttributedText.text)))
(pcase .disconnect
("badChangeset"
(message "42: disconnect (%s)" .disconnect)))
))))))
(provide 'ethersync)
;;; ethersync.el ends here

114
notes/etherpad-protocol.org
View file

@ -16,7 +16,7 @@
possibly relevant parts of the etherpad code
- https://github.com/ether/etherpad-lite/blob/develop/src/node/handler/PadMessageHandler.js
- https://github.com/payload/ethersync/blob/master/src/ethersync.coffee
- code for a [[https://github.com/JohnMcLear/etherpad-cli-client/blob/master/lib/index.js][cli-client]] and the [[https://github.com/ether/etherpad-lite/tree/develop/src/static/js][javascript client]]
- code for a [[https://github.com/JohnMcLear/etherpad-cli-client/blob/master/lib/index.js][cli-client]] and the [[https://github.com/ether/etherpad-lite/tree/develop/src/static/js][javasctipt client]]
* websockets & socket.io
@ -28,9 +28,7 @@ see https://blog.abrochard.com/websockets.html and [[https://github.com/ahyatt/
* protocol, probes & partials
browser client sends url with pad name to server (e.g. https://etherpad.wikimedia.org/p/test ) establishes session (sid) and receives token (in cookie data). updates, changes & pad metadata are sent via wss connection.
e.g. =wss://etherpad.wikimedia.org/socket.io/?EIO=3&transport=websocket&sid=Ap47gBZD98dHcW38AoqY=
browser client sends url with pad name to server (e.g. https://etherpad.wikimedia.org/p/test ) establishes session (sid) and receives token (in cookie data). updates, changes & pad metadata are sent via wss connection. (e.g. wss://etherpad.wikimedia.org/socket.io/?EIO=3&transport=websocket&sid=Ap47gBZD98dHcW38AoqY )
** overview
@ -41,7 +39,7 @@ e.g. =wss://etherpad.wikimedia.org/socket.io/?EIO=3&transport=websocket&sid=Ap47
client -> ep_server: wss://example.org//socket.io/?EIO=3&transport=websocket
ep_server --> client: 0 sid, upgrades, etc
client -> ep_server: 2 CLIENT_READY padId, token, etc
ep_server --> client: 42 CLIENT_VARS pad text, lots of detail about server, colours, authors, etc
ep_server --> client: 42 CLIENT_VARS pad text, lots of junk about server, colours, authors, etc
ep_server --> client: 42 USER_NEWINFO (if other active clients)
== local edits ==
@ -58,11 +56,12 @@ note right: COLLABROOM
== keep-alive ==
client -> ep_server: 2
ep_server --> client: 3
#+END_SRC
#+CAPTION: overview of etherpad/easysync protocol
#+ATTR_ORG: :width 400
#+ATTR_LaTeX: :height 15cm :placement [H]
#+ATTR_LaTeX: :height 15cm :placement [!h]
[[file:proto-x1.png]]
** comment plugin
@ -96,25 +95,24 @@ title comments
#+CAPTION: comments
#+ATTR_ORG: :width 400
#+ATTR_LaTeX: :height 15cm :placement [H]
#+ATTR_LaTeX: :height 15cm :placement [!h]
[[file:proto-x2.png]]
** example messages
*init/request*
#+BEGIN_SRC
#+BEGIN_SRC text
40/comment,
42/comment,0["getComments",{"padId":"test"}]
42/comment,1["getCommentReplies",{"padId":"test"}]
43/comment,0[{"comments":{"c-4U2BW8J2Lp0r68ZL":{"author":"a.0iRJZx7jiOAxVNMP","name":"zzkt","text":"yes","timestamp":1607769834917}}}]
43/comment,0[{"comments":{"c-4U2BW8J2Lp0r68ZL":{"author":"a.0iRJZx7jiOAxVNMP","name":"zzkt","text":"yes","timestamp":1607769834917}}}]
43/comment,1[{"replies":{}}]
#+END_SRC
*updates (new)*
#+BEGIN_SRC
#+BEGIN_SRC text
42/comment,["pushAddCommentReply","c-reply-vMSgWSY4bFhaCCLR",{"commentId":"c-4U2BW8J2Lp0r68ZL","text":"no","changeTo":null,"changeFrom":null,"author":"a.0iRJZx7jiOAxVNMP","name":"zzkt","timestamp":1607770300230,"replyId":"c-reply-vMSgWSY4bFhaCCLR"}]
42/comment,2["getCommentReplies",{"padId":"test"}]
@ -123,15 +121,14 @@ title comments
#+END_SRC
*updates (changes)*
#+BEGIN_SRC
#+BEGIN_SRC text
42/comment,["textCommentUpdated","c-reply-vMSgWSY4bFhaCCLR","not yet"]
#+END_SRC
*updates (deletion)*
#+BEGIN_SRC
#+BEGIN_SRC text
42/comment,["commentDeleted","c-4U2BW8J2Lp0r68ZL"]
42["message",{"type":"COLLABROOM","data":{"type":"NEW_CHANGES","newRev":234,"changeset":"Z:e>0=7*0=4$","apool":{"numToAttrib":{"0":["comment","comment-deleted"]},"attribToNum":{"comment,comment-deleted":0},"nextNum":1},"author":"a.0iRJZx7jiOAxVNMP","currentTime":1607770511397,"timeDelta":null}}]
42["message",{"type":"COLLABROOM","data":{"type":"NEW_CHANGES","newRev":234,"changeset":"Z:e>0=7*0=4$","apool":{"numToAttrib":{"0":["comment","comment-deleted"]},"attribToNum":{"comment,comment-deleted":0},"nextNum":1},"author":"a.0iRJZx7jiOAxVNMP","currentTime":1607770511397,"timeDelta":null}}]
#+END_SRC
@ -168,7 +165,7 @@ If we separate out the operations and convert the numbers to base 10, we get:
Here are descriptions of the operations, where capital letters are variables:
#+BEGIN_SRC
#+BEGIN_SRC text
":N" : Source text has length N (must be first op)
">N" : Final text is N (positive) characters longer than source text (must be second op)
"<N" : Final text is N (positive) characters shorter than source text (must be second op)
@ -218,7 +215,7 @@ Attributes in an attribution string cannot be empty, like "(bold,)", they should
** attributes, colours, authors, etc
the “apool”
#+BEGIN_SRC
#+BEGIN_SRC text
"apool":{"numToAttrib":{"0":["author","a.touCZaixjPgKDSiN"]},"nextNum":1}
#+END_SRC
@ -226,7 +223,7 @@ author ids, names & colour mapping
** CLIENT_VARS
#+BEGIN_SRC
#+BEGIN_SRC text
42["message",{"type":"CLIENT_VARS","data":{… [etc]
#+END_SRC
@ -252,7 +249,7 @@ plugins available
example/reduced
#+BEGIN_SRC
#+BEGIN_SRC js
[
"message",
{
@ -424,7 +421,7 @@ example/reduced
#+end_export
* various tools & accessories
- Firefox/Chrome/Safari → network/ws/messages/console/log etc
- Firefox/Chrome/Safari -> network/ws/messages/console/log etc
- =git clone https://github.com/guyzmo/PyEtherpadLite=
- wscat
- netcat
@ -461,84 +458,11 @@ example/reduced
#+END_SRC
#+BEGIN_SRC
#+BEGIN_SRC shell
❯ wscat -c "wss://example.org/socket.io/?EIO=3&transport=websocket"
Connected (press CTRL+C to quit)
< 0{"sid":"6_TVij3sJug26KFLAAGc","upgrades":[],"pingInterval":25000,"pingTimeout":5000}
< 40
>
#+END_SRC
#+BEGIN_SRC
[
"message",
{
"type": "COLLABROOM",
"data": {
"type": "NEW_CHANGES",
"newRev": 969,
"changeset": "Z:2r>2*0=1=5*1|1+1*1*2*3*4*5+1|1=1*6=1|1=4*7=1|1=4*8=1|1=a*9=1|1=5*0=1|1=f*0=1|1=q*5=1|1=4*6=1|1=4*7=1|1=4*8=1$\n*",
"apool": {
"numToAttrib": {
"0": [
"start",
"1"
],
"1": [
"author",
"a.TA0tvO487Oh304Up"
],
"2": [
"insertorder",
"first"
],
"3": [
"list",
"number1"
],
"4": [
"lmkr",
"1"
],
"5": [
"start",
"2"
],
"6": [
"start",
"3"
],
"7": [
"start",
"4"
],
"8": [
"start",
"5"
],
"9": [
"start",
"6"
]
},
"attribToNum": {
"start,1": 0,
"author,a.TA0tvO487Oh304Up": 1,
"insertorder,first": 2,
"list,number1": 3,
"lmkr,1": 4,
"start,2": 5,
"start,3": 6,
"start,4": 7,
"start,5": 8,
"start,6": 9
},
"nextNum": 10
},
"author": "a.TA0tvO487Oh304Up",
"currentTime": 1638017098574,
"timeDelta": 8923
}
}
]
#+END_SRC

BIN
notes/etherpad-protocol.pdf
View file

Binary file not shown.