From fc7c504e02a29237c0ea55d5d58de80b738b202d Mon Sep 17 00:00:00 2001 From: Ted Unangst Date: Sat, 23 Nov 2019 18:24:09 -0500 Subject: [PATCH 01/55] add cite tag for block quote attributions --- docs/changelog.txt | 2 ++ markitzero.go | 5 +++-- views/style.css | 3 +++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/changelog.txt b/docs/changelog.txt index f894e6f..952980f 100644 --- a/docs/changelog.txt +++ b/docs/changelog.txt @@ -2,6 +2,8 @@ changelog -- next ++ Add cite tag for block quote attributions. + + @media print styles. + Disable overscroll (pull down) refresh. diff --git a/markitzero.go b/markitzero.go index 1e41f7a..2115d32 100644 --- a/markitzero.go +++ b/markitzero.go @@ -28,7 +28,7 @@ var re_bolder = regexp.MustCompile(`(^|\W)\*\*((?s:.*?))\*\*($|\W)`) var re_italicer = regexp.MustCompile(`(^|\W)\*((?s:.*?))\*($|\W)`) var re_bigcoder = regexp.MustCompile("```(.*)\n?((?s:.*?))\n?```\n?") var re_coder = regexp.MustCompile("`([^`]*)`") -var re_quoter = regexp.MustCompile(`(?m:^> (.*)\n?)`) +var re_quoter = regexp.MustCompile(`(?m:^> (.*)(\n-(.*))?\n?)`) var re_link = regexp.MustCompile(`.?.?https?://[^\s"]+[\w/)!]`) var re_zerolink = regexp.MustCompile(`\[([^]]*)\]\(([^)]*\)?)\)`) var re_imgfix = regexp.MustCompile(`]*)>`) @@ -77,7 +77,7 @@ func markitzero(s string) string { s = re_zerolink.ReplaceAllString(s, `$1`) s = re_bolder.ReplaceAllString(s, "$1$2$3") s = re_italicer.ReplaceAllString(s, "$1$2$3") - s = re_quoter.ReplaceAllString(s, "
$1

") + s = re_quoter.ReplaceAllString(s, "

$1
$3

") s = strings.Replace(s, "\n---\n", "

", -1) s = re_lister.ReplaceAllStringFunc(s, func(m string) string { @@ -117,6 +117,7 @@ func markitzero(s string) string { // some final fixups s = strings.Replace(s, "\n", "
", -1) s = strings.Replace(s, "

", "
", -1) + s = strings.Replace(s, "
", "", -1) s = strings.Replace(s, "
", "
", -1)
 	s = strings.Replace(s, "
    ", "
      ", -1)
 	s = strings.Replace(s, "

      ", "
      ", -1)
diff --git a/views/style.css b/views/style.css
index ea0ba4a..89f0194 100644
--- a/views/style.css
+++ b/views/style.css
@@ -23,6 +23,9 @@ blockquote {
 	padding-left: 0.5em;
 	border-left: 1px solid var(--fg-subtle);
 }
+blockquote cite {
+	margin-left: 2em;
+}
 table {
 	display: block;
 	max-width: 100%;

From cad5dc4387fb225cb81567a0835c11b56a8b6d6f Mon Sep 17 00:00:00 2001
From: Ted Unangst 
Date: Sun, 24 Nov 2019 22:54:55 -0500
Subject: [PATCH 02/55] auto fixup twitter citations

---
 markitzero.go | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/markitzero.go b/markitzero.go
index 2115d32..464de15 100644
--- a/markitzero.go
+++ b/markitzero.go
@@ -28,7 +28,8 @@ var re_bolder = regexp.MustCompile(`(^|\W)\*\*((?s:.*?))\*\*($|\W)`)
 var re_italicer = regexp.MustCompile(`(^|\W)\*((?s:.*?))\*($|\W)`)
 var re_bigcoder = regexp.MustCompile("```(.*)\n?((?s:.*?))\n?```\n?")
 var re_coder = regexp.MustCompile("`([^`]*)`")
-var re_quoter = regexp.MustCompile(`(?m:^> (.*)(\n-(.*))?\n?)`)
+var re_quoter = regexp.MustCompile(`(?m:^> (.*)(\n- ?(.*))?\n?)`)
+var re_reciter = regexp.MustCompile(`()https://twitter.com/([^/]+)/.*?()`)
 var re_link = regexp.MustCompile(`.?.?https?://[^\s"]+[\w/)!]`)
 var re_zerolink = regexp.MustCompile(`\[([^]]*)\]\(([^)]*\)?)\)`)
 var re_imgfix = regexp.MustCompile(`]*)>`)
@@ -78,6 +79,7 @@ func markitzero(s string) string {
 	s = re_bolder.ReplaceAllString(s, "$1$2$3")
 	s = re_italicer.ReplaceAllString(s, "$1$2$3")
 	s = re_quoter.ReplaceAllString(s, "
      $1
      $3
      ")
+	s = re_reciter.ReplaceAllString(s, "$1$2$3")
 	s = strings.Replace(s, "\n---\n", "
      ", -1)
 
 	s = re_lister.ReplaceAllStringFunc(s, func(m string) string {

From 9539ce5df6d1af017b4cdcd8c13b59237369abbc Mon Sep 17 00:00:00 2001
From: Ted Unangst 
Date: Sun, 24 Nov 2019 23:20:33 -0500
Subject: [PATCH 03/55] more precise filter matching

---
 hfcs.go | 36 +++++++++++++++++++++++++-----------
 1 file changed, 25 insertions(+), 11 deletions(-)

diff --git a/hfcs.go b/hfcs.go
index a34d82d..42abd1f 100644
--- a/hfcs.go
+++ b/hfcs.go
@@ -214,21 +214,29 @@ func stealthmode(userid int64, r *http.Request) bool {
 }
 
 func matchfilter(h *Honk, f *Filter) bool {
+	return matchfilterX(h, f) != ""
+}
+
+func matchfilterX(h *Honk, f *Filter) string {
+	rv := ""
 	match := true
 	if match && f.Actor != "" {
 		match = false
 		if f.Actor == h.Honker || f.Actor == h.Oonker {
 			match = true
+			rv = f.Actor
 		}
 		if !match && (f.Actor == originate(h.Honker) ||
 			f.Actor == originate(h.Oonker) ||
 			f.Actor == originate(h.XID)) {
 			match = true
+			rv = f.Actor
 		}
 		if !match && f.IncludeAudience {
 			for _, a := range h.Audience {
 				if f.Actor == a || f.Actor == originate(a) {
 					match = true
+					rv = f.Actor
 					break
 				}
 			}
@@ -239,23 +247,33 @@ func matchfilter(h *Honk, f *Filter) bool {
 		if (f.AnnounceOf == "" && h.Oonker != "") || f.AnnounceOf == h.Oonker ||
 			f.AnnounceOf == originate(h.Oonker) {
 			match = true
+			rv += " announce"
 		}
 	}
 	if match && f.Text != "" {
 		match = false
 		re := f.re_text
-		if re.MatchString(h.Noise) || re.MatchString(h.Precis) {
-			match = true
+		m := re.FindString(h.Precis)
+		if m == "" {
+			m = re.FindString(h.Noise)
 		}
-		if !match {
+		if m == "" {
 			for _, d := range h.Donks {
-				if re.MatchString(d.Desc) {
-					match = true
+				m = re.FindString(d.Desc)
+				if m != "" {
+					break
 				}
 			}
 		}
+		if m != "" {
+			match = true
+			rv = m
+		}
 	}
-	return match
+	if match {
+		return rv
+	}
+	return ""
 }
 
 func rejectxonk(xonk *Honk) bool {
@@ -282,11 +300,7 @@ func skipMedia(xonk *Honk) bool {
 func unsee(userid int64, h *Honk) {
 	filts := getfilters(userid, filtCollapse)
 	for _, f := range filts {
-		if matchfilter(h, f) {
-			bad := f.Text
-			if f.Actor != "" {
-				bad = f.Actor
-			}
+		if bad := matchfilterX(h, f); bad != "" {
 			if h.Precis == "" {
 				h.Precis = bad
 			}

From c9aba7ecba955ea52be1e7130ac3860cce309e9e Mon Sep 17 00:00:00 2001
From: Ted Unangst 
Date: Mon, 25 Nov 2019 15:50:17 -0500
Subject: [PATCH 04/55] save timestamp for xonker additions

---
 activity.go  | 12 ++++++++----
 database.go  |  4 ++--
 schema.go    |  2 +-
 schema.sql   |  2 +-
 upgradedb.go |  7 ++++++-
 5 files changed, 18 insertions(+), 9 deletions(-)

diff --git a/activity.go b/activity.go
index c152eaa..0ed6f02 100644
--- a/activity.go
+++ b/activity.go
@@ -1317,7 +1317,8 @@ var handfull = cache.New(cache.Options{Filler: func(name string) (string, bool)
 		rel, _ := l.GetString("rel")
 		t, _ := l.GetString("type")
 		if rel == "self" && friendorfoe(t) {
-			_, err := stmtSaveXonker.Exec(name, href, "fishname")
+			when := time.Now().UTC().Format(dbtimeformat)
+			_, err := stmtSaveXonker.Exec(name, href, "fishname", when)
 			if err != nil {
 				log.Printf("error saving fishname: %s", err)
 			}
@@ -1430,7 +1431,8 @@ func ingestpubkey(origin string, obj junk.Junk) {
 		log.Printf("error decoding %s pubkey: %s", keyname, err)
 		return
 	}
-	_, err = stmtSaveXonker.Exec(keyname, data, "pubkey")
+	when := time.Now().UTC().Format(dbtimeformat)
+	_, err = stmtSaveXonker.Exec(keyname, data, "pubkey", when)
 	if err != nil {
 		log.Printf("error saving key: %s", err)
 	}
@@ -1455,8 +1457,9 @@ func ingestboxes(origin string, obj junk.Junk) {
 	outbox, _ := obj.GetString("outbox")
 	sbox, _ := obj.GetString("endpoints", "sharedInbox")
 	if inbox != "" {
+		when := time.Now().UTC().Format(dbtimeformat)
 		m := strings.Join([]string{inbox, outbox, sbox}, " ")
-		_, err = stmtSaveXonker.Exec(ident, m, "boxes")
+		_, err = stmtSaveXonker.Exec(ident, m, "boxes", when)
 		if err != nil {
 			log.Printf("error saving boxes: %s", err)
 		}
@@ -1479,7 +1482,8 @@ func ingesthandle(origin string, obj junk.Junk) {
 	}
 	handle, _ = obj.GetString("preferredUsername")
 	if handle != "" {
-		_, err = stmtSaveXonker.Exec(xid, handle, "handle")
+		when := time.Now().UTC().Format(dbtimeformat)
+		_, err = stmtSaveXonker.Exec(xid, handle, "handle", when)
 		if err != nil {
 			log.Printf("error saving handle: %s", err)
 		}
diff --git a/database.go b/database.go
index 2402a46..1af843c 100644
--- a/database.go
+++ b/database.go
@@ -760,8 +760,8 @@ func prepareStatements(db *sql.DB) {
 	stmtGetZonkers = preparetodie(db, "select zonkerid, name, wherefore from zonkers where userid = ? and wherefore <> 'zonk'")
 	stmtSaveZonker = preparetodie(db, "insert into zonkers (userid, name, wherefore) values (?, ?, ?)")
 	stmtGetXonker = preparetodie(db, "select info from xonkers where name = ? and flavor = ?")
-	stmtSaveXonker = preparetodie(db, "insert into xonkers (name, info, flavor) values (?, ?, ?)")
-	stmtDeleteXonker = preparetodie(db, "delete from xonkers where name = ? and flavor = ?")
+	stmtSaveXonker = preparetodie(db, "insert into xonkers (name, info, flavor, dt) values (?, ?, ?, ?)")
+	stmtDeleteXonker = preparetodie(db, "delete from xonkers where name = ? and flavor = ? and dt < ?")
 	stmtRecentHonkers = preparetodie(db, "select distinct(honker) from honks where userid = ? and honker not in (select xid from honkers where userid = ? and flavor = 'sub') order by honkid desc limit 100")
 	stmtUpdateFlags = preparetodie(db, "update honks set flags = flags | ? where honkid = ?")
 	stmtClearFlags = preparetodie(db, "update honks set flags = flags & ~ ? where honkid = ?")
diff --git a/schema.go b/schema.go
index a78fb24..82464e7 100644
--- a/schema.go
+++ b/schema.go
@@ -6,7 +6,7 @@ create table honks (honkid integer primary key, userid integer, what text, honke
 create table donks (honkid integer, fileid integer);
 create table filemeta (fileid integer primary key, xid text, name text, description text, url text, media text, local integer);
 create table honkers (honkerid integer primary key, userid integer, name text, xid text, flavor text, combos text, owner text);
-create table xonkers (xonkerid integer primary key, name text, info text, flavor text);
+create table xonkers (xonkerid integer primary key, name text, info text, flavor text, dt text);
 create table zonkers (zonkerid integer primary key, userid integer, name text, wherefore text);
 create table doovers(dooverid integer primary key, dt text, tries integer, userid integer, rcpt text, msg blob);
 create table onts (ontology text, honkid integer);
diff --git a/schema.sql b/schema.sql
index 0587f73..db02483 100644
--- a/schema.sql
+++ b/schema.sql
@@ -3,7 +3,7 @@ create table honks (honkid integer primary key, userid integer, what text, honke
 create table donks (honkid integer, fileid integer);
 create table filemeta (fileid integer primary key, xid text, name text, description text, url text, media text, local integer);
 create table honkers (honkerid integer primary key, userid integer, name text, xid text, flavor text, combos text, owner text);
-create table xonkers (xonkerid integer primary key, name text, info text, flavor text);
+create table xonkers (xonkerid integer primary key, name text, info text, flavor text, dt text);
 create table zonkers (zonkerid integer primary key, userid integer, name text, wherefore text);
 create table doovers(dooverid integer primary key, dt text, tries integer, userid integer, rcpt text, msg blob);
 create table onts (ontology text, honkid integer);
diff --git a/upgradedb.go b/upgradedb.go
index 828349a..3c968b9 100644
--- a/upgradedb.go
+++ b/upgradedb.go
@@ -24,7 +24,7 @@ import (
 	"time"
 )
 
-var myVersion = 32
+var myVersion = 33
 
 func doordie(db *sql.DB, s string, args ...interface{}) {
 	_, err := db.Exec(s, args...)
@@ -356,6 +356,11 @@ func upgradedb() {
 		doordie(db, "update config set value = 32 where key = 'dbversion'")
 		fallthrough
 	case 32:
+		doordie(db, "alter table xonkers add column dt text")
+		doordie(db, "update xonkers set dt = ?", time.Now().UTC().Format(dbtimeformat))
+		doordie(db, "update config set value = 33 where key = 'dbversion'")
+		fallthrough
+	case 33:
 
 	default:
 		log.Fatalf("can't upgrade unknown version %d", dbversion)

From d167f264187c2256d2e3a7a4e7d0c071dab0a9ae Mon Sep 17 00:00:00 2001
From: Ted Unangst 
Date: Mon, 25 Nov 2019 15:54:29 -0500
Subject: [PATCH 05/55] try a little harder to recover from httpsig failures

---
 docs/changelog.txt | 2 ++
 fun.go             | 7 +++++++
 web.go             | 8 ++++++++
 3 files changed, 17 insertions(+)

diff --git a/docs/changelog.txt b/docs/changelog.txt
index 952980f..1f15a67 100644
--- a/docs/changelog.txt
+++ b/docs/changelog.txt
@@ -2,6 +2,8 @@ changelog
 
 -- next
 
++ Try a little harder to recover from httpsig failures.
+
 + Add cite tag for block quote attributions.
 
 + @media print styles.
diff --git a/fun.go b/fun.go
index a2371ea..86f89b0 100644
--- a/fun.go
+++ b/fun.go
@@ -28,6 +28,7 @@ import (
 	"os"
 	"regexp"
 	"strings"
+	"time"
 
 	"golang.org/x/net/html"
 	"humungus.tedunangst.com/r/webs/cache"
@@ -613,6 +614,12 @@ func zaggy(keyname string) *rsa.PublicKey {
 	return key
 }
 
+func savingthrow(keyname string) {
+	when := time.Now().UTC().Sub(30 * time.Minute).Format(dbtimeformat)
+	stmtDeleteXonker.Exec(keyname, "pubkey", when)
+	zaggies.Clear(keyname)
+}
+
 func keymatch(keyname string, actor string) string {
 	hash := strings.IndexByte(keyname, '#')
 	if hash == -1 {
diff --git a/web.go b/web.go
index 950623d..62c07b1 100644
--- a/web.go
+++ b/web.go
@@ -330,6 +330,10 @@ func inbox(w http.ResponseWriter, r *http.Request) {
 	}
 
 	keyname, err := httpsig.VerifyRequest(r, payload, zaggy)
+	if err != nil && keyname != "" {
+		savingthrow(keyname)
+		keyname, err = httpsig.VerifyRequest(r, payload, zaggy)
+	}
 	if err != nil {
 		log.Printf("inbox message failed signature for %s from %s", keyname, r.Header.Get("X-Forwarded-For"))
 		if keyname != "" {
@@ -460,6 +464,10 @@ func serverinbox(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	keyname, err := httpsig.VerifyRequest(r, payload, zaggy)
+	if err != nil && keyname != "" {
+		savingthrow(keyname)
+		keyname, err = httpsig.VerifyRequest(r, payload, zaggy)
+	}
 	if err != nil {
 		log.Printf("inbox message failed signature for %s from %s", keyname, r.Header.Get("X-Forwarded-For"))
 		if keyname != "" {

From 83c89df14ba87f60b2e1be99e1d3281155cc96d7 Mon Sep 17 00:00:00 2001
From: Ted Unangst 
Date: Mon, 25 Nov 2019 15:55:30 -0500
Subject: [PATCH 06/55] i thought i compiled this

---
 fun.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fun.go b/fun.go
index 86f89b0..34a3c8e 100644
--- a/fun.go
+++ b/fun.go
@@ -615,7 +615,7 @@ func zaggy(keyname string) *rsa.PublicKey {
 }
 
 func savingthrow(keyname string) {
-	when := time.Now().UTC().Sub(30 * time.Minute).Format(dbtimeformat)
+	when := time.Now().UTC().Add(-30 * time.Minute).Format(dbtimeformat)
 	stmtDeleteXonker.Exec(keyname, "pubkey", when)
 	zaggies.Clear(keyname)
 }

From 04a82a00987b44f2c4dc7e7a6d41f32aca39afaf Mon Sep 17 00:00:00 2001
From: Ted Unangst 
Date: Mon, 25 Nov 2019 21:34:57 -0500
Subject: [PATCH 07/55] setting avatar. if you must.

---
 docs/changelog.txt |  2 ++
 docs/honk.1        |  6 ++++++
 views/account.html |  2 +-
 web.go             | 18 ++++++++++++++++++
 4 files changed, 27 insertions(+), 1 deletion(-)

diff --git a/docs/changelog.txt b/docs/changelog.txt
index 1f15a67..f1a96a4 100644
--- a/docs/changelog.txt
+++ b/docs/changelog.txt
@@ -2,6 +2,8 @@ changelog
 
 -- next
 
++ Set an avatar. If you must.
+
 + Try a little harder to recover from httpsig failures.
 
 + Add cite tag for block quote attributions.
diff --git a/docs/honk.1 b/docs/honk.1
index 299802d..d9e679d 100644
--- a/docs/honk.1
+++ b/docs/honk.1
@@ -164,6 +164,12 @@ It also allows the import of external objects via URL, either individual
 posts or actor URLs, in which case their recent outbox is imported.
 .Ss Account
 It's all about you.
+An avatar may be selected from the
+.Pa funzone
+by adding
+.Dq avatar: filename.png
+to one's profile info.
+If truly necessary.
 .Pp
 Some options to customize the site appearance:
 .Bl -tag -width skinny
diff --git a/views/account.html b/views/account.html
index c6425eb..dc2f957 100644
--- a/views/account.html
+++ b/views/account.html
@@ -7,7 +7,7 @@
 
      
 
 
      about me:
-
      
+
      
 
      
 
 
      
diff --git a/web.go b/web.go
index 62c07b1..8d1c6d0 100644
--- a/web.go
+++ b/web.go
@@ -1069,6 +1069,8 @@ func honkpage(w http.ResponseWriter, u *login.UserInfo, honks []*Honk, templinfo
 	}
 }
 
+var re_avatar = regexp.MustCompile("avatar: ?([[:alnum:]_.-]+)")
+
 func saveuser(w http.ResponseWriter, r *http.Request) {
 	whatabout := r.FormValue("whatabout")
 	u := login.GetUserInfo(r)
@@ -1085,6 +1087,17 @@ func saveuser(w http.ResponseWriter, r *http.Request) {
 	} else {
 		options.MapLink = ""
 	}
+	if ava := re_avatar.FindString(whatabout); ava != "" {
+		whatabout = re_avatar.ReplaceAllString(whatabout, "")
+		ava = ava[7:]
+		if ava[0] == ' ' {
+			ava = ava[1:]
+		}
+		options.Avatar = fmt.Sprintf("https://%s/meme/%s", serverName, ava)
+	} else {
+		options.Avatar = ""
+	}
+	whatabout = strings.TrimSpace(whatabout)
 	j, err := jsonify(options)
 	if err == nil {
 		_, err = db.Exec("update users set about = ?, options = ? where username = ?", whatabout, j, u.Username)
@@ -1861,6 +1874,11 @@ func accountpage(w http.ResponseWriter, r *http.Request) {
 	templinfo["UserCSRF"] = login.GetCSRF("saveuser", r)
 	templinfo["LogoutCSRF"] = login.GetCSRF("logout", r)
 	templinfo["User"] = user
+	about := user.About
+	if ava := user.Options.Avatar; ava != "" {
+		about += "\n\navatar: " + ava[strings.LastIndexByte(ava, '/')+1:]
+	}
+	templinfo["WhatAbout"] = about
 	err := readviews.Execute(w, "account.html", templinfo)
 	if err != nil {
 		log.Print(err)

From 0535e0af36a90d3c18efe1360d3fc0f786fd8a1d Mon Sep 17 00:00:00 2001
From: Ted Unangst 
Date: Mon, 25 Nov 2019 21:50:51 -0500
Subject: [PATCH 08/55] don't try shrinking too many images at once

---
 backend.go | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/backend.go b/backend.go
index 5d61147..ca7ba1e 100644
--- a/backend.go
+++ b/backend.go
@@ -23,6 +23,7 @@ import (
 	"os"
 	"os/exec"
 
+	"humungus.tedunangst.com/r/webs/gate"
 	"humungus.tedunangst.com/r/webs/image"
 )
 
@@ -38,7 +39,11 @@ type ShrinkerResult struct {
 	Image *image.Image
 }
 
+var shrinkgate = gate.NewLimiter(4)
+
 func (s *Shrinker) Shrink(args *ShrinkerArgs, res *ShrinkerResult) error {
+	shrinkgate.Start()
+	defer shrinkgate.Finish()
 	img, err := image.Vacuum(bytes.NewReader(args.Buf), args.Params)
 	if err != nil {
 		return err

From c871ba9468c225a5c95373fa7a399dee7d8ab1d0 Mon Sep 17 00:00:00 2001
From: Ted Unangst 
Date: Mon, 25 Nov 2019 22:00:33 -0500
Subject: [PATCH 09/55] don't make two attachment requests concurrently either

---
 activity.go | 33 ++++++++++++++++++++++-----------
 1 file changed, 22 insertions(+), 11 deletions(-)

diff --git a/activity.go b/activity.go
index 0ed6f02..e07c195 100644
--- a/activity.go
+++ b/activity.go
@@ -19,6 +19,7 @@ import (
 	"bytes"
 	"crypto/rsa"
 	"database/sql"
+	"errors"
 	"fmt"
 	"html"
 	"io"
@@ -141,6 +142,22 @@ func GetJunkTimeout(url string, timeout time.Duration) (junk.Junk, error) {
 	return j, nil
 }
 
+func fetchsome(url string) ([]byte, error) {
+	resp, err := http.Get(url)
+	if err != nil {
+		log.Printf("error fetching %s: %s", url, err)
+		return nil, err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != 200 {
+		return nil, errors.New("not 200")
+	}
+	var buf bytes.Buffer
+	limiter := io.LimitReader(resp.Body, 10*1024*1024)
+	io.Copy(&buf, limiter)
+	return buf.Bytes(), nil
+}
+
 func savedonk(url string, name, desc, media string, localize bool) *Donk {
 	if url == "" {
 		return nil
@@ -154,22 +171,16 @@ func savedonk(url string, name, desc, media string, localize bool) *Donk {
 	xid := xfiltrate()
 	data := []byte{}
 	if localize {
-		resp, err := http.Get(url)
+		fn := func() (interface{}, error) {
+			return fetchsome(url)
+		}
+		ii, err := flightdeck.Call(url, fn)
 		if err != nil {
-			log.Printf("error fetching %s: %s", url, err)
 			localize = false
 			goto saveit
 		}
-		defer resp.Body.Close()
-		if resp.StatusCode != 200 {
-			localize = false
-			goto saveit
-		}
-		var buf bytes.Buffer
-		limiter := io.LimitReader(resp.Body, 10*1024*1024)
-		io.Copy(&buf, limiter)
+		data = ii.([]byte)
 
-		data = buf.Bytes()
 		if len(data) == 10*1024*1024 {
 			log.Printf("truncation likely")
 		}

From 577eb7068d66fcfec87972573af080d8ebf3bdf4 Mon Sep 17 00:00:00 2001
From: Ted Unangst 
Date: Mon, 25 Nov 2019 22:14:27 -0500
Subject: [PATCH 10/55] add some re: re: re: to replies

---
 activity.go        | 2 +-
 docs/changelog.txt | 2 ++
 web.go             | 6 ++++++
 3 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/activity.go b/activity.go
index e07c195..684a208 100644
--- a/activity.go
+++ b/activity.go
@@ -1022,7 +1022,7 @@ func jonkjonk(user *WhatAbout, h *Honk) (junk.Junk, junk.Junk) {
 		translate(h, true)
 		jo["summary"] = html.EscapeString(h.Precis)
 		jo["content"] = h.Noise
-		if strings.HasPrefix(h.Precis, "DZ:") {
+		if h.Precis != "" {
 			jo["sensitive"] = true
 		}
 
diff --git a/docs/changelog.txt b/docs/changelog.txt
index f1a96a4..f8c83a9 100644
--- a/docs/changelog.txt
+++ b/docs/changelog.txt
@@ -2,6 +2,8 @@ changelog
 
 -- next
 
++ Add some re: re: re: to replies.
+
 + Set an avatar. If you must.
 
 + Try a little harder to recover from httpsig failures.
diff --git a/web.go b/web.go
index 8d1c6d0..5f2c52f 100644
--- a/web.go
+++ b/web.go
@@ -1427,6 +1427,12 @@ func submithonk(w http.ResponseWriter, r *http.Request, isAPI bool) {
 			}
 		}
 		honk.RID = rid
+		if xonk.Precis != "" && honk.Precis == "" {
+			honk.Precis = xonk.Precis
+			if !(strings.HasPrefix(honk.Precis, "DZ:") || strings.HasPrefix(honk.Precis, "re: re: re: ")) {
+				honk.Precis = "re: " + honk.Precis
+			}
+		}
 	} else {
 		honk.Audience = []string{thewholeworld}
 	}

From ecbd80352fc1e1fa7454edac891b78032049edf7 Mon Sep 17 00:00:00 2001
From: Ted Unangst 
Date: Tue, 26 Nov 2019 00:29:31 -0500
Subject: [PATCH 11/55] when sending updates or deletes, send to all fetch
 recipients too

---
 activity.go |  3 +++
 database.go |  2 ++
 web.go      | 33 +++++++++++++++++++++++++++++++++
 3 files changed, 38 insertions(+)

diff --git a/activity.go b/activity.go
index 684a208..9597525 100644
--- a/activity.go
+++ b/activity.go
@@ -1209,6 +1209,9 @@ func honkworldwide(user *WhatAbout, honk *Honk) {
 				rcpts[h.XID] = true
 			}
 		}
+		for _, f := range getbacktracks(honk.XID) {
+			rcpts[f] = true
+		}
 	}
 	for a := range rcpts {
 		go deliverate(0, user.ID, a, msg)
diff --git a/database.go b/database.go
index 1af843c..f0ee4ba 100644
--- a/database.go
+++ b/database.go
@@ -693,6 +693,7 @@ var stmtGetZonkers, stmtRecentHonkers, stmtGetXonker, stmtSaveXonker, stmtDelete
 var stmtAllOnts, stmtSaveOnt, stmtUpdateFlags, stmtClearFlags *sql.Stmt
 var stmtHonksForUserFirstClass, stmtSaveMeta, stmtDeleteMeta, stmtUpdateHonk *sql.Stmt
 var stmtHonksISaved, stmtGetFilters, stmtSaveFilter, stmtDeleteFilter *sql.Stmt
+var stmtGetTracks *sql.Stmt
 
 func preparetodie(db *sql.DB, s string) *sql.Stmt {
 	stmt, err := db.Prepare(s)
@@ -769,4 +770,5 @@ func prepareStatements(db *sql.DB) {
 	stmtGetFilters = preparetodie(db, "select hfcsid, json from hfcs where userid = ?")
 	stmtSaveFilter = preparetodie(db, "insert into hfcs (userid, json) values (?, ?)")
 	stmtDeleteFilter = preparetodie(db, "delete from hfcs where userid = ? and hfcsid = ?")
+	stmtGetTracks = preparetodie(db, "select fetches from tracks where xid = ?")
 }
diff --git a/web.go b/web.go
index 5f2c52f..c7bbd97 100644
--- a/web.go
+++ b/web.go
@@ -890,6 +890,33 @@ type Track struct {
 	who string
 }
 
+func getbacktracks(xid string) []string {
+	c := make(chan bool)
+	dumptracks <- c
+	<-c
+	row := stmtGetTracks.QueryRow(xid)
+	var rawtracks string
+	err := row.Scan(&rawtracks)
+	if err != nil {
+		if err != sql.ErrNoRows {
+			log.Printf("error scanning tracks: %s", err)
+		}
+		return nil
+	}
+	var rcpts []string
+	for _, f := range strings.Split(rawtracks, " ") {
+		idx := strings.LastIndexByte(f, '#')
+		if idx != -1 {
+			f = f[:idx]
+		}
+		if !strings.HasPrefix(f, "https://") {
+			f = fmt.Sprintf("%https://%s/inbox", f)
+		}
+		rcpts = append(rcpts, f)
+	}
+	return rcpts
+}
+
 func savetracks(tracks map[string][]string) {
 	db := opendatabase()
 	tx, err := db.Begin()
@@ -940,6 +967,7 @@ func savetracks(tracks map[string][]string) {
 }
 
 var trackchan = make(chan Track)
+var dumptracks = make(chan chan bool)
 
 func tracker() {
 	timeout := 4 * time.Minute
@@ -955,6 +983,11 @@ func tracker() {
 				tracks = make(map[string][]string)
 			}
 			sleeper.Reset(timeout)
+		case c := <-dumptracks:
+			if len(tracks) > 0 {
+				savetracks(tracks)
+			}
+			c <- true
 		case <-endoftheworld:
 			if len(tracks) > 0 {
 				savetracks(tracks)

From a8126b4c26e6d7e8ff6720bc05266a2febb79555 Mon Sep 17 00:00:00 2001
From: Ted Unangst 
Date: Tue, 26 Nov 2019 00:33:27 -0500
Subject: [PATCH 12/55] fix format string

---
 web.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/web.go b/web.go
index c7bbd97..60df542 100644
--- a/web.go
+++ b/web.go
@@ -910,7 +910,7 @@ func getbacktracks(xid string) []string {
 			f = f[:idx]
 		}
 		if !strings.HasPrefix(f, "https://") {
-			f = fmt.Sprintf("%https://%s/inbox", f)
+			f = fmt.Sprintf("%%https://%s/inbox", f)
 		}
 		rcpts = append(rcpts, f)
 	}

From a719f9a03e4ed60ddba3c6a66bf4351cae9a84c3 Mon Sep 17 00:00:00 2001
From: Ted Unangst 
Date: Tue, 26 Nov 2019 00:42:32 -0500
Subject: [PATCH 13/55] fix another case of bad err shadowing

---
 activity.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/activity.go b/activity.go
index 9597525..e62a050 100644
--- a/activity.go
+++ b/activity.go
@@ -288,7 +288,8 @@ var boxofboxes = cache.New(cache.Options{Filler: func(ident string) (*Box, bool)
 	err := row.Scan(&info)
 	if err != nil {
 		log.Printf("need to get boxes for %s", ident)
-		j, err := GetJunk(ident)
+		var j junk.Junk
+		j, err = GetJunk(ident)
 		if err != nil {
 			log.Printf("error getting boxes: %s", err)
 			return nil, false

From e5e3c8b6a724291041a2fbe9e12d454661f3f2fe Mon Sep 17 00:00:00 2001
From: Ted Unangst 
Date: Tue, 26 Nov 2019 00:54:20 -0500
Subject: [PATCH 14/55] mention bug fixes

---
 docs/changelog.txt | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog.txt b/docs/changelog.txt
index f8c83a9..57f3d85 100644
--- a/docs/changelog.txt
+++ b/docs/changelog.txt
@@ -2,6 +2,8 @@ changelog
 
 -- next
 
++ Fix some bugs that may have interfered with federation.
+
 + Add some re: re: re: to replies.
 
 + Set an avatar. If you must.

From 97e84447c3704cefebbaaaf719214b57e5fbfc92 Mon Sep 17 00:00:00 2001
From: Ted Unangst 
Date: Tue, 26 Nov 2019 02:32:23 -0500
Subject: [PATCH 15/55] save federated pdfs

---
 activity.go | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/activity.go b/activity.go
index e62a050..d7accfe 100644
--- a/activity.go
+++ b/activity.go
@@ -199,6 +199,12 @@ func savedonk(url string, name, desc, media string, localize bool) *Donk {
 				format = "jpg"
 			}
 			xid = xid + "." + format
+		} else if media == "application/pdf" {
+			if len(data) > 1000000 {
+				log.Printf("not saving large pdf")
+				localize = false
+				data = []byte{}
+			}
 		} else if len(data) > 100000 {
 			log.Printf("not saving large attachment")
 			localize = false
@@ -709,7 +715,8 @@ func xonksaver(user *WhatAbout, item junk.Junk, origin string) *Honk {
 				} else if at == "Document" || at == "Image" {
 					mt = strings.ToLower(mt)
 					log.Printf("attachment: %s %s", mt, u)
-					if mt == "text/plain" || strings.HasPrefix(mt, "image") {
+					if mt == "text/plain" || mt == "application/pdf" ||
+						strings.HasPrefix(mt, "image") {
 						localize = true
 					}
 				} else {

From 0a9cbd2a3d15628572afd30bc7ed98e13d3753ea Mon Sep 17 00:00:00 2001
From: Ted Unangst 
Date: Tue, 26 Nov 2019 02:37:12 -0500
Subject: [PATCH 16/55] show a little more info for attachments

---
 views/honk.html | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/views/honk.html b/views/honk.html
index 805b366..d52523c 100644
--- a/views/honk.html
+++ b/views/honk.html
@@ -66,15 +66,15 @@ in reply to: {{ .RID }}
 {{ range .Donks }}
 {{ if .Local }}
 {{ if eq .Media "text/plain" }}
-
      Attachment: {{ .Name }} {{ .Desc }}
+
      Attachment: {{ .Name }}{{ if not (eq .Desc .Name) }} {{ .Desc }}{{ end }}
 {{ else if eq .Media "application/pdf" }}
-
      Attachment: {{ .Name }} {{ .Desc }}
+
      Attachment: {{ .Name }}{{ if not (eq .Desc .Name) }} {{ .Desc }}{{ end }}
 {{ else }}
 
      {{ .Desc }}
 {{ end }}
 {{ else }}
 {{ if .XID }}
-
      External Attachment: {{ .Name }}
+
      External Attachment: {{ .Name }}{{ if not (eq .Desc .Name) }} {{ .Desc }}{{ end }}
 {{ else }}
 {{ if eq .Media "video/mp4" }}
 
      

From 79be6ae2f4d92cb37715d1750911d4f7ab959573 Mon Sep 17 00:00:00 2001
From: Ted Unangst 
Date: Tue, 26 Nov 2019 02:46:34 -0500
Subject: [PATCH 17/55] finish up the changelog

---
 docs/changelog.txt | 22 +++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)

diff --git a/docs/changelog.txt b/docs/changelog.txt
index 57f3d85..0991f3e 100644
--- a/docs/changelog.txt
+++ b/docs/changelog.txt
@@ -1,6 +1,8 @@
 changelog
 
--- next
+-- 0.8.5 Turnkey Blaster
+
++ Codenames in changelog.
 
 + Fix some bugs that may have interfered with federation.
 
@@ -28,7 +30,7 @@ changelog
 
 - mistag.
 
--- 0.8.2
+-- 0.8.2 Game Warden
 
 ++ Import command to preserve those embarssassing old posts from Twitter.
 
@@ -56,7 +58,7 @@ changelog
 	Syntax highlighting for code blocks.
 	Something resembling an actual manual.
 
--- 0.8.0
+-- 0.8.0 Ordinary Octology
 
 +++ Add Honk Filtering and Censorship System (HFCS).
 
@@ -111,7 +113,7 @@ changelog
 - Sometimes the cached state of the @me feed becomes unsynced.
 	Acked status may display incorrectly.
 
--- 0.7.7
+-- 0.7.7 More 7 Than Ever
 
 + Add another retry to workaround pixelfed's general unreliability.
 
@@ -167,7 +169,7 @@ changelog
 
 + Fix bug preventing unfollow from working.
 
--- 0.7.0
+-- 0.7.0 Father Mother Maiden Crone Honker Bonker Zonker
 
 +++ Auto fetching and inlining of hoots.
 
@@ -197,6 +199,12 @@ changelog
 
 + Add max-width for video tag.
 
--- 0.6.0 and prior
+-- 0.6.0 Sixy Delights
 
-All records from this time of primitive development have been lost.
+Most records from this time of primitive development have been lost.
+
+-- 0.5.0 Halfway to Heaven
+
+-- 0.4.0 Fore Score
+
+-- 0.3.0 Valorous Varaha

From 59cc3645b071c1a595385783df5311b0784d6ed2 Mon Sep 17 00:00:00 2001
From: Ted Unangst 
Date: Tue, 26 Nov 2019 02:46:46 -0500
Subject: [PATCH 18/55] Added tag v0.8.5 for changeset 2e9969df06dd

---
 .hgtags | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.hgtags b/.hgtags
index be50023..7d0946f 100644
--- a/.hgtags
+++ b/.hgtags
@@ -25,3 +25,4 @@ b140f7a3216b820aa13f982e45ff42781d7a8f4a v0.8.2
 8a2a90379bf60d425fec114ff88f5fd9806a4965 v0.8.2
 808ef90260d5d81db1ec98fb8894588a3ac7b369 v0.8.3
 3ada67b721e7e4a478d0effacde14f36dc16e1de v0.8.4
+2e9969df06ddab8fa07999e91437dda28ec058ae v0.8.5

From ef845b2a2c31f72e2d33b9608afdd6f6599f98c0 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" 
Date: Tue, 26 Nov 2019 10:35:10 +0000
Subject: [PATCH 19/55] Add support for [^\n] meme files & hgignore

Should be done in separated commits but my mercurial-fu is bad.
---
 .hgignore |  4 ++++
 fun.go    |  2 +-
 web.go    | 18 ++++++++++++++----
 3 files changed, 19 insertions(+), 5 deletions(-)
 create mode 100644 .hgignore

diff --git a/.hgignore b/.hgignore
new file mode 100644
index 0000000..b5de9ba
--- /dev/null
+++ b/.hgignore
@@ -0,0 +1,4 @@
+.*\.db
+memes
+emus
+honk
diff --git a/fun.go b/fun.go
index 34a3c8e..6e54cba 100644
--- a/fun.go
+++ b/fun.go
@@ -340,7 +340,7 @@ func herdofemus(noise string) []Emu {
 	return emus
 }
 
-var re_memes = regexp.MustCompile("meme: ?([[:alnum:]_.-]+)")
+var re_memes = regexp.MustCompile("meme: ?([^\n]+)")
 
 func memetize(honk *Honk) {
 	repl := func(x string) string {
diff --git a/web.go b/web.go
index 60df542..5282d18 100644
--- a/web.go
+++ b/web.go
@@ -2026,13 +2026,23 @@ func servehtml(w http.ResponseWriter, r *http.Request) {
 }
 func serveemu(w http.ResponseWriter, r *http.Request) {
 	xid := mux.Vars(r)["xid"]
+	emu, err := url.QueryUnescape(xid)
+	if err != nil {
+		log.Print(err)
+	}
+
 	w.Header().Set("Cache-Control", "max-age="+somedays())
-	http.ServeFile(w, r, dataDir+"/emus/"+xid)
+	http.ServeFile(w, r, dataDir+"/emus/"+emu)
 }
 func servememe(w http.ResponseWriter, r *http.Request) {
 	xid := mux.Vars(r)["xid"]
+	meme, err := url.QueryUnescape(xid)
+	if err != nil {
+		log.Print(err)
+	}
+
 	w.Header().Set("Cache-Control", "max-age="+somedays())
-	http.ServeFile(w, r, dataDir+"/memes/"+xid)
+	http.ServeFile(w, r, dataDir+"/memes/"+meme)
 }
 
 func servefile(w http.ResponseWriter, r *http.Request) {
@@ -2279,8 +2289,8 @@ func serve() {
 	getters.HandleFunc("/o", thelistingoftheontologies)
 	getters.HandleFunc("/o/{name:.+}", showontology)
 	getters.HandleFunc("/d/{xid:[[:alnum:].]+}", servefile)
-	getters.HandleFunc("/emu/{xid:[[:alnum:]_.-]+}", serveemu)
-	getters.HandleFunc("/meme/{xid:[[:alnum:]_.-]+}", servememe)
+	getters.HandleFunc("/emu/{xid:.+}", serveemu)
+	getters.HandleFunc("/meme/{xid:.+}", servememe)
 	getters.HandleFunc("/.well-known/webfinger", fingerlicker)
 
 	getters.HandleFunc("/server", serveractor)

From 12f7cf6ec3cd06f20b0979debc9e5b701ef070a5 Mon Sep 17 00:00:00 2001
From: Ted Unangst 
Date: Tue, 26 Nov 2019 13:47:33 -0500
Subject: [PATCH 20/55] shouldn't need to query escape meme names, and don't
 allow / to prevent traversal

---
 web.go | 16 ++++------------
 1 file changed, 4 insertions(+), 12 deletions(-)

diff --git a/web.go b/web.go
index 5282d18..dcb94ae 100644
--- a/web.go
+++ b/web.go
@@ -2025,21 +2025,13 @@ func servehtml(w http.ResponseWriter, r *http.Request) {
 	}
 }
 func serveemu(w http.ResponseWriter, r *http.Request) {
-	xid := mux.Vars(r)["xid"]
-	emu, err := url.QueryUnescape(xid)
-	if err != nil {
-		log.Print(err)
-	}
+	emu := mux.Vars(r)["emu"]
 
 	w.Header().Set("Cache-Control", "max-age="+somedays())
 	http.ServeFile(w, r, dataDir+"/emus/"+emu)
 }
 func servememe(w http.ResponseWriter, r *http.Request) {
-	xid := mux.Vars(r)["xid"]
-	meme, err := url.QueryUnescape(xid)
-	if err != nil {
-		log.Print(err)
-	}
+	meme := mux.Vars(r)["meme"]
 
 	w.Header().Set("Cache-Control", "max-age="+somedays())
 	http.ServeFile(w, r, dataDir+"/memes/"+meme)
@@ -2289,8 +2281,8 @@ func serve() {
 	getters.HandleFunc("/o", thelistingoftheontologies)
 	getters.HandleFunc("/o/{name:.+}", showontology)
 	getters.HandleFunc("/d/{xid:[[:alnum:].]+}", servefile)
-	getters.HandleFunc("/emu/{xid:.+}", serveemu)
-	getters.HandleFunc("/meme/{xid:.+}", servememe)
+	getters.HandleFunc("/emu/{emu:[^/]+}", serveemu)
+	getters.HandleFunc("/meme/{meme:[^/]+}", servememe)
 	getters.HandleFunc("/.well-known/webfinger", fingerlicker)
 
 	getters.HandleFunc("/server", serveractor)

From 02433181f496e7a2816322bb06be7d9b1608ddbe Mon Sep 17 00:00:00 2001
From: Ted Unangst 
Date: Tue, 26 Nov 2019 13:48:39 -0500
Subject: [PATCH 21/55] the avatar regexp should match the meme one. keep
 together.

---
 fun.go | 1 +
 web.go | 2 --
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/fun.go b/fun.go
index 6e54cba..8c9c77d 100644
--- a/fun.go
+++ b/fun.go
@@ -341,6 +341,7 @@ func herdofemus(noise string) []Emu {
 }
 
 var re_memes = regexp.MustCompile("meme: ?([^\n]+)")
+var re_avatar = regexp.MustCompile("avatar: ?([^\n]+)")
 
 func memetize(honk *Honk) {
 	repl := func(x string) string {
diff --git a/web.go b/web.go
index dcb94ae..bdefaec 100644
--- a/web.go
+++ b/web.go
@@ -1102,8 +1102,6 @@ func honkpage(w http.ResponseWriter, u *login.UserInfo, honks []*Honk, templinfo
 	}
 }
 
-var re_avatar = regexp.MustCompile("avatar: ?([[:alnum:]_.-]+)")
-
 func saveuser(w http.ResponseWriter, r *http.Request) {
 	whatabout := r.FormValue("whatabout")
 	u := login.GetUserInfo(r)

From f76a5d61135a394906b9775cdd12c5ec50898dd8 Mon Sep 17 00:00:00 2001
From: Ted Unangst 
Date: Tue, 26 Nov 2019 13:51:10 -0500
Subject: [PATCH 22/55] start the new changelog

---
 docs/changelog.txt | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/docs/changelog.txt b/docs/changelog.txt
index 0991f3e..20269c1 100644
--- a/docs/changelog.txt
+++ b/docs/changelog.txt
@@ -1,5 +1,9 @@
 changelog
 
+-- next
+
++ More flexible meme names.
+
 -- 0.8.5 Turnkey Blaster
 
 + Codenames in changelog.

From 3014826f6f8382d1732bbe477a6cbb45be99a414 Mon Sep 17 00:00:00 2001
From: Ted Unangst 
Date: Tue, 26 Nov 2019 14:57:23 -0500
Subject: [PATCH 23/55] add a raw sendactivity API action

---
 activity.go          | 26 +++++++++++++------
 docs/changelog.txt   |  2 ++
 docs/honk.3          | 14 +++++++++++
 toys/Makefile        |  5 +++-
 toys/README          |  4 ++-
 toys/sprayandpray.go | 59 ++++++++++++++++++++++++++++++++++++++++++++
 web.go               |  8 ++++++
 7 files changed, 108 insertions(+), 10 deletions(-)
 create mode 100644 toys/sprayandpray.go

diff --git a/activity.go b/activity.go
index d7accfe..561a338 100644
--- a/activity.go
+++ b/activity.go
@@ -1186,24 +1186,34 @@ func gimmejonk(xid string) ([]byte, bool) {
 	return j, ok
 }
 
-func honkworldwide(user *WhatAbout, honk *Honk) {
-	jonk, _ := jonkjonk(user, honk)
-	jonk["@context"] = itiswhatitis
-	msg := jonk.ToBytes()
-
+func boxuprcpts(user *WhatAbout, addresses []string, useshared bool) map[string]bool {
 	rcpts := make(map[string]bool)
-	for _, a := range honk.Audience {
-		if a == thewholeworld || a == user.URL || strings.HasSuffix(a, "/followers") {
+	for _, a := range addresses {
+		if a == "" || a == thewholeworld || a == user.URL || strings.HasSuffix(a, "/followers") {
+			continue
+		}
+		if a[0] == '%' {
+			rcpts[a] = true
 			continue
 		}
 		var box *Box
 		ok := boxofboxes.Get(a, &box)
-		if ok && honk.Public && box.Shared != "" {
+		if ok && useshared && box.Shared != "" {
 			rcpts["%"+box.Shared] = true
 		} else {
 			rcpts[a] = true
 		}
 	}
+	return rcpts
+}
+
+func honkworldwide(user *WhatAbout, honk *Honk) {
+	jonk, _ := jonkjonk(user, honk)
+	jonk["@context"] = itiswhatitis
+	msg := jonk.ToBytes()
+
+	rcpts := boxuprcpts(user, honk.Audience, honk.Public)
+
 	if honk.Public {
 		for _, h := range getdubs(user.ID) {
 			if h.XID == user.URL {
diff --git a/docs/changelog.txt b/docs/changelog.txt
index 20269c1..6cfd523 100644
--- a/docs/changelog.txt
+++ b/docs/changelog.txt
@@ -2,6 +2,8 @@ changelog
 
 -- next
 
++ A raw sendactivity API action for the bold.
+
 + More flexible meme names.
 
 -- 0.8.5 Turnkey Blaster
diff --git a/docs/honk.3 b/docs/honk.3
index c12600e..bfd524d 100644
--- a/docs/honk.3
+++ b/docs/honk.3
@@ -109,6 +109,20 @@ If there are no results, wait this many seconds for something to appear.
 .El
 .Pp
 The result will be returned as json.
+.Ss sendactivity
+Send anything.
+No limits, no error checking.
+.Bl -tag -width public
+.It Fa rcpt
+An actor to deliver the message to to.
+May be specified more than once.
+An inbox may be specified directly by prefixing with %.
+.It Fa msg
+The message.
+It should be a valid json activity, but yolo.
+.It Fa public
+Set to 1 to use shared inboxes for delivery.
+.El
 .Sh EXAMPLES
 Refer to the sample code in the
 .Pa toys
diff --git a/toys/Makefile b/toys/Makefile
index c2c2f3b..dcf7319 100644
--- a/toys/Makefile
+++ b/toys/Makefile
@@ -1,5 +1,5 @@
 
-all: gettoken saytheday youvegothonks
+all: gettoken saytheday sprayandpray youvegothonks
 
 gettoken: gettoken.go
 	go build gettoken.go
@@ -7,5 +7,8 @@ gettoken: gettoken.go
 saytheday: saytheday.go
 	go build saytheday.go
 
+sprayandpray: sprayandpray.go
+	go build sprayandpray.go
+
 youvegothonks: youvegothonks.go
 	go build youvegothonks.go
diff --git a/toys/README b/toys/README
index 9239b9a..6f98c43 100644
--- a/toys/README
+++ b/toys/README
@@ -4,6 +4,8 @@ A little of this, a little of that.
 
 gettoken.go - obtains an authorization token
 
-saytheday.go - posts a new honk
+saytheday.go - posts a new honk that's a date based look and say sequence
+
+sprayandpray.go - send an activity with no error checking and hope it works
 
 youvegothonks.go - polls for new mesages
diff --git a/toys/sprayandpray.go b/toys/sprayandpray.go
new file mode 100644
index 0000000..44ee302
--- /dev/null
+++ b/toys/sprayandpray.go
@@ -0,0 +1,59 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+)
+
+
+func sendmsg(server, token, msg, rcpt string) {
+	form := make(url.Values)
+	form.Add("token", token)
+	form.Add("action", "sendactivity")
+	form.Add("msg", msg)
+	form.Add("rcpt", rcpt)
+	apiurl := fmt.Sprintf("https://%s/api", server)
+	req, err := http.NewRequest("POST", apiurl, strings.NewReader(form.Encode()))
+	if err != nil {
+		log.Fatal(err)
+	}
+	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer resp.Body.Close()
+	answer, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		log.Fatal(err)
+	}
+	if resp.StatusCode != 200 {
+		log.Fatalf("status: %d: %s", resp.StatusCode, answer)
+	}
+}
+
+func main() {
+	var server, token, msgfile, rcpt string
+	flag.StringVar(&server, "server", server, "server to connnect")
+	flag.StringVar(&token, "token", token, "auth token to use")
+	flag.StringVar(&msgfile, "msgfile", token, "file with message to send")
+	flag.StringVar(&rcpt, "rcpt", rcpt, "rcpt to send it to")
+	flag.Parse()
+
+	if server == "" || token == "" || msgfile == "" || rcpt == "" {
+		flag.Usage()
+		os.Exit(1)
+	}
+	msg, err := ioutil.ReadFile(msgfile)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	sendmsg(server, token, string(msg), rcpt)
+}
diff --git a/web.go b/web.go
index bdefaec..8389b3f 100644
--- a/web.go
+++ b/web.go
@@ -2138,6 +2138,7 @@ func honkhonkline() {
 func apihandler(w http.ResponseWriter, r *http.Request) {
 	u := login.GetUserInfo(r)
 	userid := u.UserID
+	user, _ := butwhatabout(u.Username)
 	action := r.FormValue("action")
 	wait, _ := strconv.ParseInt(r.FormValue("wait"), 10, 0)
 	log.Printf("api request '%s' on behalf of %s", action, u.Username)
@@ -2175,6 +2176,13 @@ func apihandler(w http.ResponseWriter, r *http.Request) {
 		j := junk.New()
 		j["honks"] = honks
 		j.Write(w)
+	case "sendactivity":
+		public := r.FormValue("public") == "1"
+		rcpts := boxuprcpts(user, r.Form["rcpt"], public)
+		msg := []byte(r.FormValue("msg"))
+		for rcpt := range rcpts {
+			go deliverate(0, userid, rcpt, msg)
+		}
 	default:
 		http.Error(w, "unknown action", http.StatusNotFound)
 		return

From a144795b13c8971df155287bc3cb481c81c3025a Mon Sep 17 00:00:00 2001
From: Ted Unangst 
Date: Wed, 27 Nov 2019 14:22:20 -0500
Subject: [PATCH 24/55] rework honkmeta deletion for flexibility

---
 database.go | 21 ++++++++++++---------
 1 file changed, 12 insertions(+), 9 deletions(-)

diff --git a/database.go b/database.go
index f0ee4ba..789559c 100644
--- a/database.go
+++ b/database.go
@@ -491,7 +491,7 @@ func updatehonk(h *Honk) error {
 		return err
 	}
 
-	err = deleteextras(tx, h.ID)
+	err = deleteextras(tx, h.ID, false)
 	if err == nil {
 		_, err = tx.Stmt(stmtUpdateHonk).Exec(h.Precis, h.Noise, h.Format, h.Whofore, dt, h.ID)
 	}
@@ -527,10 +527,7 @@ func deletehonk(honkid int64) error {
 		return err
 	}
 
-	err = deleteextras(tx, honkid)
-	if err == nil {
-		_, err = tx.Stmt(stmtDeleteMeta).Exec(honkid, "nonsense")
-	}
+	err = deleteextras(tx, honkid, true)
 	if err == nil {
 		_, err = tx.Stmt(stmtDeleteHonk).Exec(honkid)
 	}
@@ -583,7 +580,7 @@ func saveextras(tx *sql.Tx, h *Honk) error {
 	return nil
 }
 
-func deleteextras(tx *sql.Tx, honkid int64) error {
+func deleteextras(tx *sql.Tx, honkid int64, everything bool) error {
 	_, err := tx.Stmt(stmtDeleteDonks).Exec(honkid)
 	if err != nil {
 		return err
@@ -592,7 +589,11 @@ func deleteextras(tx *sql.Tx, honkid int64) error {
 	if err != nil {
 		return err
 	}
-	_, err = tx.Stmt(stmtDeleteMeta).Exec(honkid, "oldrev")
+	if everything {
+		_, err = tx.Stmt(stmtDeleteAllMeta).Exec(honkid)
+	} else {
+		_, err = tx.Stmt(stmtDeleteSomeMeta).Exec(honkid)
+	}
 	if err != nil {
 		return err
 	}
@@ -691,7 +692,8 @@ var stmtAddDoover, stmtGetDoovers, stmtLoadDoover, stmtZapDoover, stmtOneHonker
 var stmtUntagged, stmtDeleteHonk, stmtDeleteDonks, stmtDeleteOnts, stmtSaveZonker *sql.Stmt
 var stmtGetZonkers, stmtRecentHonkers, stmtGetXonker, stmtSaveXonker, stmtDeleteXonker *sql.Stmt
 var stmtAllOnts, stmtSaveOnt, stmtUpdateFlags, stmtClearFlags *sql.Stmt
-var stmtHonksForUserFirstClass, stmtSaveMeta, stmtDeleteMeta, stmtUpdateHonk *sql.Stmt
+var stmtHonksForUserFirstClass *sql.Stmt
+var stmtSaveMeta, stmtDeleteAllMeta, stmtDeleteSomeMeta, stmtUpdateHonk *sql.Stmt
 var stmtHonksISaved, stmtGetFilters, stmtSaveFilter, stmtDeleteFilter *sql.Stmt
 var stmtGetTracks *sql.Stmt
 
@@ -735,7 +737,8 @@ func prepareStatements(db *sql.DB) {
 	stmtHonksByOntology = preparetodie(db, selecthonks+"join onts on honks.honkid = onts.honkid where honks.honkid > ? and onts.ontology = ? and (honks.userid = ? or (? = -1 and honks.whofore = 2))"+limit)
 
 	stmtSaveMeta = preparetodie(db, "insert into honkmeta (honkid, genus, json) values (?, ?, ?)")
-	stmtDeleteMeta = preparetodie(db, "delete from honkmeta where honkid = ? and genus <> ?")
+	stmtDeleteAllMeta = preparetodie(db, "delete from honkmeta where honkid = ?")
+	stmtDeleteSomeMeta = preparetodie(db, "delete from honkmeta where honkid = ? and genus not in ('oldrev')")
 	stmtSaveHonk = preparetodie(db, "insert into honks (userid, what, honker, xid, rid, dt, url, audience, noise, convoy, whofore, format, precis, oonker, flags) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")
 	stmtDeleteHonk = preparetodie(db, "delete from honks where honkid = ?")
 	stmtUpdateHonk = preparetodie(db, "update honks set precis = ?, noise = ?, format = ?, whofore = ?, dt = ? where honkid = ?")

From 0b89e1b06c741595e1cb8f668e6a69c337539c59 Mon Sep 17 00:00:00 2001
From: Ted Unangst 
Date: Wed, 27 Nov 2019 14:36:29 -0500
Subject: [PATCH 25/55] save mentions in honkmeta

---
 activity.go | 14 +++++++++-----
 database.go | 16 ++++++++++++++++
 fun.go      |  9 ++-------
 honk.go     |  6 ++++++
 web.go      |  1 +
 5 files changed, 34 insertions(+), 12 deletions(-)

diff --git a/activity.go b/activity.go
index 561a338..a67aa3c 100644
--- a/activity.go
+++ b/activity.go
@@ -635,7 +635,7 @@ func xonksaver(user *WhatAbout, item junk.Junk, origin string) *Honk {
 		xonk.Audience = append(xonk.Audience, xonk.Honker)
 		xonk.Audience = oneofakind(xonk.Audience)
 
-		var mentions []string
+		var mentions []Mention
 		if obj != nil {
 			ot, _ := obj.GetString("type")
 			url, _ = obj.GetString("url")
@@ -773,7 +773,9 @@ func xonksaver(user *WhatAbout, item junk.Junk, origin string) *Honk {
 					xonk.Place = p
 				}
 				if tt == "Mention" {
-					m, _ := tag.GetString("href")
+					var m Mention
+					m.Who, _ = tag.GetString("name")
+					m.Where, _ = tag.GetString("href")
 					mentions = append(mentions, m)
 				}
 			}
@@ -852,8 +854,9 @@ func xonksaver(user *WhatAbout, item junk.Junk, origin string) *Honk {
 		xonk.Precis = precis
 		xonk.Format = "html"
 		xonk.Convoy = convoy
+		xonk.Mentions = mentions
 		for _, m := range mentions {
-			if m == user.URL {
+			if m.Where == user.URL {
 				xonk.Whofore = 1
 			}
 		}
@@ -873,6 +876,7 @@ func xonksaver(user *WhatAbout, item junk.Junk, origin string) *Honk {
 				prev.Onts = xonk.Onts
 				prev.Place = xonk.Place
 				prev.Whofore = xonk.Whofore
+				prev.Mentions = xonk.Mentions
 				updatehonk(prev)
 			}
 		}
@@ -1050,8 +1054,8 @@ func jonkjonk(user *WhatAbout, h *Honk) (junk.Junk, junk.Junk) {
 		for _, m := range mentions {
 			t := junk.New()
 			t["type"] = "Mention"
-			t["name"] = m.who
-			t["href"] = m.where
+			t["name"] = m.Who
+			t["href"] = m.Where
 			tags = append(tags, t)
 		}
 		for _, o := range h.Onts {
diff --git a/database.go b/database.go
index 789559c..9cfe7b2 100644
--- a/database.go
+++ b/database.go
@@ -413,6 +413,12 @@ func donksforhonks(honks []*Honk) {
 				continue
 			}
 			h.Time = t
+		case "mentions":
+			err = unjsonify(j, &h.Mentions)
+			if err != nil {
+				log.Printf("error parsing mentions: %s", err)
+				continue
+			}
 		case "oldrev":
 		default:
 			log.Printf("unknown meta genus: %s", genus)
@@ -577,6 +583,16 @@ func saveextras(tx *sql.Tx, h *Honk) error {
 			return err
 		}
 	}
+	if m := h.Mentions; len(m) > 0 {
+		j, err := jsonify(m)
+		if err == nil {
+			_, err = tx.Stmt(stmtSaveMeta).Exec(h.ID, "mentions", j)
+		}
+		if err != nil {
+			log.Printf("error saving mentions: %s", err)
+			return err
+		}
+	}
 	return nil
 }
 
diff --git a/fun.go b/fun.go
index 8c9c77d..577bd40 100644
--- a/fun.go
+++ b/fun.go
@@ -277,11 +277,6 @@ func ontologies(s string) []string {
 	return m[:j]
 }
 
-type Mention struct {
-	who   string
-	where string
-}
-
 var re_mentions = regexp.MustCompile(`@[[:alnum:]._-]+@[[:alnum:].-]*[[:alnum:]]`)
 var re_urltions = regexp.MustCompile(`@https://\S+`)
 
@@ -307,12 +302,12 @@ func bunchofgrapes(s string) []Mention {
 	for i := range m {
 		where := gofish(m[i])
 		if where != "" {
-			mentions = append(mentions, Mention{who: m[i], where: where})
+			mentions = append(mentions, Mention{Who: m[i], Where: where})
 		}
 	}
 	m = re_urltions.FindAllString(s, -1)
 	for i := range m {
-		mentions = append(mentions, Mention{who: m[i][1:], where: m[i][1:]})
+		mentions = append(mentions, Mention{Who: m[i][1:], Where: m[i][1:]})
 	}
 	return mentions
 }
diff --git a/honk.go b/honk.go
index 7b46f6c..33c852e 100644
--- a/honk.go
+++ b/honk.go
@@ -88,6 +88,12 @@ type Honk struct {
 	Onts     []string
 	Place    *Place
 	Time     *Time
+	Mentions []Mention
+}
+
+type Mention struct {
+	Who   string
+	Where string
 }
 
 type OldRevision struct {
diff --git a/web.go b/web.go
index 8389b3f..12b04ff 100644
--- a/web.go
+++ b/web.go
@@ -1435,6 +1435,7 @@ func submithonk(w http.ResponseWriter, r *http.Request, isAPI bool) {
 	noise = strings.Replace(noise, "\r", "", -1)
 	noise = quickrename(noise, userinfo.UserID)
 	noise = hooterize(noise)
+	honk.Mentions = bunchofgrapes(noise)
 	honk.Noise = noise
 	translate(honk, false)
 

From 35375d2be9866693b47409b4c6febf1fd38e6158 Mon Sep 17 00:00:00 2001
From: Ted Unangst 
Date: Wed, 27 Nov 2019 14:44:16 -0500
Subject: [PATCH 26/55] simplify update save code

---
 activity.go | 11 ++---------
 1 file changed, 2 insertions(+), 9 deletions(-)

diff --git a/activity.go b/activity.go
index a67aa3c..692e2bc 100644
--- a/activity.go
+++ b/activity.go
@@ -869,15 +869,8 @@ func xonksaver(user *WhatAbout, item junk.Junk, origin string) *Honk {
 				log.Printf("didn't find old version for update: %s", xonk.XID)
 				isUpdate = false
 			} else {
-				prev.Noise = xonk.Noise
-				prev.Precis = xonk.Precis
-				prev.Date = xonk.Date
-				prev.Donks = xonk.Donks
-				prev.Onts = xonk.Onts
-				prev.Place = xonk.Place
-				prev.Whofore = xonk.Whofore
-				prev.Mentions = xonk.Mentions
-				updatehonk(prev)
+				xonk.ID = prev.ID
+				updatehonk(&xonk)
 			}
 		}
 		if !isUpdate && needxonk(user, &xonk) {

From c77ef4636cfae39ac5b48796a9e552c970339c5b Mon Sep 17 00:00:00 2001
From: Ted Unangst 
Date: Wed, 27 Nov 2019 14:48:18 -0500
Subject: [PATCH 27/55] it seems some implementations do check context for a
 uri, so add data:

---
 activity.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/activity.go b/activity.go
index 692e2bc..1becc82 100644
--- a/activity.go
+++ b/activity.go
@@ -899,7 +899,7 @@ func xonksaver(user *WhatAbout, item junk.Junk, origin string) *Honk {
 				convoy = currenttid
 			}
 			if convoy == "" {
-				convoy = "missing-" + xfiltrate()
+				convoy = "data:,missing-" + xfiltrate()
 				currenttid = convoy
 			}
 			xonk.Convoy = convoy

From 5435dd1b3f17f6269bf9efb138bf660ae60db1a9 Mon Sep 17 00:00:00 2001
From: Ted Unangst 
Date: Wed, 27 Nov 2019 15:58:41 -0500
Subject: [PATCH 28/55] use separate backend hooks with tighter pledge

---
 backend.go | 4 +++-
 unveil.go  | 3 +++
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/backend.go b/backend.go
index ca7ba1e..8ad3117 100644
--- a/backend.go
+++ b/backend.go
@@ -73,6 +73,8 @@ func shrinkit(data []byte) (*image.Image, error) {
 	return res.Image, nil
 }
 
+var backendhooks []func()
+
 func backendServer() {
 	log.Printf("backend server running")
 	shrinker := new(Shrinker)
@@ -92,7 +94,7 @@ func backendServer() {
 	if err != nil {
 		log.Panicf("unable to register shrinker: %s", err)
 	}
-	for _, h := range preservehooks {
+	for _, h := range backendhooks {
 		h()
 	}
 	srv.Accept(lis)
diff --git a/unveil.go b/unveil.go
index 4650ae5..efdba9b 100644
--- a/unveil.go
+++ b/unveil.go
@@ -62,4 +62,7 @@ func init() {
 		C.unveil(nil, nil)
 		Pledge("stdio rpath wpath cpath flock dns inet unix")
 	})
+	backendhooks = append(backendhooks, func() {
+		Pledge("stdio unix")
+	})
 }

From a50115d65f25c8040dbe6bf5151433b9f340dd81 Mon Sep 17 00:00:00 2001
From: Ted Unangst 
Date: Thu, 28 Nov 2019 02:14:07 -0500
Subject: [PATCH 29/55] negated search terms

---
 database.go        | 20 ++++++++++++++++++--
 docs/changelog.txt |  2 ++
 docs/honk.1        |  7 +++++--
 3 files changed, 25 insertions(+), 4 deletions(-)

diff --git a/database.go b/database.go
index 9cfe7b2..4851b6d 100644
--- a/database.go
+++ b/database.go
@@ -249,8 +249,10 @@ func gethonksbysearch(userid int64, q string, wanted int64) []*Honk {
 	withhonker := 0
 	site := ""
 	withsite := 0
+	withnotq := 0
 	terms := strings.Split(q, " ")
 	q = "%"
+	notq := "%"
 	for _, t := range terms {
 		if strings.HasPrefix(t, "site:") {
 			site = t[5:]
@@ -267,13 +269,27 @@ func gethonksbysearch(userid int64, q string, wanted int64) []*Honk {
 			withhonker = 1
 			continue
 		}
+		if t[0] == '-' {
+			if t == "-" {
+				continue
+			}
+			if len(notq) != 1 {
+				notq += " "
+			}
+			notq += t[1:]
+			continue
+		}
 		if len(q) != 1 {
 			q += " "
 		}
 		q += t
 	}
 	q += "%"
-	rows, err := stmtHonksBySearch.Query(wanted, userid, withsite, site, withhonker, honker, honker, q, userid)
+	notq += "%"
+	if notq != "%%" {
+		withnotq = 1
+	}
+	rows, err := stmtHonksBySearch.Query(wanted, userid, withsite, site, withhonker, honker, honker, q, withnotq, notq, userid)
 	honks := getsomehonks(rows, err)
 	return honks
 }
@@ -748,7 +764,7 @@ func prepareStatements(db *sql.DB) {
 	stmtHonksByHonker = preparetodie(db, selecthonks+"join honkers on (honkers.xid = honks.honker or honkers.xid = honks.oonker) where honks.honkid > ? and honks.userid = ? and honkers.name = ?"+butnotthose+limit)
 	stmtHonksByXonker = preparetodie(db, selecthonks+" where honks.honkid > ? and honks.userid = ? and (honker = ? or oonker = ?)"+butnotthose+limit)
 	stmtHonksByCombo = preparetodie(db, selecthonks+" where honks.honkid > ? and honks.userid = ? and honks.honker in (select xid from honkers where honkers.userid = ? and honkers.combos like ?) "+butnotthose+" union "+selecthonks+"join onts on honks.honkid = onts.honkid where honks.honkid > ? and honks.userid = ? and onts.ontology in (select xid from honkers where combos like ?)"+butnotthose+limit)
-	stmtHonksBySearch = preparetodie(db, selecthonks+"where honks.honkid > ? and honks.userid = ? and (? = 0 or xid like ?) and (? = 0 or honks.honker = ? or honks.oonker = ?) and noise like ?"+butnotthose+limit)
+	stmtHonksBySearch = preparetodie(db, selecthonks+"where honks.honkid > ? and honks.userid = ? and (? = 0 or xid like ?) and (? = 0 or honks.honker = ? or honks.oonker = ?) and noise like ? and (? = 0 or noise not like ?)"+butnotthose+limit)
 	stmtHonksByConvoy = preparetodie(db, selecthonks+"where honks.honkid > ? and (honks.userid = ? or (? = -1 and whofore = 2)) and convoy = ?"+limit)
 	stmtHonksByOntology = preparetodie(db, selecthonks+"join onts on honks.honkid = onts.honkid where honks.honkid > ? and onts.ontology = ? and (honks.userid = ? or (? = -1 and honks.whofore = 2))"+limit)
 
diff --git a/docs/changelog.txt b/docs/changelog.txt
index 6cfd523..cbb945d 100644
--- a/docs/changelog.txt
+++ b/docs/changelog.txt
@@ -2,6 +2,8 @@ changelog
 
 -- next
 
++ Negated search -terms.
+
 + A raw sendactivity API action for the bold.
 
 + More flexible meme names.
diff --git a/docs/honk.1 b/docs/honk.1
index d9e679d..267e785 100644
--- a/docs/honk.1
+++ b/docs/honk.1
@@ -143,11 +143,14 @@ The following keywords are supported:
 Substring match on the post domain name.
 .It honker
 Exact match, either AP actor or honker nickname.
+.It -
+Negate term.
 .El
 .Pp
 Example:
-.Dl honker:goose big moose
-This query will find honks by the goose about the big moose.
+.Dl honker:goose big moose -footloose
+This query will find honks by the goose about the big moose, but excluding
+those about footloose.
 .Ss Filtering
 Sometimes other users of the federation can get unruly.
 The honk filtering and censorship system,

From b76f377ca946db49e471e6f4a1f0afc75473ab6b Mon Sep 17 00:00:00 2001
From: Ted Unangst 
Date: Fri, 29 Nov 2019 14:53:25 -0500
Subject: [PATCH 30/55] quick mention should allow trailing .

---
 fun.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/fun.go b/fun.go
index 577bd40..f2e959b 100644
--- a/fun.go
+++ b/fun.go
@@ -374,7 +374,7 @@ func memetize(honk *Honk) {
 	honk.Noise = re_memes.ReplaceAllStringFunc(honk.Noise, repl)
 }
 
-var re_quickmention = regexp.MustCompile("(^|[ \n])@[[:alnum:]]+([ \n]|$)")
+var re_quickmention = regexp.MustCompile("(^|[ \n])@[[:alnum:]]+([ \n.]|$)")
 
 func quickrename(s string, userid int64) string {
 	nonstop := true
@@ -389,7 +389,7 @@ func quickrename(s string, userid int64) string {
 			prefix += "@"
 			m = m[1:]
 			tail := ""
-			if m[len(m)-1] == ' ' || m[len(m)-1] == '\n' {
+			if last := m[len(m)-1]; last == ' ' || last == '\n' || last == '.' {
 				tail = m[len(m)-1:]
 				m = m[:len(m)-1]
 			}

From f994120a87a02bafff835def1ebd4ae5fb1441d3 Mon Sep 17 00:00:00 2001
From: Ted Unangst 
Date: Sat, 30 Nov 2019 17:51:27 -0500
Subject: [PATCH 31/55] xonks without https are simply invalid

---
 activity.go | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/activity.go b/activity.go
index 1becc82..31315b2 100644
--- a/activity.go
+++ b/activity.go
@@ -242,6 +242,9 @@ func needxonk(user *WhatAbout, x *Honk) bool {
 	return needxonkid(user, x.XID)
 }
 func needxonkid(user *WhatAbout, xid string) bool {
+	if !strings.HasPrefix(xid, "https://") {
+		return false
+	}
 	if strings.HasPrefix(xid, user.URL+"/") {
 		return false
 	}

From d164471a9227a18a7b6c16fb652aee83a7a8046e Mon Sep 17 00:00:00 2001
From: Ted Unangst 
Date: Sun, 1 Dec 2019 12:27:09 -0500
Subject: [PATCH 32/55] rework emu save slightly for local emus. no need for
 database copy

---
 fun.go | 30 +++++++++++++++++++++++++-----
 web.go |  8 --------
 2 files changed, 25 insertions(+), 13 deletions(-)

diff --git a/fun.go b/fun.go
index f2e959b..5ae0941 100644
--- a/fun.go
+++ b/fun.go
@@ -62,6 +62,10 @@ func reverbolate(userid int64, honks []*Honk) {
 			h.Style += " limited"
 		}
 		translate(h, false)
+		local := false
+		if (h.Whofore == 2 || h.Whofore == 3) || h.What != "bonked" {
+			local = true
+		}
 		if h.Whofore == 2 || h.Whofore == 3 {
 			h.URL = h.XID
 			if h.What != "bonked" {
@@ -130,6 +134,13 @@ func reverbolate(userid int64, honks []*Honk) {
 					}
 				}
 			}
+			if local {
+				var emu Emu
+				emucache.Get(e, &emu)
+				if emu.ID != "" {
+					return fmt.Sprintf(``, emu.Name, emu.ID)
+				}
+			}
 			return e
 		}
 		h.Precis = re_emus.ReplaceAllStringFunc(h.Precis, emuxifier)
@@ -319,18 +330,27 @@ type Emu struct {
 
 var re_emus = regexp.MustCompile(`:[[:alnum:]_-]+:`)
 
+var emucache = cache.New(cache.Options{Filler: func(ename string) (Emu, bool) {
+	fname := ename[1 : len(ename)-1]
+	_, err := os.Stat(dataDir + "/emus/" + fname + ".png")
+	if err != nil {
+		return Emu{Name: ename, ID: ""}, true
+	}
+	url := fmt.Sprintf("https://%s/emu/%s.png", serverName, fname)
+	return Emu{ID: url, Name: ename}, true
+}, Duration: 10 * time.Second})
+
 func herdofemus(noise string) []Emu {
 	m := re_emus.FindAllString(noise, -1)
 	m = oneofakind(m)
 	var emus []Emu
 	for _, e := range m {
-		fname := e[1 : len(e)-1]
-		_, err := os.Stat("emus/" + fname + ".png")
-		if err != nil {
+		var emu Emu
+		emucache.Get(e, &emu)
+		if emu.ID == "" {
 			continue
 		}
-		url := fmt.Sprintf("https://%s/emu/%s.png", serverName, fname)
-		emus = append(emus, Emu{ID: url, Name: e})
+		emus = append(emus, emu)
 	}
 	return emus
 }
diff --git a/web.go b/web.go
index 12b04ff..ac964bf 100644
--- a/web.go
+++ b/web.go
@@ -1575,14 +1575,6 @@ func submithonk(w http.ResponseWriter, r *http.Request, isAPI bool) {
 			log.Printf("can't find file: %s", xid)
 		}
 	}
-	herd := herdofemus(noise)
-	for _, e := range herd {
-		donk := savedonk(e.ID, e.Name, e.Name, "image/png", true)
-		if donk != nil {
-			donk.Name = e.Name
-			honk.Donks = append(honk.Donks, donk)
-		}
-	}
 	memetize(honk)
 	imaginate(honk)
 

From d5d943b2033b92f8387d78041a2b66154cc3ab6d Mon Sep 17 00:00:00 2001
From: Ted Unangst 
Date: Sun, 1 Dec 2019 12:34:08 -0500
Subject: [PATCH 33/55] can simplify the reverb logic some

---
 fun.go | 37 ++++++++++++++++---------------------
 1 file changed, 16 insertions(+), 21 deletions(-)

diff --git a/fun.go b/fun.go
index 5ae0941..2cbc237 100644
--- a/fun.go
+++ b/fun.go
@@ -63,32 +63,27 @@ func reverbolate(userid int64, honks []*Honk) {
 		}
 		translate(h, false)
 		local := false
-		if (h.Whofore == 2 || h.Whofore == 3) || h.What != "bonked" {
+		if (h.Whofore == 2 || h.Whofore == 3) && h.What != "bonked" {
 			local = true
 		}
-		if h.Whofore == 2 || h.Whofore == 3 {
-			h.URL = h.XID
-			if h.What != "bonked" {
-				h.Noise = re_memes.ReplaceAllString(h.Noise, "")
-				h.Noise = mentionize(h.Noise)
-				h.Noise = ontologize(h.Noise)
-			}
-			h.Username, h.Handle = handles(h.Honker)
+		if local {
+			h.Noise = re_memes.ReplaceAllString(h.Noise, "")
+			h.Noise = mentionize(h.Noise)
+			h.Noise = ontologize(h.Noise)
+		}
+		h.Username, h.Handle = handles(h.Honker)
+		short := shortname(userid, h.Honker)
+		if short != "" {
+			h.Username = short
 		} else {
-			_, h.Handle = handles(h.Honker)
-			short := shortname(userid, h.Honker)
-			if short != "" {
-				h.Username = short
-			} else {
-				h.Username = h.Handle
-				if len(h.Username) > 20 {
-					h.Username = h.Username[:20] + ".."
-				}
-			}
-			if h.URL == "" {
-				h.URL = h.XID
+			h.Username = h.Handle
+			if len(h.Username) > 20 {
+				h.Username = h.Username[:20] + ".."
 			}
 		}
+		if h.URL == "" {
+			h.URL = h.XID
+		}
 		if h.Oonker != "" {
 			_, h.Oondle = handles(h.Oonker)
 		}

From e421c0ba2034bfeaeaa8ffe0fd7cdf27e6954153 Mon Sep 17 00:00:00 2001
From: Ted Unangst 
Date: Sun, 1 Dec 2019 13:16:10 -0500
Subject: [PATCH 34/55] only need the handle rename logic for !local

---
 fun.go | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/fun.go b/fun.go
index 2cbc237..31b6435 100644
--- a/fun.go
+++ b/fun.go
@@ -72,13 +72,15 @@ func reverbolate(userid int64, honks []*Honk) {
 			h.Noise = ontologize(h.Noise)
 		}
 		h.Username, h.Handle = handles(h.Honker)
-		short := shortname(userid, h.Honker)
-		if short != "" {
-			h.Username = short
-		} else {
-			h.Username = h.Handle
-			if len(h.Username) > 20 {
-				h.Username = h.Username[:20] + ".."
+		if !local {
+			short := shortname(userid, h.Honker)
+			if short != "" {
+				h.Username = short
+			} else {
+				h.Username = h.Handle
+				if len(h.Username) > 20 {
+					h.Username = h.Username[:20] + ".."
+				}
 			}
 		}
 		if h.URL == "" {

From 21aed53283f4705cef5bbdbde526b8ce065f0839 Mon Sep 17 00:00:00 2001
From: Ted Unangst 
Date: Mon, 2 Dec 2019 01:37:36 -0500
Subject: [PATCH 35/55] store some dead code in the attic

---
 skulduggery.go | 77 --------------------------------------------------
 1 file changed, 77 deletions(-)

diff --git a/skulduggery.go b/skulduggery.go
index a09a3df..c16f215 100644
--- a/skulduggery.go
+++ b/skulduggery.go
@@ -21,87 +21,10 @@ import (
 	"github.com/mattn/go-runewidth"
 )
 
-// these lists are mostly (?) accurate
-var bigboldshitz = "๐€๐๐‚๐ƒ๐„๐…๐†๐‡๐ˆ๐‰๐Š๐‹๐Œ๐๐Ž๐๐๐‘๐’๐“๐”๐•๐–๐—๐˜๐™"
-var lilboldshitz = "๐š๐›๐œ๐๐ž๐Ÿ๐ ๐ก๐ข๐ฃ๐ค๐ฅ๐ฆ๐ง๐จ๐ฉ๐ช๐ซ๐ฌ๐ญ๐ฎ๐ฏ๐ฐ๐ฑ๐ฒ๐ณ"
-var moeboldshitz = "๐—”๐—•๐—–๐——๐—˜๐—™๐—š๐—›๐—œ๐—๐—ž๐—Ÿ๐— ๐—ก๐—ข๐—ฃ๐—ค๐—ฅ๐—ฆ๐—ง๐—จ๐—ฉ๐—ช๐—ซ๐—ฌ๐—ญ"
-var morboldshitz = "๐—ฎ๐—ฏ๐—ฐ๐—ฑ๐—ฒ๐—ณ๐—ด๐—ต๐—ถ๐—ท๐—ธ๐—น๐—บ๐—ป๐—ผ๐—ฝ๐—พ๐—ฟ๐˜€๐˜๐˜‚๐˜ƒ๐˜„๐˜…๐˜†๐˜‡"
-var biggothshitz = "๐•ฌ๐•ญ๐•ฎ๐•ฏ๐•ฐ๐•ฑ๐•ฒ๐•ณ๐•ด๐•ต๐•ถ๐•ท๐•ธ๐•น๐•บ๐•ป๐•ผ๐•ฝ๐•พ๐•ฟ๐–€๐–๐–‚๐–ƒ๐–„๐–…"
-var lilgothshitz = "๐–†๐–‡๐–ˆ๐–‰๐–Š๐–‹๐–Œ๐–๐–Ž๐–๐–๐–‘๐–’๐–“๐–”๐–•๐––๐–—๐–˜๐–™๐–š๐–›๐–œ๐–๐–ž๐–Ÿ"
-var moegothshitz = "๐”„๐”…๐•ฎ๐”‡๐”ˆ๐”‰๐”Š๐•ณโ„‘๐”๐”Ž๐”๐”๐”‘๐”’๐”“๐””โ„œ๐”–๐”—๐”˜๐”™๐”š๐”›๐”œ๐–…"
-var morgothshitz = "๐”ž๐”Ÿ๐” ๐”ก๐”ข๐”ฃ๐”ค๐”ฅ๐”ฆ๐”ง๐”จ๐”ฉ๐”ช๐”ซ๐”ฌ๐”ญ๐”ฎ๐”ฏ๐”ฐ๐”ฑ๐”ฒ๐”ณ๐”ด๐”ต๐”ถ๐”ท"
-var bigitalshitz = "๐‘จ๐‘ฉ๐‘ช๐‘ซ๐‘ฌ๐‘ญ๐‘ฎ๐‘ฏ๐‘ฐ๐‘ฑ๐‘ฒ๐‘ณ๐‘ด๐‘ต๐‘ถ๐‘ท๐‘ธ๐‘น๐‘บ๐‘ป๐‘ผ๐‘ฝ๐‘พ๐‘ฟ๐’€๐’"
-var lilitalshitz = "๐’‚๐’ƒ๐’„๐’…๐’†๐’‡๐’ˆ๐’‰๐’Š๐’‹๐’Œ๐’๐’Ž๐’๐’๐’‘๐’’๐’“๐’”๐’•๐’–๐’—๐’˜๐’™๐’š๐’›"
-var moeitalshitz = "๐˜ผ๐˜ฝ๐˜พ๐˜ฟ๐™€๐™๐™‚๐™ƒ๐™„๐™…๐™†๐™‡๐™ˆ๐™‰๐™Š๐™‹๐™Œ๐™๐™Ž๐™๐™๐™‘๐™’๐™“๐™”๐™•"
-var moritalshitz = "๐™–๐™—๐™˜๐™™๐™š๐™›๐™œ๐™๐™ž๐™Ÿ๐™ ๐™ก๐™ข๐™ฃ๐™ค๐™ฅ๐™ฆ๐™ง๐™จ๐™ฉ๐™ช๐™ซ๐™ฌ๐™ญ๐™ฎ๐™ฏ"
-var bigbangshitz = "๐”ธ๐”นโ„‚๐”ป๐”ผ๐”ฝ๐”พโ„๐•€๐•๐•‚๐•ƒ๐•„โ„•๐•†โ„™โ„šโ„๐•Š๐•‹๐•Œ๐•๐•Ž๐•๐•โ„ค"
-var lilbangshitz = "๐•’๐•“๐•”๐••๐•–๐•—๐•˜๐•™๐•š๐•›๐•œ๐•๐•ž๐•Ÿ๐• ๐•ก๐•ข๐•ฃ๐•ค๐•ฅ๐•ฆ๐•ง๐•จ๐•ฉ๐•ช๐•ซ"
-var bigwideshitz = "๏ผก๏ผข๏ผฃ๏ผค๏ผฅ๏ผฆ๏ผง๏ผจ๏ผฉ๏ผช๏ผซ๏ผฌ๏ผญ๏ผฎ๏ผฏ๏ผฐ๏ผฑ๏ผฒ๏ผณ๏ผด๏ผต๏ผถ๏ผท๏ผธ๏ผน๏ผบ"
-var lilwideshitz = "๏ฝ๏ฝ‚๏ฝƒ๏ฝ„๏ฝ…๏ฝ†๏ฝ‡๏ฝˆ๏ฝ‰๏ฝŠ๏ฝ‹๏ฝŒ๏ฝ๏ฝŽ๏ฝ๏ฝ๏ฝ‘๏ฝ’๏ฝ“๏ฝ”๏ฝ•๏ฝ–๏ฝ—๏ฝ˜๏ฝ™๏ฝš"
-var bigblokshitz = "๐Ÿ…ฐ๐Ÿ…ฑ๐Ÿ…ฒ๐Ÿ…ณ๐Ÿ…ด๐Ÿ…ต๐Ÿ…ถ๐Ÿ…ท๐Ÿ…ธ๐Ÿ…น๐Ÿ…บ๐Ÿ…ป๐Ÿ…ผ๐Ÿ…ฝ๐Ÿ…พ๐Ÿ…ฟ๐Ÿ†€๐Ÿ†๐Ÿ†‚๐Ÿ†ƒ๐Ÿ†„๐Ÿ†…๐Ÿ††๐Ÿ†‡๐Ÿ†ˆ๐Ÿ†‰"
-var evenmoeshitz = "๐ด๐ต๐ถ๐ท๐ธ๐น๐บ๐ป๐ผ๐ฝ๐พ๐ฟ๐‘€๐‘๐‘‚๐‘ƒ๐‘„๐‘…๐‘†๐‘‡๐‘ˆ๐‘‰๐‘Š๐‘‹๐‘Œ๐‘"
-var evenmorshitz = "๐’ถ๐’ท๐’ธ๐’น๐‘’๐’ป๐“ฐ๐’ฝ๐’พ๐’ฟ๐“€๐“๐“‚๐“ƒ๐“ธ๐“…๐“†๐“‡๐“ˆ๐“‰๐“Š๐“‹๐“Œ๐“๐“Ž๐“"
-
-var re_alltheshitz = regexp.MustCompile(`([` +
-	bigboldshitz + lilboldshitz +
-	moeboldshitz + morboldshitz +
-	biggothshitz + lilgothshitz +
-	moegothshitz + morgothshitz +
-	bigitalshitz + lilitalshitz +
-	moeitalshitz + moritalshitz +
-	bigbangshitz + lilbangshitz +
-	bigwideshitz + lilwideshitz +
-	evenmoeshitz + evenmorshitz +
-	bigblokshitz +
-	"][ '\ufe0f]?){3,}")
-
-var allUppers = []string{bigboldshitz, moeboldshitz, biggothshitz, bigwideshitz, moegothshitz, bigitalshitz, moeitalshitz, bigbangshitz, bigblokshitz, evenmoeshitz}
-var allLowers = []string{lilboldshitz, morboldshitz, lilgothshitz, lilwideshitz, morgothshitz, lilitalshitz, moritalshitz, lilbangshitz, evenmorshitz}
 
 var skinTones = "\U0001F3FB\U0001F3FC\U0001F3FD\U0001F3FE\U0001F3FF"
 var re_moredumb = regexp.MustCompile("[\U0001f44f\U0001f6a8\U000026a0][" + skinTones + "\ufe0f]*")
 
-// this may not be especially fast
-func unpucker(s string) string {
-	fixer := func(r string) string {
-		x := make([]byte, len(r))
-		xi := 0
-	loop1:
-		for _, c := range r {
-			xi++
-			if c == ' ' || c == '\'' {
-				x[xi] = byte(c)
-				continue
-			}
-			for _, set := range allUppers {
-				i := 0
-				for _, rr := range set {
-					if rr == c {
-						x[xi] = byte('A' + i)
-						continue loop1
-					}
-					i++
-				}
-			}
-			for _, set := range allLowers {
-				i := 0
-				for _, rr := range set {
-					if rr == c {
-						x[xi] = byte('a' + i)
-						continue loop1
-					}
-					i++
-				}
-			}
-			x[xi] = '.'
-		}
-		return string(x)
-	}
-	s = re_alltheshitz.ReplaceAllStringFunc(s, fixer)
-
-	return demoji(s)
-}
-
 func demoji(s string) string {
 	s = re_moredumb.ReplaceAllString(s, ".")
 

From beaceaadb1ad6d5fc5309fe1b0ee6de735d8b551 Mon Sep 17 00:00:00 2001
From: Ted Unangst 
Date: Mon, 2 Dec 2019 02:43:52 -0500
Subject: [PATCH 36/55] add a benchmark for markdown

---
 markitzero_test.go | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/markitzero_test.go b/markitzero_test.go
index 6cd176b..2f39fcb 100644
--- a/markitzero_test.go
+++ b/markitzero_test.go
@@ -1,6 +1,7 @@
 package main
 
 import (
+	"strings"
 	"testing"
 )
 
@@ -106,3 +107,22 @@ para
 	output := `hello
      • a list
      • the list
      para
      • singleton
      `
 	doonezerotest(t, input, output)
 }
+
+var benchData, simpleData string
+
+func init() {
+	benchData = strings.Repeat("hello there sir. It is a **fine** day for some testing!\n", 100)
+	simpleData = strings.Repeat("just a few words\n", 100)
+}
+
+func BenchmarkModerateSize(b *testing.B) {
+	for n := 0; n < b.N; n++ {
+		markitzero(benchData)
+	}
+}
+
+func BenchmarkSimpleData(b *testing.B) {
+	for n := 0; n < b.N; n++ {
+		markitzero(simpleData)
+	}
+}

From 95dfba8d146fa71466879b245ef2a54eeb97213a Mon Sep 17 00:00:00 2001
From: Ted Unangst 
Date: Mon, 2 Dec 2019 02:44:24 -0500
Subject: [PATCH 37/55] a faster version of markitzero, though i'm uncertain
 it's worth it

---
 markitzero.go | 177 ++++++++++++++++++++++++++++++++------------------
 1 file changed, 112 insertions(+), 65 deletions(-)

diff --git a/markitzero.go b/markitzero.go
index 464de15..2af6e02 100644
--- a/markitzero.go
+++ b/markitzero.go
@@ -16,11 +16,11 @@
 package main
 
 import (
+	"bytes"
 	"fmt"
 	"regexp"
 	"strings"
 
-	"golang.org/x/net/html"
 	"humungus.tedunangst.com/r/webs/synlight"
 )
 
@@ -37,27 +37,8 @@ var re_lister = regexp.MustCompile(`((^|\n)(\+|-).*)+\n?`)
 
 var lighter = synlight.New(synlight.Options{Format: synlight.HTML})
 
-func markitzero(s string) string {
-	// prepare the string
-	s = strings.TrimSpace(s)
-	s = strings.Replace(s, "\r", "", -1)
-
-	// save away the code blocks so we don't mess them up further
-	var bigcodes, lilcodes, images []string
-	s = re_bigcoder.ReplaceAllStringFunc(s, func(code string) string {
-		bigcodes = append(bigcodes, code)
-		return "``````"
-	})
-	s = re_coder.ReplaceAllStringFunc(s, func(code string) string {
-		lilcodes = append(lilcodes, code)
-		return "`x`"
-	})
-	s = re_imgfix.ReplaceAllStringFunc(s, func(img string) string {
-		images = append(images, img)
-		return ""
-	})
-
-	// fewer side effects than html.EscapeString
+// fewer side effects than html.EscapeString
+func fasterescaper(s []byte) []byte {
 	buf := make([]byte, 0, len(s))
 	for _, c := range []byte(s) {
 		switch c {
@@ -71,64 +52,130 @@ func markitzero(s string) string {
 			buf = append(buf, c)
 		}
 	}
-	s = string(buf)
+	return buf
+}
 
-	// mark it zero
-	s = re_link.ReplaceAllStringFunc(s, linkreplacer)
-	s = re_zerolink.ReplaceAllString(s, `$1`)
-	s = re_bolder.ReplaceAllString(s, "$1$2$3")
-	s = re_italicer.ReplaceAllString(s, "$1$2$3")
-	s = re_quoter.ReplaceAllString(s, "
      $1
      $3
      ")
-	s = re_reciter.ReplaceAllString(s, "$1$2$3")
-	s = strings.Replace(s, "\n---\n", "
      ", -1)
+func replaceifmatch(re *regexp.Regexp, input []byte, repl []byte) []byte {
+	if !re.Match(input) {
+		return input
+	}
+	return re.ReplaceAll(input, repl)
+}
 
-	s = re_lister.ReplaceAllStringFunc(s, func(m string) string {
-		m = strings.Trim(m, "\n")
-		items := strings.Split(m, "\n")
-		r := "
        "
-		for _, item := range items {
-			r += "
      • " + strings.Trim(item[1:], " ")
-		}
-		r += "
      "
-		return r
+func replaceifmatchfn(re *regexp.Regexp, input []byte, repl func([]byte) []byte) []byte {
+	if !re.Match(input) {
+		return input
+	}
+	return re.ReplaceAllFunc(input, repl)
+}
+
+func replacenocopy(input []byte, pat []byte, repl []byte) []byte {
+	if !bytes.Contains(input, pat) {
+		return input
+	}
+	return bytes.Replace(input, pat, repl, -1)
+}
+
+func markitzero(ss string) string {
+	s := []byte(ss)
+	// prepare the string
+	s = bytes.TrimSpace(s)
+	s = replacenocopy(s, []byte("\r"), []byte(""))
+
+	hascode := bytes.Contains(s, []byte("`"))
+
+	// save away the code blocks so we don't mess them up further
+	var bigcodes, lilcodes, images [][]byte
+	if hascode {
+		s = replaceifmatchfn(re_bigcoder, s, func(code []byte) []byte {
+			bigcodes = append(bigcodes, code)
+			return []byte("``````")
+		})
+		s = replaceifmatchfn(re_coder, s, func(code []byte) []byte {
+			lilcodes = append(lilcodes, code)
+			return []byte("`x`")
+		})
+	}
+	s = replaceifmatchfn(re_imgfix, s, func(img []byte) []byte {
+		images = append(images, img)
+		return []byte("")
 	})
 
+	s = fasterescaper(s)
+
+	// mark it zero
+	if bytes.Contains(s, []byte("http")) {
+		s = replaceifmatchfn(re_link, s, linkreplacer)
+	}
+	s = replaceifmatch(re_zerolink, s, []byte(`$1`))
+	if bytes.Contains(s, []byte("**")) {
+		s = replaceifmatch(re_bolder, s, []byte("$1$2$3"))
+	}
+	if bytes.Contains(s, []byte("*")) {
+		s = replaceifmatch(re_italicer, s, []byte("$1$2$3"))
+	}
+	if bytes.Contains(s, []byte("> ")) {
+		s = replaceifmatch(re_quoter, s, []byte("
      $1
      $3
      "))
+		s = replaceifmatch(re_reciter, s, []byte("$1$2$3"))
+	}
+	s = replacenocopy(s, []byte("\n---\n"), []byte("
      "))
+
+	if bytes.Contains(s, []byte("\n+")) || bytes.Contains(s, []byte("\n-")) {
+		s = replaceifmatchfn(re_lister, s, func(m []byte) []byte {
+			m = bytes.Trim(m, "\n")
+			items := bytes.Split(m, []byte("\n"))
+			r := []byte("
        ")
+			for _, item := range items {
+				r = append(r, []byte("
      • ")...)
+				r = append(r, bytes.Trim(item[1:], " ")...)
+			}
+			r = append(r, []byte("
      ")...)
+			return r
+		})
+	}
+
 	// restore images
-	s = strings.Replace(s, "<img x>", "", -1)
-	s = re_imgfix.ReplaceAllStringFunc(s, func(string) string {
+	s = replacenocopy(s, []byte("<img x>"), []byte(""))
+	s = replaceifmatchfn(re_imgfix, s, func([]byte) []byte {
 		img := images[0]
 		images = images[1:]
 		return img
 	})
 
 	// now restore the code blocks
-	s = re_coder.ReplaceAllStringFunc(s, func(string) string {
-		code := lilcodes[0]
-		lilcodes = lilcodes[1:]
-		code = html.EscapeString(code)
-		return code
-	})
-	s = re_bigcoder.ReplaceAllStringFunc(s, func(string) string {
-		code := bigcodes[0]
-		bigcodes = bigcodes[1:]
-		m := re_bigcoder.FindStringSubmatch(code)
-		return "
      " + lighter.HighlightString(m[2], m[1]) + "
      "
-	})
-	s = re_coder.ReplaceAllString(s, "$1")
+	if hascode {
+		s = replaceifmatchfn(re_coder, s, func([]byte) []byte {
+			code := lilcodes[0]
+			lilcodes = lilcodes[1:]
+			return fasterescaper(code)
+		})
+		s = replaceifmatchfn(re_bigcoder, s, func([]byte) []byte {
+			code := bigcodes[0]
+			bigcodes = bigcodes[1:]
+			m := re_bigcoder.FindSubmatch(code)
+			var buf bytes.Buffer
+			buf.WriteString("
      ")
+			lighter.Highlight(m[2], string(m[1]), &buf)
+			buf.WriteString("
      ")
+			return buf.Bytes()
+		})
+		s = replaceifmatch(re_coder, s, []byte("$1"))
+	}
 
 	// some final fixups
-	s = strings.Replace(s, "\n", "
      ", -1)
-	s = strings.Replace(s, "
      ", "
      ", -1)
-	s = strings.Replace(s, "
      ", "", -1)
-	s = strings.Replace(s, "
      ", "
      ", -1)
-	s = strings.Replace(s, "