enough of the honk filtering and censoring system (hfcs) to get going.

activity.go

@@ -203,7 +203,7 @@ func iszonked(userid int64, xid string) bool {
 }
 
 func needxonk(user *WhatAbout, x *Honk) bool {
-	if thoudostbitethythumb(user.ID, x.Audience, x.XID) {
+	if rejectnote(x) {
 		log.Printf("not saving thumb biter? %s via %s", x.XID, x.Honker)
 		return false
 	}
@@ -213,7 +213,7 @@ func needxonkid(user *WhatAbout, xid string) bool {
 	if strings.HasPrefix(xid, user.URL+"/") {
 		return false
 	}
-	if thoudostbitethythumb(user.ID, nil, xid) {
+	if rejectorigin(user.ID, xid) {
 		log.Printf("don't need thumb biter? %s", xid)
 		return false
 	}

database.go

@@ -515,6 +515,7 @@ var stmtThumbBiters, stmtDeleteHonk, stmtDeleteDonks, stmtDeleteOnts, stmtSaveZo
 var stmtGetZonkers, stmtRecentHonkers, stmtGetXonker, stmtSaveXonker, stmtDeleteXonker *sql.Stmt
 var stmtSelectOnts, stmtSaveOnt, stmtUpdateFlags, stmtClearFlags *sql.Stmt
 var stmtHonksForUserFirstClass, stmtSaveMeta, stmtDeleteMeta, stmtUpdateHonk *sql.Stmt
+var stmtGetFilters *sql.Stmt
 
 func preparetodie(db *sql.DB, s string) *sql.Stmt {
 	stmt, err := db.Prepare(s)
@@ -581,4 +582,5 @@ func prepareStatements(db *sql.DB) {
 	stmtUpdateFlags = preparetodie(db, "update honks set flags = flags | ? where honkid = ?")
 	stmtClearFlags = preparetodie(db, "update honks set flags = flags & ~ ? where honkid = ?")
 	stmtSelectOnts = preparetodie(db, "select distinct(ontology) from onts join honks on onts.honkid = honks.honkid where (honks.userid = ? or honks.whofore = 2)")
+	stmtGetFilters = preparetodie(db, "select hfcsid, json from hfcs where userid = ?")
 }

docs/changelog.txt

@@ -2,6 +2,8 @@ changelog
 
 -- next
 
++++ Add Honk Filtering and Censoring System (HFCS).
+
 + Inline images in received posts.
 
 + Times for events.

docs/manual.txt

@@ -53,11 +53,6 @@ You can zonk anything you like (or dislike), either your own honk or
 those of others that you're tired of seeing. Be advised that deletion
 works poorly in a federated environment. It's more like please disregard.
 
-The zonkzone supports muting unwanted contacts. One may mute an actor
-(zonker), a domain (zomain), thread (zonvoy), or word (zord).
-
-Posts can be collapsed, but not hidden, by specifying a zilence regex.
-
 -- privacy
 
 Posted honks are public. The federated environment lacks robust privacy

fun.go

@@ -35,7 +35,7 @@ import (
 func reverbolate(userid int64, honks []*Honk) {
 	filt := htfilter.New()
 	filt.Imager = replaceimg
-	zilences := getzilences(userid)
+	zilences := getfilters(userid, filtCollapse)
 	for _, h := range honks {
 		h.What += "ed"
 		if h.What == "tonked" {
@@ -76,7 +76,7 @@ func reverbolate(userid int64, honks []*Honk) {
 				h.Open = ""
 			}
 		} else {
-			if badword := unsee(zilences, h.Precis, h.Noise, h.Donks); badword != "" {
+			if badword := unsee(zilences, h); badword != "" {
 				if h.Precis == "" {
 					h.Precis = badword
 				}
@@ -179,50 +179,6 @@ func translate(honk *Honk) {
 	honk.Onts = oneofakind(ontologies(honk.Noise))
 }
 
-func unsee(zilences []*regexp.Regexp, precis string, noise string, donks []*Donk) string {
-	for _, z := range zilences {
-		if z.MatchString(precis) || z.MatchString(noise) {
-			if precis == "" {
-				w := z.String()
-				return w[6 : len(w)-3]
-			}
-			return precis
-		}
-		for _, d := range donks {
-			if z.MatchString(d.Desc) {
-				if precis == "" {
-					w := z.String()
-					return w[6 : len(w)-3]
-				}
-				return precis
-			}
-		}
-	}
-	return ""
-}
-
-func osmosis(honks []*Honk, userid int64) []*Honk {
-	zords := getzords(userid)
-	j := 0
-outer:
-	for _, h := range honks {
-		for _, z := range zords {
-			if z.MatchString(h.Precis) || z.MatchString(h.Noise) {
-				continue outer
-			}
-			for _, d := range h.Donks {
-				if z.MatchString(d.Desc) {
-					continue outer
-				}
-			}
-		}
-		honks[j] = h
-		j++
-	}
-	honks = honks[0:j]
-	return honks
-}
-
 func shortxid(xid string) string {
 	idx := strings.LastIndexByte(xid, '/')
 	if idx == -1 {
@@ -695,117 +651,6 @@ func makeitworksomehowwithoutregardforkeycontinuity(keyname string, r *http.Requ
 	return httpsig.VerifyRequest(r, payload, zaggy)
 }
 
-var thumbbiters map[int64]map[string]bool
-var zoggles map[int64]map[string]bool
-var zordses map[int64][]*regexp.Regexp
-var zilences map[int64][]*regexp.Regexp
-var thumblock sync.Mutex
-
-func bitethethumbs() {
-	rows, err := stmtThumbBiters.Query()
-	if err != nil {
-		log.Printf("error getting thumbbiters: %s", err)
-		return
-	}
-	defer rows.Close()
-
-	thumblock.Lock()
-	defer thumblock.Unlock()
-	thumbbiters = make(map[int64]map[string]bool)
-	zoggles = make(map[int64]map[string]bool)
-	zordses = make(map[int64][]*regexp.Regexp)
-	zilences = make(map[int64][]*regexp.Regexp)
-	for rows.Next() {
-		var userid int64
-		var name, wherefore string
-		err = rows.Scan(&userid, &name, &wherefore)
-		if err != nil {
-			log.Printf("error scanning zonker: %s", err)
-			continue
-		}
-		if wherefore == "zord" || wherefore == "zilence" {
-			zord := "\\b(?i:" + name + ")\\b"
-			re, err := regexp.Compile(zord)
-			if err != nil {
-				log.Printf("error compiling zord: %s", err)
-			} else {
-				if wherefore == "zord" {
-					zordses[userid] = append(zordses[userid], re)
-				} else {
-					zilences[userid] = append(zilences[userid], re)
-				}
-			}
-		}
-		if wherefore == "zoggle" {
-			m := zoggles[userid]
-			if m == nil {
-				m = make(map[string]bool)
-				zoggles[userid] = m
-			}
-			m[name] = true
-		}
-		if wherefore == "zonker" || wherefore == "zomain" {
-			m := thumbbiters[userid]
-			if m == nil {
-				m = make(map[string]bool)
-				thumbbiters[userid] = m
-			}
-			m[name] = true
-		}
-	}
-}
-
-func getzords(userid int64) []*regexp.Regexp {
-	thumblock.Lock()
-	defer thumblock.Unlock()
-	return zordses[userid]
-}
-
-func getzilences(userid int64) []*regexp.Regexp {
-	thumblock.Lock()
-	defer thumblock.Unlock()
-	return zilences[userid]
-}
-
-func thoudostbitethythumb(userid int64, who []string, objid string) bool {
-	thumblock.Lock()
-	biters := thumbbiters[userid]
-	thumblock.Unlock()
-	objwhere := originate(objid)
-	if objwhere != "" && biters[objwhere] {
-		log.Printf("thumbbiter: %s", objid)
-		return true
-	}
-	for _, w := range who {
-		if biters[w] {
-			log.Printf("thumbbiter: %s", w)
-			return true
-		}
-		where := originate(w)
-		if where != "" {
-			if biters[where] {
-				log.Printf("thumbbiter: %s", w)
-				return true
-			}
-		}
-	}
-	return false
-}
-
-func stealthmode(userid int64, r *http.Request) bool {
-	agent := r.UserAgent()
-	agent = originate(agent)
-	addr := r.Header.Get("X-Forwarded-For")
-	thumblock.Lock()
-	biters := thumbbiters[userid]
-	thumblock.Unlock()
-	fake := (agent != "" && biters[agent]) || (addr != "" && biters[addr])
-	if fake {
-		log.Printf("faking 404 for %s from %s", agent, addr)
-	}
-	return fake
-}
-
 func keymatch(keyname string, actor string) string {
 	hash := strings.IndexByte(keyname, '#')
 	if hash == -1 {

hfcs.go

@@ -17,16 +17,176 @@ package main
 
 import (
 	"log"
+	"net/http"
+	"regexp"
 )
 
-func skipMedia(xonk *Honk) bool {
-	thumblock.Lock()
-	goggles := zoggles[xonk.UserID]
-	thumblock.Unlock()
+type Filter struct {
+	ID              int64
+	Actor           string
+	IncludeAudience bool
+	Text            string
+	IsAnnounce      bool
+	Reject          bool
+	SkipMedia       bool
+	Hide            bool
+	Collapse        bool
+	Rewrite         string
+	re_rewrite      *regexp.Regexp
+	Replace         string
+}
 
-	if goggles[xonk.Honker] || goggles[xonk.Oonker] {
-		log.Printf("skipping media")
-		return true
+type filtType uint
+
+const (
+	filtNone filtType = iota
+	filtAny
+	filtReject
+	filtSkipMedia
+	filtHide
+	filtCollapse
+	filtRewrite
+)
+
+var filtNames = []string{"None", "Any", "Reject", "SkipMedia", "Hide", "Collapse", "Rewrite"}
+
+func (ft filtType) String() string {
+	return filtNames[ft]
+}
+
+type afiltermap map[filtType][]*Filter
+
+var filtcache = cacheNew(func(userid int64) (afiltermap, bool) {
+	rows, err := stmtGetFilters.Query(userid)
+	if err != nil {
+		log.Printf("error querying filters: %s", err)
+		return nil, false
+	}
+	defer rows.Close()
+
+	filtmap := make(afiltermap)
+	for rows.Next() {
+		filt := new(Filter)
+		var j string
+		var filterid int64
+		err = rows.Scan(&filterid, &j)
+		if err == nil {
+			err = unjsonify(j, filt)
+		}
+		if err != nil {
+			log.Printf("error scanning filter: %s", err)
+			continue
+		}
+		filt.ID = filterid
+		if filt.Reject {
+			filtmap[filtReject] = append(filtmap[filtReject], filt)
+		}
+		if filt.SkipMedia {
+			filtmap[filtSkipMedia] = append(filtmap[filtSkipMedia], filt)
+		}
+		if filt.Hide {
+			filtmap[filtHide] = append(filtmap[filtHide], filt)
+		}
+		if filt.Collapse {
+			filtmap[filtCollapse] = append(filtmap[filtCollapse], filt)
+		}
+		if filt.Rewrite != "" {
+			filtmap[filtRewrite] = append(filtmap[filtRewrite], filt)
+		}
+	}
+	return filtmap, true
+})
+
+func getfilters(userid int64, scope filtType) []*Filter {
+	var filtmap afiltermap
+	ok := filtcache.Get(userid, &filtmap)
+	if ok {
+		return filtmap[scope]
+	}
+	return nil
+}
+
+func rejectorigin(userid int64, origin string) bool {
+	if o := originate(origin); o != "" {
+		origin = o
+	}
+	filts := getfilters(userid, filtReject)
+	for _, f := range filts {
+		if f.Actor == origin {
+			log.Printf("rejecting origin: %s", origin)
+			return true
+		}
 	}
 	return false
 }
+
+func rejectactor(userid int64, actor string) bool {
+	origin := originate(actor)
+	filts := getfilters(userid, filtReject)
+	for _, f := range filts {
+		if f.Actor == actor || (origin != "" && f.Actor == origin) {
+			log.Printf("rejecting actor: %s", actor)
+			return true
+		}
+	}
+	return false
+}
+
+func stealthmode(userid int64, r *http.Request) bool {
+	agent := r.UserAgent()
+	agent = originate(agent)
+	if agent != "" {
+		fake := rejectorigin(userid, agent)
+		if fake {
+			log.Printf("faking 404 for %s", agent)
+			return fake
+		}
+	}
+	return false
+}
+
+func matchfilter(h *Honk, filts []*Filter) bool {
+	origin := originate(h.XID)
+	for _, f := range filts {
+		if f.Actor == origin || f.Actor == h.Honker {
+			return true
+		}
+		if f.Text != "" {
+			for _, d := range h.Donks {
+				if d.Desc == f.Text {
+					return true
+				}
+			}
+		}
+	}
+	return false
+}
+
+func rejectnote(xonk *Honk) bool {
+	filts := getfilters(xonk.UserID, filtReject)
+	return matchfilter(xonk, filts)
+}
+
+func skipMedia(xonk *Honk) bool {
+	filts := getfilters(xonk.UserID, filtSkipMedia)
+	return matchfilter(xonk, filts)
+}
+
+func unsee(filts []*Filter, h *Honk) string {
+	return ""
+}
+
+func osmosis(honks []*Honk, userid int64) []*Honk {
+	filts := getfilters(userid, filtHide)
+	j := 0
+outer:
+	for _, h := range honks {
+		if matchfilter(h, filts) {
+			continue outer
+		}
+		honks[j] = h
+		j++
+	}
+	honks = honks[0:j]
+	return honks
+}

schema.sql

@@ -8,6 +8,7 @@ create table zonkers (zonkerid integer primary key, userid integer, name text, w
 create table doovers(dooverid integer primary key, dt text, tries integer, username text, rcpt text, msg blob);
 create table onts (ontology text, honkid integer);
 create table honkmeta (honkid integer, genus text, json text);
+create table hfcs (hfcsid integer primary key, userid integer, json text);
 
 create index idx_honksxid on honks(xid);
 create index idx_honksconvoy on honks(convoy);
@@ -21,6 +22,7 @@ create index idx_filesurl on filemeta(url);
 create index idx_ontology on onts(ontology);
 create index idx_onthonkid on onts(honkid);
 create index idx_honkmetaid on honkmeta(honkid);
+create index idx_hfcsuser on hfcs(userid);
 
 create table config (key text, value text);
 

upgradedb.go

@@ -221,12 +221,11 @@ func upgradedb() {
 			var xid, media string
 			var data []byte
 			err = rows.Scan(&xid, &media, &data)
-			if err != nil {
-				log.Fatal(err)
+			if err == nil {
+				_, err = tx.Exec("insert into filedata (xid, media, content) values (?, ?, ?)", xid, media, data)
 			}
-			_, err = tx.Exec("insert into filedata (xid, media, content) values (?, ?, ?)", xid, media, data)
 			if err != nil {
-				log.Fatal(err)
+				log.Fatalf("can't save filedata: %s", err)
 			}
 		}
 		rows.Close()
@@ -263,8 +262,10 @@ func upgradedb() {
 		}
 		for honkid, p := range places {
 			j, err := jsonify(p)
-			_, err = tx.Exec("insert into honkmeta (honkid, genus, json) values (?, ?, ?)",
-				honkid, "place", j)
+			if err == nil {
+				_, err = tx.Exec("insert into honkmeta (honkid, genus, json) values (?, ?, ?)",
+					honkid, "place", j)
+			}
 			if err != nil {
 				log.Fatal(err)
 			}
@@ -276,6 +277,66 @@ func upgradedb() {
 		doordie(db, "update config set value = 23 where key = 'dbversion'")
 		fallthrough
 	case 23:
+		doordie(db, "create table hfcs (hfcsid integer primary key, userid integer, json text)")
+		doordie(db, "create index idx_hfcsuser on hfcs(userid)")
+		rows, err := db.Query("select userid, name, wherefore from zonkers where wherefore in ('zord', 'zilence', 'zoggle', 'zonker', 'zomain')")
+		if err != nil {
+			log.Fatalf("can't query zonkers: %s", err)
+		}
+		filtmap := make(map[int64][]*Filter)
+		for rows.Next() {
+			var userid int64
+			var name, wherefore string
+			err = rows.Scan(&userid, &name, &wherefore)
+			if err != nil {
+				log.Fatal("error scanning zonker: %s", err)
+			}
+			f := new(Filter)
+			switch wherefore {
+			case "zord":
+				f.Text = name
+				f.Hide = true
+			case "zilence":
+				f.Text = name
+				f.Collapse = true
+			case "zoggle":
+				f.Actor = name
+				f.SkipMedia = true
+			case "zonker":
+				f.Actor = name
+				f.IncludeAudience = true
+				f.Reject = true
+			case "zomain":
+				f.Actor = name
+				f.IncludeAudience = true
+				f.Reject = true
+			}
+			filtmap[userid] = append(filtmap[userid], f)
+		}
+		rows.Close()
+		tx, err := db.Begin()
+		if err != nil {
+			log.Fatalf("can't begin: %s", err)
+		}
+		for userid, filts := range filtmap {
+			for _, f := range filts {
+				j, err := jsonify(f)
+				if err == nil {
+					_, err = tx.Exec("insert into hfcs (userid, json) values (?, ?)", userid, j)
+				}
+				if err != nil {
+					log.Fatalf("can't save filter: %s", err)
+				}
+			}
+		}
+		err = tx.Commit()
+		if err != nil {
+			log.Fatalf("can't commit: %s", err)
+		}
+		doordie(db, "delete from zonkers where wherefore in ('zord', 'zilence', 'zoggle', 'zonker', 'zomain')")
+		doordie(db, "update config set value = 24 where key = 'dbversion'")
+		fallthrough
+	case 24:
 
 	default:
 		log.Fatalf("can't upgrade unknown version %d", dbversion)

util.go

@@ -73,7 +73,7 @@ var alreadyopendb *sql.DB
 var dbname = "honk.db"
 var blobdbname = "blob.db"
 var stmtConfig *sql.Stmt
-var myVersion = 23
+var myVersion = 24
 
 func initdb() {
 	schema, err := ioutil.ReadFile("schema.sql")

views/zonkers.html

@@ -2,37 +2,15 @@
 <main>
 <div class="info">
 <p>
-<form action="/zonkzonk" method="POST">
-<span class="title">it's zonking time!</span>
-<input type="hidden" name="CSRF" value="{{ .ZonkCSRF }}">
-<p>
-<input tabindex=1 type="text" name="name" value="" autocomplete=off> - name
-<p>
-<input type="radio" id="iszonker" name="wherefore" value="zonker">
-<label for="iszonker">Zonker</label>
-<p>
-<input type="radio" id="iszomain" name="wherefore" value="zomain">
-<label for="iszomain">Zomain</label>
-<p>
-<input type="radio" id="iszonvoy" name="wherefore" value="zonvoy">
-<label for="iszonvoy">Zonvoy</label>
-<p>
-<input type="radio" id="iszord" name="wherefore" value="zord">
-<label for="iszord">Zord</label>
-<p>
-<input type="radio" id="iszilence" name="wherefore" value="zilence">
-<label for="iszilence">Zilence</label>
-<p>
-<input type="radio" id="iszoggle" name="wherefore" value="zoggle">
-<label for="iszoggle">Zoggle</label>
-<p><br><button tabindex=1 name="zonk" value="zonk!">zonk!</button>
-</form>
+Work in progress
 </div>
 {{ $zonkcsrf := .ZonkCSRF }}
-{{ range .Zonkers }}
+{{ range $how, $filters := .Filters }}
+{{ range $filters }}
 <section class="honk">
-<p>What: {{ .Name }}
-<p>Where: {{ .Wherefore }}
+<p>How: {{ $how }}
+{{ with .Actor }}<p>Who: {{ . }}{{ end }}
+{{ with .Text }}<p>What: {{ . }}{{ end }}
 <form action="/zonkzonk" method="POST">
 <input type="hidden" name="CSRF" value="{{ $zonkcsrf }}">
 <input type="hidden" name="zonkerid" value="{{ .ID }}">
@@ -42,4 +20,5 @@
 <p>
 </section>
 {{ end }}
+{{ end }}
 </main>

web.go

@@ -307,8 +307,7 @@ func inbox(w http.ResponseWriter, r *http.Request) {
 		log.Printf("keyname actor mismatch: %s <> %s", keyname, who)
 		return
 	}
-	objid, _ := j.GetString("id")
-	if thoudostbitethythumb(user.ID, []string{who}, objid) {
+	if rejectactor(user.ID, who) {
 		log.Printf("ignoring thumb sucker %s", who)
 		return
 	}
@@ -1240,37 +1239,14 @@ func submithonker(w http.ResponseWriter, r *http.Request) {
 
 func zonkzone(w http.ResponseWriter, r *http.Request) {
 	userinfo := login.GetUserInfo(r)
-	rows, err := stmtGetZonkers.Query(userinfo.UserID)
-	if err != nil {
-		log.Printf("err: %s", err)
-		return
-	}
-	defer rows.Close()
-	var zonkers []Zonker
-	for rows.Next() {
-		var z Zonker
-		rows.Scan(&z.ID, &z.Name, &z.Wherefore)
-		zonkers = append(zonkers, z)
-	}
-	sort.Slice(zonkers, func(i, j int) bool {
-		w1 := zonkers[i].Wherefore
-		w2 := zonkers[j].Wherefore
-		if w1 == w2 {
-			return zonkers[i].Name < zonkers[j].Name
-		}
-		if w1 == "zonvoy" {
-			w1 = "zzzzzzz"
-		}
-		if w2 == "zonvoy" {
-			w2 = "zzzzzzz"
-		}
-		return w1 < w2
-	})
+
+	var filters afiltermap
+	filtcache.Get(userinfo.UserID, &filters)
 
 	templinfo := getInfo(r)
-	templinfo["Zonkers"] = zonkers
+	templinfo["Filters"] = filters
 	templinfo["ZonkCSRF"] = login.GetCSRF("zonkzonk", r)
-	err = readviews.Execute(w, "zonkers.html", templinfo)
+	err := readviews.Execute(w, "zonkers.html", templinfo)
 	if err != nil {
 		log.Print(err)
 	}
@@ -1282,9 +1258,9 @@ func zonkzonk(w http.ResponseWriter, r *http.Request) {
 	if itsok == "iforgiveyou" {
 		zonkerid, _ := strconv.ParseInt(r.FormValue("zonkerid"), 10, 0)
 		db := opendatabase()
-		db.Exec("delete from zonkers where userid = ? and zonkerid = ?",
+		db.Exec("delete from hfcs where userid = ? and hfcsid = ?",
 			userinfo.UserID, zonkerid)
-		bitethethumbs()
+		filtcache.Clear(userinfo.UserID)
 		http.Redirect(w, r, "/zonkzone", http.StatusSeeOther)
 		return
 	}
@@ -1308,7 +1284,7 @@ func zonkzonk(w http.ResponseWriter, r *http.Request) {
 		userinfo.UserID, name, wherefore)
 
 	if wherefore != "zonvoy" {
-		bitethethumbs()
+		filtcache.Clear(userinfo.UserID)
 	}
 
 	http.Redirect(w, r, "/zonkzone", http.StatusSeeOther)
@@ -1550,8 +1526,6 @@ func serve() {
 		}
 	}
 
-	bitethethumbs()
-
 	mux := mux.NewRouter()
 	mux.Use(login.Checker)