From de36230cecea43022f2d5bb72c3217db064513cc Mon Sep 17 00:00:00 2001 From: Ted Unangst Date: Fri, 4 Oct 2019 16:30:33 -0400 Subject: [PATCH] enough of the honk filtering and censoring system (hfcs) to get going. --- activity.go | 4 +- database.go | 2 + docs/changelog.txt | 2 + docs/manual.txt | 5 -- fun.go | 159 +---------------------------------------- hfcs.go | 174 +++++++++++++++++++++++++++++++++++++++++++-- schema.sql | 2 + upgradedb.go | 73 +++++++++++++++++-- util.go | 2 +- views/zonkers.html | 35 ++------- web.go | 44 +++--------- 11 files changed, 261 insertions(+), 241 deletions(-) diff --git a/activity.go b/activity.go index 0bd691e..cc77b94 100644 --- a/activity.go +++ b/activity.go @@ -203,7 +203,7 @@ func iszonked(userid int64, xid string) bool { } func needxonk(user *WhatAbout, x *Honk) bool { - if thoudostbitethythumb(user.ID, x.Audience, x.XID) { + if rejectnote(x) { log.Printf("not saving thumb biter? %s via %s", x.XID, x.Honker) return false } @@ -213,7 +213,7 @@ func needxonkid(user *WhatAbout, xid string) bool { if strings.HasPrefix(xid, user.URL+"/") { return false } - if thoudostbitethythumb(user.ID, nil, xid) { + if rejectorigin(user.ID, xid) { log.Printf("don't need thumb biter? %s", xid) return false } diff --git a/database.go b/database.go index 86cb946..212b24e 100644 --- a/database.go +++ b/database.go @@ -515,6 +515,7 @@ var stmtThumbBiters, stmtDeleteHonk, stmtDeleteDonks, stmtDeleteOnts, stmtSaveZo var stmtGetZonkers, stmtRecentHonkers, stmtGetXonker, stmtSaveXonker, stmtDeleteXonker *sql.Stmt var stmtSelectOnts, stmtSaveOnt, stmtUpdateFlags, stmtClearFlags *sql.Stmt var stmtHonksForUserFirstClass, stmtSaveMeta, stmtDeleteMeta, stmtUpdateHonk *sql.Stmt +var stmtGetFilters *sql.Stmt func preparetodie(db *sql.DB, s string) *sql.Stmt { stmt, err := db.Prepare(s) @@ -581,4 +582,5 @@ func prepareStatements(db *sql.DB) { stmtUpdateFlags = preparetodie(db, "update honks set flags = flags | ? where honkid = ?") stmtClearFlags = preparetodie(db, "update honks set flags = flags & ~ ? where honkid = ?") stmtSelectOnts = preparetodie(db, "select distinct(ontology) from onts join honks on onts.honkid = honks.honkid where (honks.userid = ? or honks.whofore = 2)") + stmtGetFilters = preparetodie(db, "select hfcsid, json from hfcs where userid = ?") } diff --git a/docs/changelog.txt b/docs/changelog.txt index 205a96a..a38e178 100644 --- a/docs/changelog.txt +++ b/docs/changelog.txt @@ -2,6 +2,8 @@ changelog -- next ++++ Add Honk Filtering and Censoring System (HFCS). + + Inline images in received posts. + Times for events. diff --git a/docs/manual.txt b/docs/manual.txt index b3179ce..ca855ad 100644 --- a/docs/manual.txt +++ b/docs/manual.txt @@ -53,11 +53,6 @@ You can zonk anything you like (or dislike), either your own honk or those of others that you're tired of seeing. Be advised that deletion works poorly in a federated environment. It's more like please disregard. -The zonkzone supports muting unwanted contacts. One may mute an actor -(zonker), a domain (zomain), thread (zonvoy), or word (zord). - -Posts can be collapsed, but not hidden, by specifying a zilence regex. - -- privacy Posted honks are public. The federated environment lacks robust privacy diff --git a/fun.go b/fun.go index c21d11f..19acb39 100644 --- a/fun.go +++ b/fun.go @@ -35,7 +35,7 @@ import ( func reverbolate(userid int64, honks []*Honk) { filt := htfilter.New() filt.Imager = replaceimg - zilences := getzilences(userid) + zilences := getfilters(userid, filtCollapse) for _, h := range honks { h.What += "ed" if h.What == "tonked" { @@ -76,7 +76,7 @@ func reverbolate(userid int64, honks []*Honk) { h.Open = "" } } else { - if badword := unsee(zilences, h.Precis, h.Noise, h.Donks); badword != "" { + if badword := unsee(zilences, h); badword != "" { if h.Precis == "" { h.Precis = badword } @@ -179,50 +179,6 @@ func translate(honk *Honk) { honk.Onts = oneofakind(ontologies(honk.Noise)) } -func unsee(zilences []*regexp.Regexp, precis string, noise string, donks []*Donk) string { - for _, z := range zilences { - if z.MatchString(precis) || z.MatchString(noise) { - if precis == "" { - w := z.String() - return w[6 : len(w)-3] - } - return precis - } - for _, d := range donks { - if z.MatchString(d.Desc) { - if precis == "" { - w := z.String() - return w[6 : len(w)-3] - } - return precis - } - } - } - return "" -} - -func osmosis(honks []*Honk, userid int64) []*Honk { - zords := getzords(userid) - j := 0 -outer: - for _, h := range honks { - for _, z := range zords { - if z.MatchString(h.Precis) || z.MatchString(h.Noise) { - continue outer - } - for _, d := range h.Donks { - if z.MatchString(d.Desc) { - continue outer - } - } - } - honks[j] = h - j++ - } - honks = honks[0:j] - return honks -} - func shortxid(xid string) string { idx := strings.LastIndexByte(xid, '/') if idx == -1 { @@ -695,117 +651,6 @@ func makeitworksomehowwithoutregardforkeycontinuity(keyname string, r *http.Requ return httpsig.VerifyRequest(r, payload, zaggy) } -var thumbbiters map[int64]map[string]bool -var zoggles map[int64]map[string]bool -var zordses map[int64][]*regexp.Regexp -var zilences map[int64][]*regexp.Regexp -var thumblock sync.Mutex - -func bitethethumbs() { - rows, err := stmtThumbBiters.Query() - if err != nil { - log.Printf("error getting thumbbiters: %s", err) - return - } - defer rows.Close() - - thumblock.Lock() - defer thumblock.Unlock() - thumbbiters = make(map[int64]map[string]bool) - zoggles = make(map[int64]map[string]bool) - zordses = make(map[int64][]*regexp.Regexp) - zilences = make(map[int64][]*regexp.Regexp) - for rows.Next() { - var userid int64 - var name, wherefore string - err = rows.Scan(&userid, &name, &wherefore) - if err != nil { - log.Printf("error scanning zonker: %s", err) - continue - } - if wherefore == "zord" || wherefore == "zilence" { - zord := "\\b(?i:" + name + ")\\b" - re, err := regexp.Compile(zord) - if err != nil { - log.Printf("error compiling zord: %s", err) - } else { - if wherefore == "zord" { - zordses[userid] = append(zordses[userid], re) - } else { - zilences[userid] = append(zilences[userid], re) - } - } - } - if wherefore == "zoggle" { - m := zoggles[userid] - if m == nil { - m = make(map[string]bool) - zoggles[userid] = m - } - m[name] = true - } - if wherefore == "zonker" || wherefore == "zomain" { - m := thumbbiters[userid] - if m == nil { - m = make(map[string]bool) - thumbbiters[userid] = m - } - m[name] = true - } - } -} - -func getzords(userid int64) []*regexp.Regexp { - thumblock.Lock() - defer thumblock.Unlock() - return zordses[userid] -} - -func getzilences(userid int64) []*regexp.Regexp { - thumblock.Lock() - defer thumblock.Unlock() - return zilences[userid] -} - -func thoudostbitethythumb(userid int64, who []string, objid string) bool { - thumblock.Lock() - biters := thumbbiters[userid] - thumblock.Unlock() - objwhere := originate(objid) - if objwhere != "" && biters[objwhere] { - log.Printf("thumbbiter: %s", objid) - return true - } - for _, w := range who { - if biters[w] { - log.Printf("thumbbiter: %s", w) - return true - } - where := originate(w) - if where != "" { - if biters[where] { - log.Printf("thumbbiter: %s", w) - return true - } - } - } - return false -} - -func stealthmode(userid int64, r *http.Request) bool { - agent := r.UserAgent() - agent = originate(agent) - addr := r.Header.Get("X-Forwarded-For") - thumblock.Lock() - biters := thumbbiters[userid] - thumblock.Unlock() - fake := (agent != "" && biters[agent]) || (addr != "" && biters[addr]) - if fake { - log.Printf("faking 404 for %s from %s", agent, addr) - } - return fake -} - func keymatch(keyname string, actor string) string { hash := strings.IndexByte(keyname, '#') if hash == -1 { diff --git a/hfcs.go b/hfcs.go index 4b550da..1c55c8f 100644 --- a/hfcs.go +++ b/hfcs.go @@ -17,16 +17,176 @@ package main import ( "log" + "net/http" + "regexp" ) -func skipMedia(xonk *Honk) bool { - thumblock.Lock() - goggles := zoggles[xonk.UserID] - thumblock.Unlock() +type Filter struct { + ID int64 + Actor string + IncludeAudience bool + Text string + IsAnnounce bool + Reject bool + SkipMedia bool + Hide bool + Collapse bool + Rewrite string + re_rewrite *regexp.Regexp + Replace string +} - if goggles[xonk.Honker] || goggles[xonk.Oonker] { - log.Printf("skipping media") - return true +type filtType uint + +const ( + filtNone filtType = iota + filtAny + filtReject + filtSkipMedia + filtHide + filtCollapse + filtRewrite +) + +var filtNames = []string{"None", "Any", "Reject", "SkipMedia", "Hide", "Collapse", "Rewrite"} + +func (ft filtType) String() string { + return filtNames[ft] +} + +type afiltermap map[filtType][]*Filter + +var filtcache = cacheNew(func(userid int64) (afiltermap, bool) { + rows, err := stmtGetFilters.Query(userid) + if err != nil { + log.Printf("error querying filters: %s", err) + return nil, false + } + defer rows.Close() + + filtmap := make(afiltermap) + for rows.Next() { + filt := new(Filter) + var j string + var filterid int64 + err = rows.Scan(&filterid, &j) + if err == nil { + err = unjsonify(j, filt) + } + if err != nil { + log.Printf("error scanning filter: %s", err) + continue + } + filt.ID = filterid + if filt.Reject { + filtmap[filtReject] = append(filtmap[filtReject], filt) + } + if filt.SkipMedia { + filtmap[filtSkipMedia] = append(filtmap[filtSkipMedia], filt) + } + if filt.Hide { + filtmap[filtHide] = append(filtmap[filtHide], filt) + } + if filt.Collapse { + filtmap[filtCollapse] = append(filtmap[filtCollapse], filt) + } + if filt.Rewrite != "" { + filtmap[filtRewrite] = append(filtmap[filtRewrite], filt) + } + } + return filtmap, true +}) + +func getfilters(userid int64, scope filtType) []*Filter { + var filtmap afiltermap + ok := filtcache.Get(userid, &filtmap) + if ok { + return filtmap[scope] + } + return nil +} + +func rejectorigin(userid int64, origin string) bool { + if o := originate(origin); o != "" { + origin = o + } + filts := getfilters(userid, filtReject) + for _, f := range filts { + if f.Actor == origin { + log.Printf("rejecting origin: %s", origin) + return true + } } return false } + +func rejectactor(userid int64, actor string) bool { + origin := originate(actor) + filts := getfilters(userid, filtReject) + for _, f := range filts { + if f.Actor == actor || (origin != "" && f.Actor == origin) { + log.Printf("rejecting actor: %s", actor) + return true + } + } + return false +} + +func stealthmode(userid int64, r *http.Request) bool { + agent := r.UserAgent() + agent = originate(agent) + if agent != "" { + fake := rejectorigin(userid, agent) + if fake { + log.Printf("faking 404 for %s", agent) + return fake + } + } + return false +} + +func matchfilter(h *Honk, filts []*Filter) bool { + origin := originate(h.XID) + for _, f := range filts { + if f.Actor == origin || f.Actor == h.Honker { + return true + } + if f.Text != "" { + for _, d := range h.Donks { + if d.Desc == f.Text { + return true + } + } + } + } + return false +} + +func rejectnote(xonk *Honk) bool { + filts := getfilters(xonk.UserID, filtReject) + return matchfilter(xonk, filts) +} + +func skipMedia(xonk *Honk) bool { + filts := getfilters(xonk.UserID, filtSkipMedia) + return matchfilter(xonk, filts) +} + +func unsee(filts []*Filter, h *Honk) string { + return "" +} + +func osmosis(honks []*Honk, userid int64) []*Honk { + filts := getfilters(userid, filtHide) + j := 0 +outer: + for _, h := range honks { + if matchfilter(h, filts) { + continue outer + } + honks[j] = h + j++ + } + honks = honks[0:j] + return honks +} diff --git a/schema.sql b/schema.sql index f9f8273..e28b090 100644 --- a/schema.sql +++ b/schema.sql @@ -8,6 +8,7 @@ create table zonkers (zonkerid integer primary key, userid integer, name text, w create table doovers(dooverid integer primary key, dt text, tries integer, username text, rcpt text, msg blob); create table onts (ontology text, honkid integer); create table honkmeta (honkid integer, genus text, json text); +create table hfcs (hfcsid integer primary key, userid integer, json text); create index idx_honksxid on honks(xid); create index idx_honksconvoy on honks(convoy); @@ -21,6 +22,7 @@ create index idx_filesurl on filemeta(url); create index idx_ontology on onts(ontology); create index idx_onthonkid on onts(honkid); create index idx_honkmetaid on honkmeta(honkid); +create index idx_hfcsuser on hfcs(userid); create table config (key text, value text); diff --git a/upgradedb.go b/upgradedb.go index 3e901f6..35be9d7 100644 --- a/upgradedb.go +++ b/upgradedb.go @@ -221,12 +221,11 @@ func upgradedb() { var xid, media string var data []byte err = rows.Scan(&xid, &media, &data) - if err != nil { - log.Fatal(err) + if err == nil { + _, err = tx.Exec("insert into filedata (xid, media, content) values (?, ?, ?)", xid, media, data) } - _, err = tx.Exec("insert into filedata (xid, media, content) values (?, ?, ?)", xid, media, data) if err != nil { - log.Fatal(err) + log.Fatalf("can't save filedata: %s", err) } } rows.Close() @@ -263,8 +262,10 @@ func upgradedb() { } for honkid, p := range places { j, err := jsonify(p) - _, err = tx.Exec("insert into honkmeta (honkid, genus, json) values (?, ?, ?)", - honkid, "place", j) + if err == nil { + _, err = tx.Exec("insert into honkmeta (honkid, genus, json) values (?, ?, ?)", + honkid, "place", j) + } if err != nil { log.Fatal(err) } @@ -276,6 +277,66 @@ func upgradedb() { doordie(db, "update config set value = 23 where key = 'dbversion'") fallthrough case 23: + doordie(db, "create table hfcs (hfcsid integer primary key, userid integer, json text)") + doordie(db, "create index idx_hfcsuser on hfcs(userid)") + rows, err := db.Query("select userid, name, wherefore from zonkers where wherefore in ('zord', 'zilence', 'zoggle', 'zonker', 'zomain')") + if err != nil { + log.Fatalf("can't query zonkers: %s", err) + } + filtmap := make(map[int64][]*Filter) + for rows.Next() { + var userid int64 + var name, wherefore string + err = rows.Scan(&userid, &name, &wherefore) + if err != nil { + log.Fatal("error scanning zonker: %s", err) + } + f := new(Filter) + switch wherefore { + case "zord": + f.Text = name + f.Hide = true + case "zilence": + f.Text = name + f.Collapse = true + case "zoggle": + f.Actor = name + f.SkipMedia = true + case "zonker": + f.Actor = name + f.IncludeAudience = true + f.Reject = true + case "zomain": + f.Actor = name + f.IncludeAudience = true + f.Reject = true + } + filtmap[userid] = append(filtmap[userid], f) + } + rows.Close() + tx, err := db.Begin() + if err != nil { + log.Fatalf("can't begin: %s", err) + } + for userid, filts := range filtmap { + for _, f := range filts { + j, err := jsonify(f) + if err == nil { + _, err = tx.Exec("insert into hfcs (userid, json) values (?, ?)", userid, j) + } + if err != nil { + log.Fatalf("can't save filter: %s", err) + } + } + } + err = tx.Commit() + if err != nil { + log.Fatalf("can't commit: %s", err) + } + doordie(db, "delete from zonkers where wherefore in ('zord', 'zilence', 'zoggle', 'zonker', 'zomain')") + doordie(db, "update config set value = 24 where key = 'dbversion'") + fallthrough + case 24: default: log.Fatalf("can't upgrade unknown version %d", dbversion) diff --git a/util.go b/util.go index f0fdb4d..b0e4c1d 100644 --- a/util.go +++ b/util.go @@ -73,7 +73,7 @@ var alreadyopendb *sql.DB var dbname = "honk.db" var blobdbname = "blob.db" var stmtConfig *sql.Stmt -var myVersion = 23 +var myVersion = 24 func initdb() { schema, err := ioutil.ReadFile("schema.sql") diff --git a/views/zonkers.html b/views/zonkers.html index f5389b3..1a15b3c 100644 --- a/views/zonkers.html +++ b/views/zonkers.html @@ -2,37 +2,15 @@

-

-it's zonking time! - -

- - name -

- - -

- - -

- - -

- - -

- - -

- - -


-

+Work in progress
{{ $zonkcsrf := .ZonkCSRF }} -{{ range .Zonkers }} +{{ range $how, $filters := .Filters }} +{{ range $filters }}
-

What: {{ .Name }} -

Where: {{ .Wherefore }} +

How: {{ $how }} +{{ with .Actor }}

Who: {{ . }}{{ end }} +{{ with .Text }}

What: {{ . }}{{ end }}

@@ -42,4 +20,5 @@

{{ end }} +{{ end }}
diff --git a/web.go b/web.go index 7b35450..200c30b 100644 --- a/web.go +++ b/web.go @@ -307,8 +307,7 @@ func inbox(w http.ResponseWriter, r *http.Request) { log.Printf("keyname actor mismatch: %s <> %s", keyname, who) return } - objid, _ := j.GetString("id") - if thoudostbitethythumb(user.ID, []string{who}, objid) { + if rejectactor(user.ID, who) { log.Printf("ignoring thumb sucker %s", who) return } @@ -1240,37 +1239,14 @@ func submithonker(w http.ResponseWriter, r *http.Request) { func zonkzone(w http.ResponseWriter, r *http.Request) { userinfo := login.GetUserInfo(r) - rows, err := stmtGetZonkers.Query(userinfo.UserID) - if err != nil { - log.Printf("err: %s", err) - return - } - defer rows.Close() - var zonkers []Zonker - for rows.Next() { - var z Zonker - rows.Scan(&z.ID, &z.Name, &z.Wherefore) - zonkers = append(zonkers, z) - } - sort.Slice(zonkers, func(i, j int) bool { - w1 := zonkers[i].Wherefore - w2 := zonkers[j].Wherefore - if w1 == w2 { - return zonkers[i].Name < zonkers[j].Name - } - if w1 == "zonvoy" { - w1 = "zzzzzzz" - } - if w2 == "zonvoy" { - w2 = "zzzzzzz" - } - return w1 < w2 - }) + + var filters afiltermap + filtcache.Get(userinfo.UserID, &filters) templinfo := getInfo(r) - templinfo["Zonkers"] = zonkers + templinfo["Filters"] = filters templinfo["ZonkCSRF"] = login.GetCSRF("zonkzonk", r) - err = readviews.Execute(w, "zonkers.html", templinfo) + err := readviews.Execute(w, "zonkers.html", templinfo) if err != nil { log.Print(err) } @@ -1282,9 +1258,9 @@ func zonkzonk(w http.ResponseWriter, r *http.Request) { if itsok == "iforgiveyou" { zonkerid, _ := strconv.ParseInt(r.FormValue("zonkerid"), 10, 0) db := opendatabase() - db.Exec("delete from zonkers where userid = ? and zonkerid = ?", + db.Exec("delete from hfcs where userid = ? and hfcsid = ?", userinfo.UserID, zonkerid) - bitethethumbs() + filtcache.Clear(userinfo.UserID) http.Redirect(w, r, "/zonkzone", http.StatusSeeOther) return } @@ -1308,7 +1284,7 @@ func zonkzonk(w http.ResponseWriter, r *http.Request) { userinfo.UserID, name, wherefore) if wherefore != "zonvoy" { - bitethethumbs() + filtcache.Clear(userinfo.UserID) } http.Redirect(w, r, "/zonkzone", http.StatusSeeOther) @@ -1550,8 +1526,6 @@ func serve() { } } - bitethethumbs() - mux := mux.NewRouter() mux.Use(login.Checker)