From 72a92a992ca0cedf89138df4f1f0ac9a96929f50 Mon Sep 17 00:00:00 2001 From: Ted Unangst Date: Wed, 5 Jun 2019 03:52:56 -0400 Subject: [PATCH] hoot support! --- docs/manual.txt | 5 +++ go.mod | 5 +-- go.sum | 7 ++-- honk.go | 1 + hoot.go | 87 +++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 101 insertions(+), 4 deletions(-) create mode 100644 hoot.go diff --git a/docs/manual.txt b/docs/manual.txt index 2fcb65f..998ef58 100644 --- a/docs/manual.txt +++ b/docs/manual.txt @@ -69,6 +69,11 @@ To save disk space and avoid repeated uploads, the memes directory may be prepopulated with bandwidth wasting reactions and referenced by meme: filename. Example: "Woah. meme: woahface.mp4" +-- hoots + +Link and inline a hoot from that other bird site. +hoot: https://twitter.com/tedunangst/status/839169710675611658 + -- cleanup One should occasionally run `honk cleanup` to free up internal space in the diff --git a/go.mod b/go.mod index 9709678..43d2791 100644 --- a/go.mod +++ b/go.mod @@ -1,10 +1,11 @@ module humungus.tedunangst.com/r/honk require ( + github.com/andybalholm/cascadia v1.0.0 github.com/gorilla/mux v1.7.2 github.com/mattn/go-runewidth v0.0.4 golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f - golang.org/x/net v0.0.0-20190522155817-f3200d17e092 // indirect + golang.org/x/net v0.0.0-20190522155817-f3200d17e092 humungus.tedunangst.com/r/go-sqlite3 v1.1.3 - humungus.tedunangst.com/r/webs v0.5.1 + humungus.tedunangst.com/r/webs v0.5.2 ) diff --git a/go.sum b/go.sum index fdf63c1..32c8efd 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/andybalholm/cascadia v1.0.0 h1:hOCXnnZ5A+3eVDX8pvgl4kofXv2ELss0bKcqRySc45o= +github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/gorilla/mux v1.7.2 h1:zoNxOV7WjqXptQOVngLmcSQgXmgk4NMz1HibBchjl/I= github.com/gorilla/mux v1.7.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y= @@ -7,6 +9,7 @@ golang.org/x/crypto v0.0.0-20190424203555-c05e17bb3b2d h1:adrbvkTDn9rGnXg2IJDKoz golang.org/x/crypto v0.0.0-20190424203555-c05e17bb3b2d/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f h1:R423Cnkcp5JABoeemiGEPlt9tHXFfw5kvc0yqlxRPWo= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 h1:0GoQqolDA55aaLxZyTzK/Y2ePZzZTUrRacwib7cNsYQ= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190522155817-f3200d17e092 h1:4QSRKanuywn15aTZvI/mIDEgPQpswuFndXpOj3rKEco= @@ -16,5 +19,5 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= humungus.tedunangst.com/r/go-sqlite3 v1.1.3 h1:G2N4wzDS0NbuvrZtQJhh4F+3X+s7BF8b9ga8k38geUI= humungus.tedunangst.com/r/go-sqlite3 v1.1.3/go.mod h1:FtEEmQM7U2Ey1TuEEOyY1BmphTZnmiEjPsNLEAkpf/M= -humungus.tedunangst.com/r/webs v0.5.1 h1:I57oIU/ZkKclixRtXs0nJQUWe3Gq+9QVqL7BtV2vzAA= -humungus.tedunangst.com/r/webs v0.5.1/go.mod h1:79Ww3HmgE1m+HXU0r0b9hkOD3JuDzXoGiEauHuKcwBI= +humungus.tedunangst.com/r/webs v0.5.2 h1:/HQ4xd33i1As3/5eD7RJXk7CewSg9WFa2HDBIkYlnxU= +humungus.tedunangst.com/r/webs v0.5.2/go.mod h1:79Ww3HmgE1m+HXU0r0b9hkOD3JuDzXoGiEauHuKcwBI= diff --git a/honk.go b/honk.go index 76bb5c0..1aef51c 100644 --- a/honk.go +++ b/honk.go @@ -821,6 +821,7 @@ func savehonk(w http.ResponseWriter, r *http.Request) { noise = noise[idx+1:] } } + noise = hooterize(noise) noise = strings.TrimSpace(noise) honk.Precis = strings.TrimSpace(honk.Precis) diff --git a/hoot.go b/hoot.go new file mode 100644 index 0000000..eb0fc26 --- /dev/null +++ b/hoot.go @@ -0,0 +1,87 @@ +package main + +import ( + "fmt" + "log" + "net/http" + "regexp" + "strings" + + "github.com/andybalholm/cascadia" + "golang.org/x/net/html" + "humungus.tedunangst.com/r/webs/htfilter" +) + +var tweetsel = cascadia.MustCompile("p.tweet-text") +var linksel = cascadia.MustCompile(".time a.tweet-timestamp") +var authorregex = regexp.MustCompile("twitter.com/([^/]+)") + +func hootfixer(hoot string) string { + url := hoot[5:] + if url[0] == ' ' { + url = url[1:] + } + url = strings.ReplaceAll(url, "mobile.twitter.com", "twitter.com") + log.Printf("hooterizing %s", url) + req, err := http.NewRequest("GET", url, nil) + if err != nil { + log.Printf("error: %s", err) + return hoot + } + req.Header.Set("User-Agent", "Mozilla/5.0 (X11; CrOS x86_64 11021.56.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.76 Safari/537.36") + req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8") + req.Header.Set("Accept-Language", "en-US,en;q=0.9") + resp, err := http.DefaultClient.Do(req) + if err != nil { + log.Printf("error: %s", err) + return hoot + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + log.Printf("error getting %s: %d", url, resp.StatusCode) + return hoot + } + + root, _ := html.Parse(resp.Body) + divs := tweetsel.MatchAll(root) + + authormatch := authorregex.FindStringSubmatch(url) + if len(authormatch) < 2 { + log.Printf("no author") + return hoot + } + wanted := authormatch[1] + var buf strings.Builder + + fmt.Fprintf(&buf, "hoot: %s\n", url) + for _, div := range divs { + twp := div.Parent.Parent.Parent + alink := linksel.MatchFirst(twp) + if alink == nil { + log.Printf("missing link") + continue + } + link := "https://twitter.com" + htfilter.GetAttr(alink, "href") + authormatch = authorregex.FindStringSubmatch(link) + if len(authormatch) < 2 { + log.Printf("no author?") + continue + } + author := authormatch[1] + if author != wanted { + continue + } + text := htfilter.TextOnly(div) + text = strings.ReplaceAll(text, "\n", " ") + text = strings.ReplaceAll(text, "pic.twitter.com", "https://pic.twitter.com") + + fmt.Fprintf(&buf, "> @%s: %s\n", author, text) + } + return buf.String() +} + +var re_hoots = regexp.MustCompile(`hoot: ?https://\S+`) + +func hooterize(noise string) string { + return re_hoots.ReplaceAllStringFunc(noise, hootfixer) +}