hoot support!

docs/manual.txt

@@ -69,6 +69,11 @@ To save disk space and avoid repeated uploads, the memes directory may be
 prepopulated with bandwidth wasting reactions and referenced by meme: filename.
 Example: "Woah. meme: woahface.mp4"
 
+-- hoots
+
+Link and inline a hoot from that other bird site.
+hoot: https://twitter.com/tedunangst/status/839169710675611658
+
 -- cleanup
 
 One should occasionally run `honk cleanup` to free up internal space in the

go.mod

@@ -1,10 +1,11 @@
 module humungus.tedunangst.com/r/honk
 
 require (
+	github.com/andybalholm/cascadia v1.0.0
 	github.com/gorilla/mux v1.7.2
 	github.com/mattn/go-runewidth v0.0.4
 	golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f
-	golang.org/x/net v0.0.0-20190522155817-f3200d17e092 // indirect
+	golang.org/x/net v0.0.0-20190522155817-f3200d17e092
 	humungus.tedunangst.com/r/go-sqlite3 v1.1.3
-	humungus.tedunangst.com/r/webs v0.5.1
+	humungus.tedunangst.com/r/webs v0.5.2
 )

go.sum

@@ -1,3 +1,5 @@
+github.com/andybalholm/cascadia v1.0.0 h1:hOCXnnZ5A+3eVDX8pvgl4kofXv2ELss0bKcqRySc45o=
+github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
 github.com/gorilla/mux v1.7.2 h1:zoNxOV7WjqXptQOVngLmcSQgXmgk4NMz1HibBchjl/I=
 github.com/gorilla/mux v1.7.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
 github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y=
@@ -7,6 +9,7 @@ golang.org/x/crypto v0.0.0-20190424203555-c05e17bb3b2d h1:adrbvkTDn9rGnXg2IJDKoz
 golang.org/x/crypto v0.0.0-20190424203555-c05e17bb3b2d/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f h1:R423Cnkcp5JABoeemiGEPlt9tHXFfw5kvc0yqlxRPWo=
 golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 h1:0GoQqolDA55aaLxZyTzK/Y2ePZzZTUrRacwib7cNsYQ=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190522155817-f3200d17e092 h1:4QSRKanuywn15aTZvI/mIDEgPQpswuFndXpOj3rKEco=
@@ -16,5 +19,5 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 humungus.tedunangst.com/r/go-sqlite3 v1.1.3 h1:G2N4wzDS0NbuvrZtQJhh4F+3X+s7BF8b9ga8k38geUI=
 humungus.tedunangst.com/r/go-sqlite3 v1.1.3/go.mod h1:FtEEmQM7U2Ey1TuEEOyY1BmphTZnmiEjPsNLEAkpf/M=
-humungus.tedunangst.com/r/webs v0.5.1 h1:I57oIU/ZkKclixRtXs0nJQUWe3Gq+9QVqL7BtV2vzAA=
+humungus.tedunangst.com/r/webs v0.5.2 h1:/HQ4xd33i1As3/5eD7RJXk7CewSg9WFa2HDBIkYlnxU=
-humungus.tedunangst.com/r/webs v0.5.1/go.mod h1:79Ww3HmgE1m+HXU0r0b9hkOD3JuDzXoGiEauHuKcwBI=
+humungus.tedunangst.com/r/webs v0.5.2/go.mod h1:79Ww3HmgE1m+HXU0r0b9hkOD3JuDzXoGiEauHuKcwBI=

honk.go

@@ -821,6 +821,7 @@ func savehonk(w http.ResponseWriter, r *http.Request) {
 			noise = noise[idx+1:]
 		}
 	}
+	noise = hooterize(noise)
 	noise = strings.TrimSpace(noise)
 	honk.Precis = strings.TrimSpace(honk.Precis)
 

hoot.go

@@ -0,0 +1,87 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"net/http"
+	"regexp"
+	"strings"
+
+	"github.com/andybalholm/cascadia"
+	"golang.org/x/net/html"
+	"humungus.tedunangst.com/r/webs/htfilter"
+)
+
+var tweetsel = cascadia.MustCompile("p.tweet-text")
+var linksel = cascadia.MustCompile(".time a.tweet-timestamp")
+var authorregex = regexp.MustCompile("twitter.com/([^/]+)")
+
+func hootfixer(hoot string) string {
+	url := hoot[5:]
+	if url[0] == ' ' {
+		url = url[1:]
+	}
+	url = strings.ReplaceAll(url, "mobile.twitter.com", "twitter.com")
+	log.Printf("hooterizing %s", url)
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		log.Printf("error: %s", err)
+		return hoot
+	}
+	req.Header.Set("User-Agent", "Mozilla/5.0 (X11; CrOS x86_64 11021.56.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.76 Safari/537.36")
+	req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
+	req.Header.Set("Accept-Language", "en-US,en;q=0.9")
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		log.Printf("error: %s", err)
+		return hoot
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != 200 {
+		log.Printf("error getting %s: %d", url, resp.StatusCode)
+		return hoot
+	}
+
+	root, _ := html.Parse(resp.Body)
+	divs := tweetsel.MatchAll(root)
+
+	authormatch := authorregex.FindStringSubmatch(url)
+	if len(authormatch) < 2 {
+		log.Printf("no author")
+		return hoot
+	}
+	wanted := authormatch[1]
+	var buf strings.Builder
+
+	fmt.Fprintf(&buf, "hoot: %s\n", url)
+	for _, div := range divs {
+		twp := div.Parent.Parent.Parent
+		alink := linksel.MatchFirst(twp)
+		if alink == nil {
+			log.Printf("missing link")
+			continue
+		}
+		link := "https://twitter.com" + htfilter.GetAttr(alink, "href")
+		authormatch = authorregex.FindStringSubmatch(link)
+		if len(authormatch) < 2 {
+			log.Printf("no author?")
+			continue
+		}
+		author := authormatch[1]
+		if author != wanted {
+			continue
+		}
+		text := htfilter.TextOnly(div)
+		text = strings.ReplaceAll(text, "\n", " ")
+		text = strings.ReplaceAll(text, "pic.twitter.com", "https://pic.twitter.com")
+
+		fmt.Fprintf(&buf, "> @%s: %s\n", author, text)
+	}
+	return buf.String()
+}
+
+var re_hoots = regexp.MustCompile(`hoot: ?https://\S+`)
+
+func hooterize(noise string) string {
+	return re_hoots.ReplaceAllStringFunc(noise, hootfixer)
+}