dirk
/
honk
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add timeout to last remaining http get

This commit is contained in:
Ted Unangst 2020-05-11 00:52:17 -04:00
parent 03720a9aa4
commit 4944996253
1 changed files with 12 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
activity.go
View File

@ -19,7 +19,6 @@ import (
"bytes" "bytes"
"context" "context"
"database/sql" "database/sql"
"errors"
"fmt" "fmt"
"html" "html"
"io" "io"
@ -146,14 +145,23 @@ func GetJunkTimeout(url string, timeout time.Duration) (junk.Junk, error) {
} }
func fetchsome(url string) ([]byte, error) { func fetchsome(url string) ([]byte, error) {
resp, err := http.Get(url) client := http.DefaultClient
req, err := http.NewRequest("GET", url, nil)
if err != nil {
log.Printf("error fetching %s: %s", url, err)
return nil, err
}
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
defer cancel()
req = req.WithContext(ctx)
resp, err := client.Do(req)
if err != nil { if err != nil {
log.Printf("error fetching %s: %s", url, err) log.Printf("error fetching %s: %s", url, err)
return nil, err return nil, err
} }
defer resp.Body.Close() defer resp.Body.Close()
if resp.StatusCode != 200 { if resp.StatusCode != 200 {
return nil, errors.New("not 200") return nil, fmt.Errorf("not 200: %d %s", resp.StatusCode, url)
} }
var buf bytes.Buffer var buf bytes.Buffer
limiter := io.LimitReader(resp.Body, 10*1024*1024) limiter := io.LimitReader(resp.Body, 10*1024*1024)