avoid some unnessary requests by fully processing what we get

activity.go

@@ -275,28 +275,22 @@ var boxofboxes = cache.New(cache.Options{Filler: func(ident string) (*Box, bool)
 	var info string
 	row := stmtGetXonker.QueryRow(ident, "boxes")
 	err := row.Scan(&info)
-	if err == nil {
+	if err != nil {
-		m := strings.Split(info, " ")
-		b := &Box{In: m[0], Out: m[1], Shared: m[2]}
-		return b, true
-	}
 		j, err := GetJunk(ident)
 		if err != nil {
 			log.Printf("error getting boxes: %s", err)
 			return nil, false
 		}
-	inbox, _ := j.GetString("inbox")
+		allinjest(originate(ident), j)
-	outbox, _ := j.GetString("outbox")
+		row = stmtGetXonker.QueryRow(ident, "boxes")
-	sbox, _ := j.GetString("endpoints", "sharedInbox")
+		err = row.Scan(&info)
-	b := &Box{In: inbox, Out: outbox, Shared: sbox}
-	if inbox != "" {
-		m := strings.Join([]string{inbox, outbox, sbox}, " ")
-		_, err = stmtSaveXonker.Exec(ident, m, "boxes")
-		if err != nil {
-			log.Printf("error saving boxes: %s", err)
-		}
 	}
+	if err == nil {
+		m := strings.Split(info, " ")
+		b := &Box{In: m[0], Out: m[1], Shared: m[2]}
 		return b, true
+	}
+	return nil, false
 }})
 
 func gimmexonks(user *WhatAbout, outbox string) {
@@ -1354,6 +1348,7 @@ func investigate(name string) (*SomeThing, error) {
 	if err != nil {
 		return nil, err
 	}
+	allinjest(originate(name), obj)
 	return somethingabout(obj)
 }
 
@@ -1387,3 +1382,78 @@ func somethingabout(obj junk.Junk) (*SomeThing, error) {
 	}
 	return info, nil
 }
+
+func allinjest(origin string, obj junk.Junk) {
+	keyobj, ok := obj.GetMap("publicKey")
+	if ok {
+		ingestpubkey(origin, keyobj)
+	}
+	ingestboxes(origin, obj)
+}
+
+func ingestpubkey(origin string, obj junk.Junk) {
+	keyobj, ok := obj.GetMap("publicKey")
+	if ok {
+		obj = keyobj
+	}
+	keyname, ok := obj.GetString("id")
+	row := stmtGetXonker.QueryRow(keyname, "pubkey")
+	var data string
+	err := row.Scan(&data)
+	if err == nil {
+		return
+	}
+	if !ok || origin != originate(keyname) {
+		log.Printf("bad key origin %s <> %s", origin, keyname)
+		return
+	}
+	log.Printf("ingesting a needed pubkey: %s", keyname)
+	owner, ok := obj.GetString("owner")
+	if !ok {
+		log.Printf("error finding %s pubkey owner", keyname)
+		return
+	}
+	data, ok = obj.GetString("publicKeyPem")
+	if !ok {
+		log.Printf("error finding %s pubkey", keyname)
+		return
+	}
+	if originate(owner) != origin {
+		log.Printf("bad key owner: %s <> %s", owner, origin)
+		return
+	}
+	_, _, err = httpsig.DecodeKey(data)
+	if err != nil {
+		log.Printf("error decoding %s pubkey: %s", keyname, err)
+		return
+	}
+	_, err = stmtSaveXonker.Exec(keyname, data, "pubkey")
+	if err != nil {
+		log.Printf("error saving key: %s", err)
+	}
+	return
+}
+
+func ingestboxes(origin string, obj junk.Junk) {
+	ident, _ := obj.GetString("id")
+	if ident == "" {
+		return
+	}
+	var info string
+	row := stmtGetXonker.QueryRow(ident, "boxes")
+	err := row.Scan(&info)
+	if err == nil {
+		return
+	}
+	log.Printf("ingesting boxes: %s", ident)
+	inbox, _ := obj.GetString("inbox")
+	outbox, _ := obj.GetString("outbox")
+	sbox, _ := obj.GetString("endpoints", "sharedInbox")
+	if inbox != "" {
+		m := strings.Join([]string{inbox, outbox, sbox}, " ")
+		_, err = stmtSaveXonker.Exec(ident, m, "boxes")
+		if err != nil {
+			log.Printf("error saving boxes: %s", err)
+		}
+	}
+}

fun.go

@@ -587,6 +587,17 @@ var zaggies = cache.New(cache.Options{Filler: func(keyname string) (*rsa.PublicK
 	row := stmtGetXonker.QueryRow(keyname, "pubkey")
 	var data string
 	err := row.Scan(&data)
+	if err != nil {
+		log.Printf("hitting the webs for missing pubkey: %s", keyname)
+		j, err := GetJunk(keyname)
+		if err != nil {
+			log.Printf("error getting %s pubkey: %s", keyname, err)
+			return nil, true
+		}
+		allinjest(originate(keyname), j)
+		row = stmtGetXonker.QueryRow(keyname, "pubkey")
+		err = row.Scan(&data)
+	}
 	if err == nil {
 		_, key, err := httpsig.DecodeKey(data)
 		if err != nil {
@@ -594,36 +605,7 @@ var zaggies = cache.New(cache.Options{Filler: func(keyname string) (*rsa.PublicK
 		}
 		return key, true
 	}
-	log.Printf("hitting the webs for missing pubkey: %s", keyname)
-	j, err := GetJunk(keyname)
-	if err != nil {
-		log.Printf("error getting %s pubkey: %s", keyname, err)
 	return nil, true
-	}
-	keyobj, ok := j.GetMap("publicKey")
-	if ok {
-		j = keyobj
-	}
-	data, ok = j.GetString("publicKeyPem")
-	if !ok {
-		log.Printf("error finding %s pubkey", keyname)
-		return nil, true
-	}
-	_, ok = j.GetString("owner")
-	if !ok {
-		log.Printf("error finding %s pubkey owner", keyname)
-		return nil, true
-	}
-	_, key, err := httpsig.DecodeKey(data)
-	if err != nil {
-		log.Printf("error decoding %s pubkey: %s", keyname, err)
-		return nil, true
-	}
-	_, err = stmtSaveXonker.Exec(keyname, data, "pubkey")
-	if err != nil {
-		log.Printf("error saving key: %s", err)
-	}
-	return key, true
 }})
 
 func zaggy(keyname string) *rsa.PublicKey {

web.go

@@ -554,6 +554,7 @@ func ximport(w http.ResponseWriter, r *http.Request) {
 			log.Printf("error getting external object: %s", err)
 			return
 		}
+		allinjest(originate(xid), j)
 		log.Printf("importing %s", xid)
 		user, _ := butwhatabout(u.Username)