split media into separate database. keeps main db nice and tidy.

activity.go

@@ -178,12 +178,12 @@ func savedonk(url string, name, desc, media string, localize bool) *Donk {
 		}
 	}
 saveit:
-	res, err := stmtSaveFile.Exec(xid, name, desc, url, media, localize, data)
+	fileid, err := savefile(xid, name, desc, url, media, localize, data)
 	if err != nil {
 		log.Printf("error saving file %s: %s", url, err)
 		return nil
 	}
-	donk.FileID, _ = res.LastInsertId()
+	donk.FileID = fileid
 	return &donk
 }
 

database.go

@@ -210,7 +210,7 @@ func donksforhonks(honks []*Honk) {
 		hmap[h.ID] = h
 	}
 	// grab donks
-	q := fmt.Sprintf("select honkid, donks.fileid, xid, name, description, url, media, local from donks join files on donks.fileid = files.fileid where honkid in (%s)", strings.Join(ids, ","))
+	q := fmt.Sprintf("select honkid, donks.fileid, xid, name, description, url, media, local from donks join filemeta on donks.fileid = filemeta.fileid where honkid in (%s)", strings.Join(ids, ","))
 	rows, err := db.Query(q)
 	if err != nil {
 		log.Printf("error querying donks: %s", err)
@@ -272,6 +272,19 @@ func donksforhonks(honks []*Honk) {
 	rows.Close()
 }
 
+func savefile(xid string, name string, desc string, url string, media string, local bool, data []byte) (int64, error) {
+	res, err := stmtSaveFile.Exec(xid, name, desc, url, media, local)
+	if err != nil {
+		return 0, err
+	}
+	fileid, _ := res.LastInsertId()
+	_, err = stmtSaveFileData.Exec(xid, media, data)
+	if err != nil {
+		return 0, err
+	}
+	return fileid, nil
+}
+
 func savehonk(h *Honk) error {
 	dt := h.Date.UTC().Format(dbtimeformat)
 	aud := strings.Join(h.Audience, " ")
@@ -365,7 +378,7 @@ func cleanupdb(arg string) {
 		doordie(db, "delete from donks where honkid in (select honkid from honks where dt < ? and whofore = 0 and convoy not in (select convoy from honks where whofore = 2 or whofore = 3))", expdate)
 		doordie(db, "delete from honks where dt < ? and whofore = 0 and convoy not in (select convoy from honks where whofore = 2 or whofore = 3)", expdate)
 	}
-	doordie(db, "delete from files where fileid not in (select fileid from donks)")
+	doordie(db, "delete from filemeta where fileid not in (select fileid from donks)")
 	for _, u := range allusers() {
 		doordie(db, "delete from zonkers where userid = ? and wherefore = 'zonvoy' and zonkerid < (select zonkerid from zonkers where userid = ? and wherefore = 'zonvoy' order by zonkerid desc limit 1 offset 200)", u.UserID, u.UserID)
 	}
@@ -374,8 +387,9 @@ func cleanupdb(arg string) {
 var stmtHonkers, stmtDubbers, stmtSaveHonker, stmtUpdateFlavor, stmtUpdateCombos *sql.Stmt
 var stmtOneXonk, stmtPublicHonks, stmtUserHonks, stmtHonksByCombo, stmtHonksByConvoy *sql.Stmt
 var stmtHonksByOntology, stmtHonksForUser, stmtHonksForMe, stmtSaveDub, stmtHonksByXonker *sql.Stmt
-var stmtHonksBySearch, stmtHonksByHonker, stmtSaveHonk, stmtFileData, stmtWhatAbout *sql.Stmt
+var stmtHonksBySearch, stmtHonksByHonker, stmtSaveHonk, stmtWhatAbout *sql.Stmt
-var stmtOneBonk, stmtFindZonk, stmtFindXonk, stmtSaveDonk, stmtFindFile, stmtSaveFile *sql.Stmt
+var stmtOneBonk, stmtFindZonk, stmtFindXonk, stmtSaveDonk *sql.Stmt
+var stmtFindFile, stmtGetFileData, stmtSaveFileData, stmtSaveFile *sql.Stmt
 var stmtAddDoover, stmtGetDoovers, stmtLoadDoover, stmtZapDoover, stmtOneHonker *sql.Stmt
 var stmtThumbBiters, stmtDeleteHonk, stmtDeleteDonks, stmtDeleteOnts, stmtSaveZonker *sql.Stmt
 var stmtGetZonkers, stmtRecentHonkers, stmtGetXonker, stmtSaveXonker, stmtDeleteXonker *sql.Stmt
@@ -425,10 +439,15 @@ func prepareStatements(db *sql.DB) {
 	stmtDeleteOnts = preparetodie(db, "delete from onts where honkid = ?")
 	stmtSaveDonk = preparetodie(db, "insert into donks (honkid, fileid) values (?, ?)")
 	stmtDeleteDonks = preparetodie(db, "delete from donks where honkid = ?")
-	stmtSaveFile = preparetodie(db, "insert into files (xid, name, description, url, media, local, content) values (?, ?, ?, ?, ?, ?, ?)")
+	stmtSaveFile = preparetodie(db, "insert into filemeta (xid, name, description, url, media, local) values (?, ?, ?, ?, ?, ?)")
-	stmtFileData = preparetodie(db, "select media, content from files where xid = ?")
+	blobdb, err := sql.Open("sqlite3", blobdbname)
+	if err != nil {
+		log.Fatal(err)
+	}
+	stmtSaveFileData = preparetodie(blobdb, "insert into filedata (xid, media, content) values (?, ?, ?)")
+	stmtGetFileData = preparetodie(blobdb, "select media, content from filedata where xid = ?")
 	stmtFindXonk = preparetodie(db, "select honkid from honks where userid = ? and xid = ?")
-	stmtFindFile = preparetodie(db, "select fileid from files where url = ? and local = 1")
+	stmtFindFile = preparetodie(db, "select fileid from filemeta where url = ? and local = 1")
 	stmtWhatAbout = preparetodie(db, "select userid, username, displayname, about, pubkey, options from users where username = ?")
 	stmtSaveDub = preparetodie(db, "insert into honkers (userid, name, xid, flavor) values (?, ?, ?, ?)")
 	stmtAddDoover = preparetodie(db, "insert into doovers (dt, tries, username, rcpt, msg) values (?, ?, ?, ?, ?)")

docs/changelog.txt

@@ -2,6 +2,8 @@ changelog
 
 -- next
 
++ Split media database into separate blob.db.
+
 + Location checkin. Welcome to the... danger zone!
 
 +++ Editing honks (Update activity).

schema.sql

@@ -1,7 +1,7 @@
 
 create table honks (honkid integer primary key, userid integer, what text, honker text, xid text, rid text, dt text, url text, audience text, noise text, convoy text, whofore integer, format text, precis text, oonker text, flags integer);
 create table donks (honkid integer, fileid integer);
-create table files(fileid integer primary key, xid text, name text, description text, url text, media text, local integer, content blob);
+create table filemeta (fileid integer primary key, xid text, name text, description text, url text, media text, local integer);
 create table honkers (honkerid integer primary key, userid integer, name text, xid text, flavor text, combos text);
 create table xonkers (xonkerid integer primary key, name text, info text, flavor text);
 create table zonkers (zonkerid integer primary key, userid integer, name text, wherefore text);
@@ -17,8 +17,8 @@ create index idx_honksoonker on honks(oonker);
 create index idx_honkerxid on honkers(xid);
 create index idx_xonkername on xonkers(name);
 create index idx_zonkersname on zonkers(name);
-create index idx_filesxid on files(xid);
+create index idx_filesxid on filemeta(xid);
-create index idx_filesurl on files(url);
+create index idx_filesurl on filemeta(url);
 create index idx_ontology on onts(ontology);
 create index idx_onthonkid on onts(honkid);
 create index idx_placehonkid on places(honkid);

upgradedb.go

@@ -185,14 +185,60 @@ func upgradedb() {
 	case 18:
 		doordie(db, "create index idx_onthonkid on onts(honkid)")
 		doordie(db, "update config set value = 19 where key = 'dbversion'")
+		fallthrough
 	case 19:
 		doordie(db, "create table places (honkid integer, name text, latitude real, longitude real)")
 		doordie(db, "create index idx_placehonkid on places(honkid)")
+		fallthrough
 	case 20:
 		doordie(db, "alter table places add column url text")
 		doordie(db, "update places set url = ''")
 		doordie(db, "update config set value = 21 where key = 'dbversion'")
+		fallthrough
 	case 21:
+		// here we go...
+		initblobdb()
+		blobdb, err := sql.Open("sqlite3", blobdbname)
+		if err != nil {
+			log.Fatal(err)
+		}
+		tx, err := blobdb.Begin()
+		if err != nil {
+			log.Fatalf("can't begin: %s", err)
+		}
+		doordie(db, "drop index idx_filesxid")
+		doordie(db, "drop index idx_filesurl")
+		doordie(db, "create table filemeta (fileid integer primary key, xid text, name text, description text, url text, media text, local integer)")
+		doordie(db, "insert into filemeta select fileid, xid, name, description, url, media, local from files")
+		doordie(db, "create index idx_filesxid on filemeta(xid)")
+		doordie(db, "create index idx_filesurl on filemeta(url)")
+
+		rows, err := db.Query("select xid, media, content from files")
+		if err != nil {
+			log.Fatal(err)
+		}
+		for rows.Next() {
+			var xid, media string
+			var data []byte
+			err = rows.Scan(&xid, &media, &data)
+			if err != nil {
+				log.Fatal(err)
+			}
+			_, err = tx.Exec("insert into filedata (xid, media, content) values (?, ?, ?)", xid, media, data)
+			if err != nil {
+				log.Fatal(err)
+			}
+		}
+		rows.Close()
+		err = tx.Commit()
+		if err != nil {
+			log.Fatalf("can't commit: %s", err)
+		}
+		doordie(db, "drop table files")
+		doordie(db, "vacuum")
+		doordie(db, "update config set value = 22 where key = 'dbversion'")
+		fallthrough
+	case 22:
 	default:
 		log.Fatalf("can't upgrade unknown version %d", dbversion)
 	}

util.go

@@ -71,8 +71,9 @@ var dbtimeformat = "2006-01-02 15:04:05"
 
 var alreadyopendb *sql.DB
 var dbname = "honk.db"
+var blobdbname = "blob.db"
 var stmtConfig *sql.Stmt
-var myVersion = 21
+var myVersion = 22
 
 func initdb() {
 	schema, err := ioutil.ReadFile("schema.sql")
@@ -162,12 +163,38 @@ func initdb() {
 		log.Print(err)
 		return
 	}
+
+	initblobdb()
+
 	prepareStatements(db)
 	db.Close()
 	fmt.Printf("done.\n")
 	os.Exit(0)
 }
 
+func initblobdb() {
+	_, err := os.Stat(blobdbname)
+	if err == nil {
+		log.Fatalf("%s already exists", blobdbname)
+	}
+	blobdb, err := sql.Open("sqlite3", blobdbname)
+	if err != nil {
+		log.Print(err)
+		return
+	}
+	_, err = blobdb.Exec("create table filedata (xid text, media text, content blob)")
+	if err != nil {
+		log.Print(err)
+		return
+	}
+	_, err = blobdb.Exec("create index idx_filexid on filedata(xid)")
+	if err != nil {
+		log.Print(err)
+		return
+	}
+	blobdb.Close()
+}
+
 func adduser() {
 	db := opendatabase()
 	defer func() {

web.go

@@ -1007,13 +1007,13 @@ func submithonk(w http.ResponseWriter, r *http.Request) {
 				desc = name
 			}
 			url := fmt.Sprintf("https://%s/d/%s", serverName, xid)
-			res, err := stmtSaveFile.Exec(xid, name, desc, url, media, 1, data)
+			fileid, err := savefile(xid, name, desc, url, media, true, data)
 			if err != nil {
 				log.Printf("unable to save image: %s", err)
 				return
 			}
 			var d Donk
-			d.FileID, _ = res.LastInsertId()
+			d.FileID = fileid
 			honk.Donks = append(honk.Donks, &d)
 			donkxid = d.XID
 		}
@@ -1404,7 +1404,7 @@ func servememe(w http.ResponseWriter, r *http.Request) {
 
 func servefile(w http.ResponseWriter, r *http.Request) {
 	xid := mux.Vars(r)["xid"]
-	row := stmtFileData.QueryRow(xid)
+	row := stmtGetFileData.QueryRow(xid)
 	var media string
 	var data []byte
 	err := row.Scan(&media, &data)