From dbe8a3c55ca512d004a002988f4c411b46c1716c Mon Sep 17 00:00:00 2001 From: Ted Unangst Date: Sun, 29 Sep 2019 21:40:37 -0400 Subject: [PATCH] split media into separate database. keeps main db nice and tidy. --- activity.go | 4 ++-- database.go | 33 ++++++++++++++++++++++++++------- docs/changelog.txt | 2 ++ schema.sql | 6 +++--- upgradedb.go | 46 ++++++++++++++++++++++++++++++++++++++++++++++ util.go | 29 ++++++++++++++++++++++++++++- web.go | 6 +++--- 7 files changed, 110 insertions(+), 16 deletions(-) diff --git a/activity.go b/activity.go index a79d397..d7d635a 100644 --- a/activity.go +++ b/activity.go @@ -178,12 +178,12 @@ func savedonk(url string, name, desc, media string, localize bool) *Donk { } } saveit: - res, err := stmtSaveFile.Exec(xid, name, desc, url, media, localize, data) + fileid, err := savefile(xid, name, desc, url, media, localize, data) if err != nil { log.Printf("error saving file %s: %s", url, err) return nil } - donk.FileID, _ = res.LastInsertId() + donk.FileID = fileid return &donk } diff --git a/database.go b/database.go index 16be546..7798008 100644 --- a/database.go +++ b/database.go @@ -210,7 +210,7 @@ func donksforhonks(honks []*Honk) { hmap[h.ID] = h } // grab donks - q := fmt.Sprintf("select honkid, donks.fileid, xid, name, description, url, media, local from donks join files on donks.fileid = files.fileid where honkid in (%s)", strings.Join(ids, ",")) + q := fmt.Sprintf("select honkid, donks.fileid, xid, name, description, url, media, local from donks join filemeta on donks.fileid = filemeta.fileid where honkid in (%s)", strings.Join(ids, ",")) rows, err := db.Query(q) if err != nil { log.Printf("error querying donks: %s", err) @@ -272,6 +272,19 @@ func donksforhonks(honks []*Honk) { rows.Close() } +func savefile(xid string, name string, desc string, url string, media string, local bool, data []byte) (int64, error) { + res, err := stmtSaveFile.Exec(xid, name, desc, url, media, local) + if err != nil { + return 0, err + } + fileid, _ := res.LastInsertId() + _, err = stmtSaveFileData.Exec(xid, media, data) + if err != nil { + return 0, err + } + return fileid, nil +} + func savehonk(h *Honk) error { dt := h.Date.UTC().Format(dbtimeformat) aud := strings.Join(h.Audience, " ") @@ -365,7 +378,7 @@ func cleanupdb(arg string) { doordie(db, "delete from donks where honkid in (select honkid from honks where dt < ? and whofore = 0 and convoy not in (select convoy from honks where whofore = 2 or whofore = 3))", expdate) doordie(db, "delete from honks where dt < ? and whofore = 0 and convoy not in (select convoy from honks where whofore = 2 or whofore = 3)", expdate) } - doordie(db, "delete from files where fileid not in (select fileid from donks)") + doordie(db, "delete from filemeta where fileid not in (select fileid from donks)") for _, u := range allusers() { doordie(db, "delete from zonkers where userid = ? and wherefore = 'zonvoy' and zonkerid < (select zonkerid from zonkers where userid = ? and wherefore = 'zonvoy' order by zonkerid desc limit 1 offset 200)", u.UserID, u.UserID) } @@ -374,8 +387,9 @@ func cleanupdb(arg string) { var stmtHonkers, stmtDubbers, stmtSaveHonker, stmtUpdateFlavor, stmtUpdateCombos *sql.Stmt var stmtOneXonk, stmtPublicHonks, stmtUserHonks, stmtHonksByCombo, stmtHonksByConvoy *sql.Stmt var stmtHonksByOntology, stmtHonksForUser, stmtHonksForMe, stmtSaveDub, stmtHonksByXonker *sql.Stmt -var stmtHonksBySearch, stmtHonksByHonker, stmtSaveHonk, stmtFileData, stmtWhatAbout *sql.Stmt -var stmtOneBonk, stmtFindZonk, stmtFindXonk, stmtSaveDonk, stmtFindFile, stmtSaveFile *sql.Stmt +var stmtHonksBySearch, stmtHonksByHonker, stmtSaveHonk, stmtWhatAbout *sql.Stmt +var stmtOneBonk, stmtFindZonk, stmtFindXonk, stmtSaveDonk *sql.Stmt +var stmtFindFile, stmtGetFileData, stmtSaveFileData, stmtSaveFile *sql.Stmt var stmtAddDoover, stmtGetDoovers, stmtLoadDoover, stmtZapDoover, stmtOneHonker *sql.Stmt var stmtThumbBiters, stmtDeleteHonk, stmtDeleteDonks, stmtDeleteOnts, stmtSaveZonker *sql.Stmt var stmtGetZonkers, stmtRecentHonkers, stmtGetXonker, stmtSaveXonker, stmtDeleteXonker *sql.Stmt @@ -425,10 +439,15 @@ func prepareStatements(db *sql.DB) { stmtDeleteOnts = preparetodie(db, "delete from onts where honkid = ?") stmtSaveDonk = preparetodie(db, "insert into donks (honkid, fileid) values (?, ?)") stmtDeleteDonks = preparetodie(db, "delete from donks where honkid = ?") - stmtSaveFile = preparetodie(db, "insert into files (xid, name, description, url, media, local, content) values (?, ?, ?, ?, ?, ?, ?)") - stmtFileData = preparetodie(db, "select media, content from files where xid = ?") + stmtSaveFile = preparetodie(db, "insert into filemeta (xid, name, description, url, media, local) values (?, ?, ?, ?, ?, ?)") + blobdb, err := sql.Open("sqlite3", blobdbname) + if err != nil { + log.Fatal(err) + } + stmtSaveFileData = preparetodie(blobdb, "insert into filedata (xid, media, content) values (?, ?, ?)") + stmtGetFileData = preparetodie(blobdb, "select media, content from filedata where xid = ?") stmtFindXonk = preparetodie(db, "select honkid from honks where userid = ? and xid = ?") - stmtFindFile = preparetodie(db, "select fileid from files where url = ? and local = 1") + stmtFindFile = preparetodie(db, "select fileid from filemeta where url = ? and local = 1") stmtWhatAbout = preparetodie(db, "select userid, username, displayname, about, pubkey, options from users where username = ?") stmtSaveDub = preparetodie(db, "insert into honkers (userid, name, xid, flavor) values (?, ?, ?, ?)") stmtAddDoover = preparetodie(db, "insert into doovers (dt, tries, username, rcpt, msg) values (?, ?, ?, ?, ?)") diff --git a/docs/changelog.txt b/docs/changelog.txt index dd0400f..7a57040 100644 --- a/docs/changelog.txt +++ b/docs/changelog.txt @@ -2,6 +2,8 @@ changelog -- next ++ Split media database into separate blob.db. + + Location checkin. Welcome to the... danger zone! +++ Editing honks (Update activity). diff --git a/schema.sql b/schema.sql index de50ab0..197b417 100644 --- a/schema.sql +++ b/schema.sql @@ -1,7 +1,7 @@ create table honks (honkid integer primary key, userid integer, what text, honker text, xid text, rid text, dt text, url text, audience text, noise text, convoy text, whofore integer, format text, precis text, oonker text, flags integer); create table donks (honkid integer, fileid integer); -create table files(fileid integer primary key, xid text, name text, description text, url text, media text, local integer, content blob); +create table filemeta (fileid integer primary key, xid text, name text, description text, url text, media text, local integer); create table honkers (honkerid integer primary key, userid integer, name text, xid text, flavor text, combos text); create table xonkers (xonkerid integer primary key, name text, info text, flavor text); create table zonkers (zonkerid integer primary key, userid integer, name text, wherefore text); @@ -17,8 +17,8 @@ create index idx_honksoonker on honks(oonker); create index idx_honkerxid on honkers(xid); create index idx_xonkername on xonkers(name); create index idx_zonkersname on zonkers(name); -create index idx_filesxid on files(xid); -create index idx_filesurl on files(url); +create index idx_filesxid on filemeta(xid); +create index idx_filesurl on filemeta(url); create index idx_ontology on onts(ontology); create index idx_onthonkid on onts(honkid); create index idx_placehonkid on places(honkid); diff --git a/upgradedb.go b/upgradedb.go index dc0a26b..7c49139 100644 --- a/upgradedb.go +++ b/upgradedb.go @@ -185,14 +185,60 @@ func upgradedb() { case 18: doordie(db, "create index idx_onthonkid on onts(honkid)") doordie(db, "update config set value = 19 where key = 'dbversion'") + fallthrough case 19: doordie(db, "create table places (honkid integer, name text, latitude real, longitude real)") doordie(db, "create index idx_placehonkid on places(honkid)") + fallthrough case 20: doordie(db, "alter table places add column url text") doordie(db, "update places set url = ''") doordie(db, "update config set value = 21 where key = 'dbversion'") + fallthrough case 21: + // here we go... + initblobdb() + blobdb, err := sql.Open("sqlite3", blobdbname) + if err != nil { + log.Fatal(err) + } + tx, err := blobdb.Begin() + if err != nil { + log.Fatalf("can't begin: %s", err) + } + doordie(db, "drop index idx_filesxid") + doordie(db, "drop index idx_filesurl") + doordie(db, "create table filemeta (fileid integer primary key, xid text, name text, description text, url text, media text, local integer)") + doordie(db, "insert into filemeta select fileid, xid, name, description, url, media, local from files") + doordie(db, "create index idx_filesxid on filemeta(xid)") + doordie(db, "create index idx_filesurl on filemeta(url)") + + rows, err := db.Query("select xid, media, content from files") + if err != nil { + log.Fatal(err) + } + for rows.Next() { + var xid, media string + var data []byte + err = rows.Scan(&xid, &media, &data) + if err != nil { + log.Fatal(err) + } + _, err = tx.Exec("insert into filedata (xid, media, content) values (?, ?, ?)", xid, media, data) + if err != nil { + log.Fatal(err) + } + } + rows.Close() + err = tx.Commit() + if err != nil { + log.Fatalf("can't commit: %s", err) + } + doordie(db, "drop table files") + doordie(db, "vacuum") + doordie(db, "update config set value = 22 where key = 'dbversion'") + fallthrough + case 22: default: log.Fatalf("can't upgrade unknown version %d", dbversion) } diff --git a/util.go b/util.go index f4d6b3c..5ef9726 100644 --- a/util.go +++ b/util.go @@ -71,8 +71,9 @@ var dbtimeformat = "2006-01-02 15:04:05" var alreadyopendb *sql.DB var dbname = "honk.db" +var blobdbname = "blob.db" var stmtConfig *sql.Stmt -var myVersion = 21 +var myVersion = 22 func initdb() { schema, err := ioutil.ReadFile("schema.sql") @@ -162,12 +163,38 @@ func initdb() { log.Print(err) return } + + initblobdb() + prepareStatements(db) db.Close() fmt.Printf("done.\n") os.Exit(0) } +func initblobdb() { + _, err := os.Stat(blobdbname) + if err == nil { + log.Fatalf("%s already exists", blobdbname) + } + blobdb, err := sql.Open("sqlite3", blobdbname) + if err != nil { + log.Print(err) + return + } + _, err = blobdb.Exec("create table filedata (xid text, media text, content blob)") + if err != nil { + log.Print(err) + return + } + _, err = blobdb.Exec("create index idx_filexid on filedata(xid)") + if err != nil { + log.Print(err) + return + } + blobdb.Close() +} + func adduser() { db := opendatabase() defer func() { diff --git a/web.go b/web.go index 9b54f44..0b8d817 100644 --- a/web.go +++ b/web.go @@ -1007,13 +1007,13 @@ func submithonk(w http.ResponseWriter, r *http.Request) { desc = name } url := fmt.Sprintf("https://%s/d/%s", serverName, xid) - res, err := stmtSaveFile.Exec(xid, name, desc, url, media, 1, data) + fileid, err := savefile(xid, name, desc, url, media, true, data) if err != nil { log.Printf("unable to save image: %s", err) return } var d Donk - d.FileID, _ = res.LastInsertId() + d.FileID = fileid honk.Donks = append(honk.Donks, &d) donkxid = d.XID } @@ -1404,7 +1404,7 @@ func servememe(w http.ResponseWriter, r *http.Request) { func servefile(w http.ResponseWriter, r *http.Request) { xid := mux.Vars(r)["xid"] - row := stmtFileData.QueryRow(xid) + row := stmtGetFileData.QueryRow(xid) var media string var data []byte err := row.Scan(&media, &data)