Simon/siyuan
1
0
Fork 0
mirror of https://github.com/siyuan-note/siyuan.git synced 2026-06-28 06:46:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

🎨 Improve rebuild index https://github.com/siyuan-note/siyuan/issues/17713

Browse Source 
Signed-off-by: Daniel <845765@qq.com>
This commit is contained in:
Daniel
2026-05-24 21:05:03 +08:00
parent 915c118a67
commit 4afa0adfa6
1 changed files with 76 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

96
kernel/sql/database.go
View File

@@ -84,6 +84,8 @@ func initDatabase(forceRebuild bool) {
if forceRebuild {
ClearQueue()
closeDatabase()
util.RemoveDatabaseFile(util.DBPath)
}
initDBConnection()
@@ -148,22 +150,18 @@ func initDBTables() {
logging.LogFatalf(logging.ExitCodeUnavailableDatabase, "create index [idx_blocks_root_id_id_hash] failed: %s", err)
}
_, err = db.Exec("DROP TABLE IF EXISTS blocks_fts")
if err != nil {
logging.LogFatalf(logging.ExitCodeUnavailableDatabase, "drop table [blocks_fts] failed: %s", err)
}
_, err = db.Exec("CREATE VIRTUAL TABLE blocks_fts USING fts5(id UNINDEXED, parent_id UNINDEXED, root_id UNINDEXED, hash UNINDEXED, box UNINDEXED, path UNINDEXED, hpath UNINDEXED, name, alias, memo, tag, content, fcontent, markdown UNINDEXED, length UNINDEXED, type UNINDEXED, subtype UNINDEXED, ial, sort UNINDEXED, created UNINDEXED, updated UNINDEXED, tokenize=\"siyuan\")")
if err != nil {
logging.LogFatalf(logging.ExitCodeUnavailableDatabase, "create table [blocks_fts] failed: %s", err)
}
_, err = db.Exec("DROP TABLE IF EXISTS blocks_fts_case_insensitive")
if err != nil {
logging.LogFatalf(logging.ExitCodeUnavailableDatabase, "drop table [blocks_fts_case_insensitive] failed: %s", err)
}
_, err = db.Exec("CREATE VIRTUAL TABLE blocks_fts_case_insensitive USING fts5(id UNINDEXED, parent_id UNINDEXED, root_id UNINDEXED, hash UNINDEXED, box UNINDEXED, path UNINDEXED, hpath UNINDEXED, name, alias, memo, tag, content, fcontent, markdown UNINDEXED, length UNINDEXED, type UNINDEXED, subtype UNINDEXED, ial, sort UNINDEXED, created UNINDEXED, updated UNINDEXED, tokenize=\"siyuan case_insensitive\")")
if err != nil {
logging.LogFatalf(logging.ExitCodeUnavailableDatabase, "create table [blocks_fts_case_insensitive] failed: %s", err)
if err = initFTSBlocks(); err != nil {
if isRecoverableDBFileError(err) {
logging.LogWarnf("create fts tables failed: %s, retrying with clean database...", err)
closeDatabase()
util.RemoveDatabaseFile(util.DBPath)
time.Sleep(time.Second)
initDBConnection()
err = initFTSBlocks()
}
if err != nil {
logging.LogFatalf(logging.ExitCodeUnavailableDatabase, "create fts tables failed: %s", err)
}
}
_, err = db.Exec("DROP TABLE IF EXISTS spans")
@@ -228,6 +226,24 @@ func initDBTables() {
}
}
func initFTSBlocks() (err error) {
_, err = db.Exec("DROP TABLE IF EXISTS blocks_fts")
if err != nil {
return
}
_, err = db.Exec("CREATE VIRTUAL TABLE blocks_fts USING fts5(id UNINDEXED, parent_id UNINDEXED, root_id UNINDEXED, hash UNINDEXED, box UNINDEXED, path UNINDEXED, hpath UNINDEXED, name, alias, memo, tag, content, fcontent, markdown UNINDEXED, length UNINDEXED, type UNINDEXED, subtype UNINDEXED, ial, sort UNINDEXED, created UNINDEXED, updated UNINDEXED, tokenize=\"siyuan\")")
if err != nil {
return
}
_, err = db.Exec("DROP TABLE IF EXISTS blocks_fts_case_insensitive")
if err != nil {
return
}
_, err = db.Exec("CREATE VIRTUAL TABLE blocks_fts_case_insensitive USING fts5(id UNINDEXED, parent_id UNINDEXED, root_id UNINDEXED, hash UNINDEXED, box UNINDEXED, path UNINDEXED, hpath UNINDEXED, name, alias, memo, tag, content, fcontent, markdown UNINDEXED, length UNINDEXED, type UNINDEXED, subtype UNINDEXED, ial, sort UNINDEXED, created UNINDEXED, updated UNINDEXED, tokenize=\"siyuan case_insensitive\")")
return
}
func initDBConnection() {
closeDatabase()
@@ -308,7 +324,21 @@ func initHistoryDBTables() {
historyDB.Exec("DROP TABLE histories_fts_case_insensitive")
_, err := historyDB.Exec("CREATE VIRTUAL TABLE histories_fts_case_insensitive USING fts5(id UNINDEXED, type UNINDEXED, op UNINDEXED, title, content, path UNINDEXED, created UNINDEXED, tokenize=\"siyuan case_insensitive\")")
if err != nil {
logging.LogFatalf(logging.ExitCodeUnavailableDatabase, "create table [histories_fts_case_insensitive] failed: %s", err)
if isRecoverableDBFileError(err) {
logging.LogWarnf("create history fts table failed: %s, retrying with clean database...", err)
historyDB.Close()
historyDB = nil
if removeErr := os.RemoveAll(util.HistoryDBPath); nil != removeErr {
logging.LogErrorf("remove history database file [%s] failed: %s", util.HistoryDBPath, removeErr)
}
time.Sleep(time.Second)
initHistoryDBConnection()
historyDB.Exec("DROP TABLE histories_fts_case_insensitive")
_, err = historyDB.Exec("CREATE VIRTUAL TABLE histories_fts_case_insensitive USING fts5(id UNINDEXED, type UNINDEXED, op UNINDEXED, title, content, path UNINDEXED, created UNINDEXED, tokenize=\"siyuan case_insensitive\")")
}
if err != nil {
logging.LogFatalf(logging.ExitCodeUnavailableDatabase, "create table [histories_fts_case_insensitive] failed: %s", err)
}
}
}
@@ -368,7 +398,21 @@ func initAssetContentDBTables() {
assetContentDB.Exec("DROP TABLE asset_contents_fts_case_insensitive")
_, err := assetContentDB.Exec("CREATE VIRTUAL TABLE asset_contents_fts_case_insensitive USING fts5(id UNINDEXED, name, ext, path, size UNINDEXED, updated UNINDEXED, content, tokenize=\"siyuan case_insensitive\")")
if err != nil {
logging.LogFatalf(logging.ExitCodeUnavailableDatabase, "create table [asset_contents_fts_case_insensitive] failed: %s", err)
if isRecoverableDBFileError(err) {
logging.LogWarnf("create asset content fts table failed: %s, retrying with clean database...", err)
assetContentDB.Close()
assetContentDB = nil
if removeErr := os.RemoveAll(util.AssetContentDBPath); nil != removeErr {
logging.LogErrorf("remove asset content database file [%s] failed: %s", util.AssetContentDBPath, removeErr)
}
time.Sleep(time.Second)
initAssetContentDBConnection()
assetContentDB.Exec("DROP TABLE asset_contents_fts_case_insensitive")
_, err = assetContentDB.Exec("CREATE VIRTUAL TABLE asset_contents_fts_case_insensitive USING fts5(id UNINDEXED, name, ext, path, size UNINDEXED, updated UNINDEXED, content, tokenize=\"siyuan case_insensitive\")")
}
if err != nil {
logging.LogFatalf(logging.ExitCodeUnavailableDatabase, "create table [asset_contents_fts_case_insensitive] failed: %s", err)
}
}
}
@@ -1509,9 +1553,10 @@ func prepareExecInsertTx(tx *sql.Tx, stmtSQL string, args []any) (err error) {
tx.Rollback()
logging.LogErrorf("exec database stmt [%s] failed: %s\n %s", stmtSQL, err, logging.ShortStack())
if strings.Contains(err.Error(), "database disk image is malformed") {
if isRecoverableDBFileError(err) {
closeDatabase()
util.RemoveDatabaseFile(util.DBPath)
time.Sleep(time.Second)
initDatabase(true)
logging.LogFatalf(logging.ExitCodeUnavailableDatabase, "database disk image [%s] is malformed, please restart SiYuan kernel to rebuild it\n\t%s\n\t%v", util.DBPath, stmtSQL, args)
}
@@ -1525,9 +1570,10 @@ func execStmtTx(tx *sql.Tx, stmt string, args ...any) (err error) {
tx.Rollback()
logging.LogErrorf("exec database stmt [%s] failed: %s\n %s", stmt, err, logging.ShortStack())
if strings.Contains(err.Error(), "database disk image is malformed") {
if isRecoverableDBFileError(err) {
closeDatabase()
util.RemoveDatabaseFile(util.DBPath)
time.Sleep(time.Second)
initDatabase(true)
logging.LogFatalf(logging.ExitCodeUnavailableDatabase, "database disk image [%s] is malformed, please restart SiYuan kernel to rebuild it\n\t%s\n\t%v", util.DBPath, stmt, args)
}
@@ -1583,6 +1629,16 @@ func ialAttr(ial, name string) (ret string) {
return
}
func isRecoverableDBFileError(err error) bool {
if nil == err {
return false
}
errStr := err.Error()
return strings.Contains(errStr, "database disk image is malformed") ||
strings.Contains(errStr, "The parameter is incorrect") ||
strings.Contains(errStr, "unable to open database file")
}
func closeDatabase() {
if nil == db {
return