diff --git a/main.go b/main.go index 24d4d81..d6994fa 100644 --- a/main.go +++ b/main.go @@ -110,7 +110,7 @@ func main() { // {{{ http.HandleFunc("/user/authenticate", AuthManager.AuthenticationHandler) - http.HandleFunc("/node/tree/{timestamp}", authenticated(actionNodeTree)) + http.HandleFunc("/node/tree/{timestamp}/{offset}", authenticated(actionNodeTree)) http.HandleFunc("/node/retrieve/{id}", authenticated(actionNodeRetrieve)) http.HandleFunc("/service_worker.js", pageServiceWorker) @@ -226,8 +226,9 @@ func pageSync(w http.ResponseWriter, r *http.Request) { // {{{ func actionNodeTree(w http.ResponseWriter, r *http.Request) { // {{{ user := getUser(r) changedFrom, _ := strconv.Atoi(r.PathValue("timestamp")) + offset, _ := strconv.Atoi(r.PathValue("offset")) - nodes, maxSeq, err := NodeTree(user.ID, uint64(changedFrom)) + nodes, maxSeq, moreRowsExist, err := NodeTree(user.ID, offset, uint64(changedFrom)) if err != nil { Log.Error("/node/tree", "error", err) httpError(w, err) @@ -238,7 +239,8 @@ func actionNodeTree(w http.ResponseWriter, r *http.Request) { // {{{ OK bool Nodes []TreeNode MaxSeq uint64 - }{true, nodes, maxSeq}) + Continue bool + }{true, nodes, maxSeq, moreRowsExist}) Log.Debug("tree", "nodes", nodes) w.Write(j) } // }}} diff --git a/node.go b/node.go index d936fd5..60a88a2 100644 --- a/node.go +++ b/node.go @@ -57,7 +57,8 @@ type Node struct { Markdown bool } -func NodeTree(userID int, synced uint64) (nodes []TreeNode, maxSeq uint64, err error) { // {{{ +func NodeTree(userID, offset int, synced uint64) (nodes []TreeNode, maxSeq uint64, moreRowsExist bool, err error) { // {{{ + const LIMIT = 8 var rows *sqlx.Rows rows, err = db.Queryx(` SELECT @@ -74,14 +75,17 @@ func NodeTree(userID int, synced uint64) (nodes []TreeNode, maxSeq uint64, err e public.node WHERE user_id = $1 AND ( - created_seq > $2 OR - updated_seq > $2 OR - deleted_seq > $2 + created_seq > $4 OR + updated_seq > $4 OR + deleted_seq > $4 ) ORDER BY created ASC + LIMIT $2 OFFSET $3 `, userID, + LIMIT + 1, + offset, synced, ) if err != nil { @@ -95,12 +99,24 @@ func NodeTree(userID int, synced uint64) (nodes []TreeNode, maxSeq uint64, err e } nodes = []TreeNode{} + numNodes := 0 for rows.Next() { + // Query selects up to one more row than the decided limit. + // Saves one SQL query for row counting. + // Thus if numNodes is larger than the limit, more rows exist for the next call. + numNodes++ + if numNodes > LIMIT { + moreRowsExist = true + return + } + node := TreeNode{} if err = rows.StructScan(&node); err != nil { return } nodes = append(nodes, node) + + // DeletedSeq will be 0 if invalid, and thus not be a problem for the max function. maxSeq = max(maxSeq, node.CreatedSeq, node.UpdatedSeq, uint64(node.DeletedSeq.Int64)) } diff --git a/static/js/node_store.mjs b/static/js/node_store.mjs index a7cb684..b93e669 100644 --- a/static/js/node_store.mjs +++ b/static/js/node_store.mjs @@ -97,35 +97,38 @@ export class NodeStore { async updateTreeRecords(records) {//{{{ return new Promise((resolve, reject) => { - try { - let max = 0 - const t = this.db.transaction('treeNodes', 'readwrite') - const nodeStore = t.objectStore('treeNodes') - t.onerror = (event) => { - console.log('transaction error', event.target.error) + const t = this.db.transaction('treeNodes', 'readwrite') + const nodeStore = t.objectStore('treeNodes') + t.onerror = (event) => { + console.log('transaction error', event.target.error) + reject(event.target.error) + } + t.oncomplete = () => { + resolve() + } + + // records is an object, not an array. + for (const i in records) { + const record = records[i] + + let addReq + let op + if (record.Deleted) { + op = 'deleting' + addReq = nodeStore.delete(record.UUID) + } else { + op = 'upserting' + addReq = nodeStore.put(record) + } + addReq.onsuccess = () => { + console.log(`${op} ${record.UUID} (${record.Name})`) + } + addReq.onerror = (event) => { + console.log(`error ${op} ${record.UUID}`, event.target.error) reject(event.target.error) } - t.oncomplete = () => { - console.log(max) - resolve(max) - } - - // records is an object, not an array. - for (const i in records) { - const record = records[i] - const addReq = nodeStore.put(record) - addReq.onsuccess = () => { - max = Math.max(max, record.CreatedSeq, record.UpdatedSeq, record.DeletedSeq.Int64) - console.log('OK!', record.UUID, record.Name) - } - addReq.onerror = (event) => { - console.log('Error!', event.target.error, record.UUID) - } - } - - } catch (e) { - console.log(e) } + }) }//}}} async add(records) {//{{{ diff --git a/static/js/sync.mjs b/static/js/sync.mjs index 12c080b..080cfcb 100644 --- a/static/js/sync.mjs +++ b/static/js/sync.mjs @@ -2,7 +2,30 @@ import { API } from 'api' export class Sync { static async tree() { - let oldMax = 0 + try { + const state = await nodeStore.getAppState('latest_sync') + let oldMax = (state?.value ? state.value : 0) + let newMax = 0 + + let offset = 0 + let res = { Continue: false } + let batch = 0 + do { + batch++ + console.log(`Batch #${batch}`) + res = await API.query('POST', `/node/tree/${oldMax}/${offset}`, {}) + offset += res.Nodes.length + newMax = res.MaxSeq + await nodeStore.updateTreeRecords(res.Nodes) + } while (res.Continue) + + nodeStore.setAppState('latest_sync', Math.max(oldMax, newMax)) + } catch (e) { + console.log('sync node tree', e) + } + + + /* nodeStore.getAppState('latest_sync') .then(state => { if (state !== null) { @@ -11,9 +34,22 @@ export class Sync { } return 0 }) - .then(sequence => API.query('POST', `/node/tree/${sequence}`, {})) - .then(res => nodeStore.updateTreeRecords(res.Nodes)) - .then(newMax => nodeStore.setAppState('latest_sync', Math.max(oldMax, newMax))) - .catch(e => alert(e)) + .then(async sequence => { + let offset = 0 + let res = { Continue: false } + try { + do { + res = await API.query('POST', `/node/tree/${sequence}/${offset}`, {}) + offset += res.Nodes.length + newMax = res.MaxSeq + await nodeStore.updateTreeRecords(res.Nodes) + } while (res.Continue) + } catch (e) { + return new Promise((_, reject) => reject(e)) + } + }) + .then(() => nodeStore.setAppState('latest_sync', Math.max(oldMax, newMax))) + .catch(e => console.log('sync', e)) + */ } } diff --git a/static/service_worker.js b/static/service_worker.js index 35a8efc..d702ee3 100644 --- a/static/service_worker.js +++ b/static/service_worker.js @@ -61,6 +61,6 @@ self.addEventListener('activate', event => { }) self.addEventListener('fetch', event => { - console.log('SERVICE WORKER: fetch') + //console.log('SERVICE WORKER: fetch') event.respondWith(fetchAsset(event)) }) diff --git a/views/pages/sync.gotmpl b/views/pages/sync.gotmpl index b941d64..eb79c77 100644 --- a/views/pages/sync.gotmpl +++ b/views/pages/sync.gotmpl @@ -1,9 +1,11 @@ {{ define "page" }}

Sync

+