wip

main.go

@@ -110,7 +110,7 @@ func main() { // {{{
 
 	http.HandleFunc("/user/authenticate", AuthManager.AuthenticationHandler)
 
-	http.HandleFunc("/node/tree/{timestamp}", authenticated(actionNodeTree))
+	http.HandleFunc("/node/tree/{timestamp}/{offset}", authenticated(actionNodeTree))
 	http.HandleFunc("/node/retrieve/{id}", authenticated(actionNodeRetrieve))
 
 	http.HandleFunc("/service_worker.js", pageServiceWorker)
@@ -226,8 +226,9 @@ func pageSync(w http.ResponseWriter, r *http.Request) { // {{{
 func actionNodeTree(w http.ResponseWriter, r *http.Request) { // {{{
 	user := getUser(r)
 	changedFrom, _ := strconv.Atoi(r.PathValue("timestamp"))
+	offset, _ := strconv.Atoi(r.PathValue("offset"))
 
-	nodes, maxSeq, err := NodeTree(user.ID, uint64(changedFrom))
+	nodes, maxSeq, moreRowsExist, err := NodeTree(user.ID, offset, uint64(changedFrom))
 	if err != nil {
 		Log.Error("/node/tree", "error", err)
 		httpError(w, err)
@@ -238,7 +239,8 @@ func actionNodeTree(w http.ResponseWriter, r *http.Request) { // {{{
 		OK    bool
 		Nodes []TreeNode
 		MaxSeq uint64
-	}{true, nodes, maxSeq})
+		Continue bool
+	}{true, nodes, maxSeq, moreRowsExist})
 	Log.Debug("tree", "nodes", nodes)
 	w.Write(j)
 } // }}}

node.go

@@ -57,7 +57,8 @@ type Node struct {
 	Markdown         bool
 }
 
-func NodeTree(userID int, synced uint64) (nodes []TreeNode, maxSeq uint64, err error) { // {{{
+func NodeTree(userID, offset int, synced uint64) (nodes []TreeNode, maxSeq uint64, moreRowsExist bool, err error) { // {{{
+	const LIMIT = 8
 	var rows *sqlx.Rows
 	rows, err = db.Queryx(`
 		SELECT
@@ -74,14 +75,17 @@ func NodeTree(userID int, synced uint64) (nodes []TreeNode, maxSeq uint64, err e
 			public.node
 		WHERE
 			user_id = $1 AND (
-				created_seq > $2 OR
+				created_seq > $4 OR
-				updated_seq > $2 OR
+				updated_seq > $4 OR
-				deleted_seq > $2
+				deleted_seq > $4
 			)
 		ORDER BY
 			created ASC
+		LIMIT $2 OFFSET $3
 		`,
 		userID,
+		LIMIT + 1,
+		offset,
 		synced,
 	)
 	if err != nil {
@@ -95,12 +99,24 @@ func NodeTree(userID int, synced uint64) (nodes []TreeNode, maxSeq uint64, err e
 	}
 
 	nodes = []TreeNode{}
+	numNodes := 0
 	for rows.Next() {
+		// Query selects up to one more row than the decided limit.
+		// Saves one SQL query for row counting.
+		// Thus if numNodes is larger than the limit, more rows exist for the next call.
+		numNodes++
+		if numNodes > LIMIT {
+			moreRowsExist = true
+			return
+		}
+
 		node := TreeNode{}
 		if err = rows.StructScan(&node); err != nil {
 			return
 		}
 		nodes = append(nodes, node)
+
+		// DeletedSeq will be 0 if invalid, and thus not be a problem for the max function.
 		maxSeq = max(maxSeq, node.CreatedSeq, node.UpdatedSeq, uint64(node.DeletedSeq.Int64))
 	}
 

static/js/node_store.mjs

@@ -97,35 +97,38 @@ export class NodeStore {
 
 	async updateTreeRecords(records) {//{{{
 		return new Promise((resolve, reject) => {
-			try {
+			const t = this.db.transaction('treeNodes', 'readwrite')
-				let max = 0
+			const nodeStore = t.objectStore('treeNodes')
-				const t = this.db.transaction('treeNodes', 'readwrite')
+			t.onerror = (event) => {
-				const nodeStore = t.objectStore('treeNodes')
+				console.log('transaction error', event.target.error)
-				t.onerror = (event) => {
+				reject(event.target.error)
-					console.log('transaction error', event.target.error)
+			}
+			t.oncomplete = () => {
+				resolve()
+			}
+
+			// records is an object, not an array.
+			for (const i in records) {
+				const record = records[i]
+
+				let addReq
+				let op
+				if (record.Deleted) {
+					op = 'deleting'
+					addReq = nodeStore.delete(record.UUID)
+				} else {
+					op = 'upserting'
+					addReq = nodeStore.put(record)
+				}
+				addReq.onsuccess = () => {
+					console.log(`${op} ${record.UUID} (${record.Name})`)
+				}
+				addReq.onerror = (event) => {
+					console.log(`error ${op} ${record.UUID}`, event.target.error)
 					reject(event.target.error)
 				}
-				t.oncomplete = () => {
-					console.log(max)
-					resolve(max)
-				}
-
-				// records is an object, not an array.
-				for (const i in records) {
-					const record = records[i]
-					const addReq = nodeStore.put(record)
-					addReq.onsuccess = () => {
-						max = Math.max(max, record.CreatedSeq, record.UpdatedSeq, record.DeletedSeq.Int64)
-						console.log('OK!', record.UUID, record.Name)
-					}
-					addReq.onerror = (event) => {
-						console.log('Error!', event.target.error, record.UUID)
-					}
-				}
-
-			} catch (e) {
-				console.log(e)
 			}
+
 		})
 	}//}}}
 	async add(records) {//{{{

static/js/sync.mjs

@@ -2,7 +2,30 @@ import { API } from 'api'
 
 export class Sync {
 	static async tree() {
-		let oldMax = 0
+		try {
+			const state = await nodeStore.getAppState('latest_sync')
+			let oldMax = (state?.value ? state.value : 0)
+			let newMax = 0
+
+			let offset = 0
+			let res = { Continue: false }
+			let batch = 0
+			do {
+				batch++
+				console.log(`Batch #${batch}`)
+				res = await API.query('POST', `/node/tree/${oldMax}/${offset}`, {})
+				offset += res.Nodes.length
+				newMax = res.MaxSeq
+				await nodeStore.updateTreeRecords(res.Nodes)
+			} while (res.Continue)
+
+			nodeStore.setAppState('latest_sync', Math.max(oldMax, newMax))
+		} catch (e) {
+			console.log('sync node tree', e)
+		}
+
+
+		/*
 		nodeStore.getAppState('latest_sync')
 			.then(state => {
 				if (state !== null) {
@@ -11,9 +34,22 @@ export class Sync {
 				}
 				return 0
 			})
-			.then(sequence => API.query('POST', `/node/tree/${sequence}`, {}))
+			.then(async sequence => {
-			.then(res => nodeStore.updateTreeRecords(res.Nodes))
+				let offset = 0
-			.then(newMax => nodeStore.setAppState('latest_sync', Math.max(oldMax, newMax)))
+				let res = { Continue: false }
-			.catch(e => alert(e))
+				try {
+					do {
+						res = await API.query('POST', `/node/tree/${sequence}/${offset}`, {})
+						offset += res.Nodes.length
+						newMax = res.MaxSeq
+						await nodeStore.updateTreeRecords(res.Nodes)
+					} while (res.Continue)
+				} catch (e) {
+					return new Promise((_, reject) => reject(e))
+				}
+			})
+			.then(() => nodeStore.setAppState('latest_sync', Math.max(oldMax, newMax)))
+			.catch(e => console.log('sync', e))
+		*/
 	}
 }

static/service_worker.js

@@ -61,6 +61,6 @@ self.addEventListener('activate', event => {
 })
 
 self.addEventListener('fetch', event => {
-	console.log('SERVICE WORKER: fetch')
+	//console.log('SERVICE WORKER: fetch')
 	event.respondWith(fetchAsset(event))
 })

views/pages/sync.gotmpl

@@ -1,9 +1,11 @@
 {{ define "page" }}
 <div style="margin: 32px">
 	<h1>Sync</h1>
+
 	<script type="module">
 		import { NodeStore } from 'node_store'
 		import { Sync } from 'sync'
+		window.Sync = Sync
 		window.nodeStore = new NodeStore()
 		window.nodeStore.initializeDB().then(()=>{
 			Sync.tree()