File sync

main.go

@@ -27,7 +27,7 @@ import (
 
 const VERSION = "v30"
 const CONTEXT_USER = 1
-const SYNC_PAGINATION = 200
+const SYNC_PAGINATION = 1
 
 var (
 	FlagGenerate       bool
@@ -287,6 +287,9 @@ func pageSync(w http.ResponseWriter, r *http.Request) { // {{{
 } // }}}
 
 func actionSyncFromServer(w http.ResponseWriter, r *http.Request) { // {{{
+	// XXX - DELETE ME!
+	time.Sleep(time.Millisecond * 200)
+
 	// The purpose of the Client UUID is to avoid
 	// sending nodes back once again to a client that
 	// just created or modified it.
@@ -551,6 +554,10 @@ func actionFile(w http.ResponseWriter, r *http.Request) { // {{{
 		return
 	}
 
+	w.Header().Add("X-File-Name", md.Name)
+	w.Header().Add("X-File-Modified", fmt.Sprintf("%d", md.Modified.UnixMilli()))
+	w.Header().Add("Content-Type", md.Type)
+
 	http.ServeContent(w, r, md.Name, md.Modified, f)
 } // }}}
 func actionFileMetadata(w http.ResponseWriter, r *http.Request) { // {{{

node.go

@@ -129,6 +129,7 @@ func NodesCount(userID int, synced uint64, clientUUID string) (count int, err er
 		FROM
 			public.node
 		WHERE
+			NOT special AND
 			user_id = $1 AND
 			client != $3 AND
 			(

static/css/notes2.css

@@ -178,6 +178,34 @@ button {
 	border-right: 1px solid var(--line-color);
 
 	n2-sidebar {
+		display: grid;
+		grid-template-rows: min-content min-content min-content 1fr;
+
+		.el-sync-status {
+			display: grid;
+			grid-template-columns: min-content 1fr;
+			grid-gap: 0px 8px;
+			align-items: center;
+			padding: 8px 16px 8px 16px;
+			border-bottom: 1px solid var(--line-color);
+
+			.el-sync-icon {
+				grid-column: 1;
+				grid-row: 1 / 3;
+			}
+
+			.el-sync-label {
+				grid-column: 2;
+				grid-row: 1;
+			}
+
+			progress {
+				grid-column: 2;
+				grid-row: 2;
+				width: 100%;
+			}
+		}
+
 		.el-treenodes {
 			margin: 24px 32px 32px 32px;
 		}

static/js/node_store.mjs

@@ -566,21 +566,32 @@ export class NodeStore {
 		})
 	}//}}}
 
-	async storeFile(file) {// {{{
+	async storeFile(file, reader, progressCallback) {// {{{
 		const metadata = new FileMetadata(file)
+		let processedBytes = 0
+		let stream
 
 		try {
 			// Do potentially larger blocks.
 			const BLOCKSIZE = 1048576
-			const stream = file.stream()
-			const reader = stream.getReader({ mode: 'byob' })
 			let buffer = new Uint8Array(BLOCKSIZE)
 
+			// Files from server got through fetch() will bring its own
+			// reader without going through a stream.
+			if (!reader) {
+				stream = file.stream()
+				reader = stream.getReader({ mode: 'byob' })
+			}
+
 			let seq = 0
 			while (true) {
 				const { done, value } = await reader.read(buffer)
 				if (done) break
 
+				processedBytes += value.length
+				if (progressCallback)
+					progressCallback(processedBytes)
+
 				// Block is added as a file segment.
 				await this.fileSegments.add({
 					data: {
@@ -895,7 +906,6 @@ class FileNodesLinkStore extends SimpleNodeStore {
 
 class FileMetadata {
 	constructor(data) {// {{{
-		console.log('md', data)
 		this.UUID = data.UUID || uuidv7()
 		this.uploaded = data.uploaded || 'only_on_device'
 

static/js/page_storage.mjs

@@ -20,9 +20,16 @@ export class N2PageStorage extends CustomHTMLElement {
 					padding: 4px 8px;
 					border-radius: 4px;
 				}
+
+				h2 {
+					margin-top: 24px;
+					margin-bottom: 8px;
+				}
 			</style>
 
 			<h1>Local storage</h1>
+
+			<h2>Nodes</h2>
 			<div class="table">
 				<div>Local nodes</div>
 				<div data-el="count-nodes"></div>
@@ -34,11 +41,14 @@ export class N2PageStorage extends CustomHTMLElement {
 				<div data-el="count-history-nodes"></div>
 			</div>
 
-			<h1>Files</h1>
+			<h2>Files</h2>
 			<div class="table">
 				<div>Files on server</div>
 				<div data-el="count-files-on-server"></div>
 
+				<div>Files on device</div>
+				<div data-el="count-files-on-device"></div>
+
 				<div>Files to send</div>
 				<div data-el="count-queued-files"></div>
 			</div>
@@ -68,6 +78,8 @@ export class N2PageStorage extends CustomHTMLElement {
 		this.elCountQueuedNodes.innerText = await globalThis.nodeStore.sendQueue.count()
 		this.elCountHistoryNodes.innerText = await globalThis.nodeStore.nodesHistory.count()
 		this.elCountQueuedFiles.innerText = await globalThis.nodeStore.filesMetadata.countToUpload()
+
+		this.elCountFilesOnDevice.innerText = await globalThis.nodeStore.filesMetadata.count()
 	}
 }
 customElements.define('n2-pagestorage', N2PageStorage)

static/js/sidebar.mjs

@@ -84,7 +84,7 @@ export class N2Sidebar extends CustomHTMLElement {
 						justify-content: start;
 						margin-top: 16px;
 						gap: 8px 16px;
-						padding-left: 40px;
+						padding-left: 16px;
 						padding-bottom: 16px;
 						border-bottom: 1px solid var(--line-color);
 
@@ -112,6 +112,11 @@ export class N2Sidebar extends CustomHTMLElement {
 				<img data-el="search"   class='search colorize'   src="/images/${_VERSION}/icon_search.svg" style="height: 22px" />
 				<img data-el="settings" class='settings colorize' src="/images/${_VERSION}/icon_settings.svg" />
 			</div>
+			<div data-el="sync-status">
+				<img data-el="sync-icon" class="colorize">
+				<div data-el="sync-label"></div>
+				<progress data-el="sync-progress" min="0" max="100" value="0"></progress>
+			</div>
 			<div data-el="treenodes"></div>
 		`
 	}// }}}
@@ -126,6 +131,8 @@ export class N2Sidebar extends CustomHTMLElement {
 		this.expandedNodes = {} // keyed on UUID
 		this.selectedNode = null
 		this.rendered = false
+		this.nodesDownloadCount = 0
+		this.nodesUploadCount = 0
 
 		new TreeExpansionHandler()
 
@@ -134,7 +141,7 @@ export class N2Sidebar extends CustomHTMLElement {
 		this.elSync.addEventListener('click', () => _sync.run())
 		this.elStorage.addEventListener('click', () => _mbus.dispatch('SHOW_PAGE', { page: 'storage' }))
 		this.elLogo.addEventListener('click', () => _app.goToNode(ROOT_NODE, false, false))
-		this.elSettings.addEventListener('click', ()=> _mbus.dispatch('SHOW_PAGE', { page: 'preferences' }))
+		this.elSettings.addEventListener('click', () => _mbus.dispatch('SHOW_PAGE', { page: 'preferences' }))
 		this.elHideTree.addEventListener('click', event => {
 			event.stopPropagation()
 			_mbus.dispatch('TREE_EXPANSION', { expand: false })
@@ -151,6 +158,46 @@ export class N2Sidebar extends CustomHTMLElement {
 			treenode.render(true)
 		})
 
+		// SYNC_DOWNLOAD_COUNT and SYNC_UPLOAD_COUNT will be sent first before progress messages comes for either.
+		_mbus.subscribe('SYNC_DOWNLOAD_COUNT', event => this.nodesDownloadCount = event.detail.data.count)
+		_mbus.subscribe('SYNC_UPLOAD_COUNT', event => this.nodesUploadCount = event.detail.data.count)
+		_mbus.subscribe('SYNC_DOWNLOADED', event => {
+			this.elSyncIcon.classList.add('colorize')
+			this.elSyncIcon.setAttribute('src', `/images/${_VERSION}/icon_node_download.svg`)
+			this.elSyncProgress.setAttribute('max', this.nodesDownloadCount)
+			this.elSyncProgress.value = event.detail.data.handled
+		})
+		_mbus.subscribe('SYNC_UPLOADED', event => {
+			this.elSyncIcon.classList.add('colorize')
+			this.elSyncIcon.setAttribute('src', `/images/${_VERSION}/icon_node_upload.svg`)
+			this.elSyncProgress.setAttribute('max', this.nodesUploadCount)
+			this.elSyncProgress.value = event.detail.data.count
+		})
+		_mbus.subscribe('SYNC_DONE', () => {
+			this.elSyncIcon.classList.remove('colorize')
+			this.elSyncIcon.setAttribute('src', `/images/${_VERSION}/icon_sync_done.svg`)
+		})
+
+		_mbus.subscribe('FILE_SYNC', event => {
+			const { direction, name } = event.detail.data
+			this.elSyncIcon.setAttribute('src', `/images/${_VERSION}/icon_file_${direction}.svg`)
+			this.elSyncLabel.innerText = name
+		})
+
+		_mbus.subscribe('FILE_DOWNLOAD_PROGRESS', event => {
+			const { uuid, size, downloaded, percent, done } = event.detail.data
+			console.log(percent)
+
+			if (percent) {
+				this.elSyncProgress.value = percent
+			}
+
+			if (done) {
+				this.elSyncLabel.innerText = ''
+				this.elSyncProgress.value = 0
+			}
+		})
+
 		/* XXX - set color */
 		let color = new Color(0x80, 0x00, 0x33)
 		let solver = new Solver(color)

static/js/sync.mjs

@@ -10,8 +10,6 @@ export class Sync {
 
 	async run() {//{{{
 		try {
-			let duration = 0 // in ms
-
 			// The latest sync node value is used to retrieve the changes
 			// from the backend.
 			const state = await nodeStore.getAppState('latest_sync_node')
@@ -24,11 +22,12 @@ export class Sync {
 			_mbus.dispatch('SYNC_DOWNLOAD_COUNT', { count: nodeCountDownload })
 			_mbus.dispatch('SYNC_UPLOAD_COUNT', { count: nodeCountUpload })
 
-			await this.nodesFromServer(oldMax)
-				.then(durationNodes => {
-					duration = durationNodes // in ms
-					console.log(`Total time: ${Math.round(1000 * durationNodes) / 1000}s`)
-				})
+			const durationNodes = await this.nodesFromServer(oldMax)
+			console.log(`Total time: ${Math.round(1000 * durationNodes) / 1000}s`)
+
+			// Files are uploaded first since nodes can have references to them
+			// and the server requires linking a reference to the node and the file table.
+			await this.filesToServer()
 
 			// Uploads of modified nodes to server.
 			await this.nodesToServer()
@@ -80,8 +79,9 @@ export class Sync {
 					await this.handleNode(backendNode, objstore)
 
 					handled++
-					if (handled % 100 === 0)
-						_mbus.dispatch('SYNC_DOWNLOADED', { handled })
+					// XXX - should this be back for performance or not?
+					// if (handled % 100 === 0)
+					_mbus.dispatch('SYNC_DOWNLOADED', { handled })
 				}
 
 			} while (res.Continue)
@@ -195,7 +195,6 @@ export class Sync {
 			alert(e.message)
 		}
 	}// }}}
-
 	async filesFromServer() {// {{{
 		let offset = 0
 		let more = true
@@ -204,13 +203,64 @@ export class Sync {
 			while (more) {
 				const res = await API.query('GET', `/file/server_uuids/${offset}`)
 				more = res.MoreRowsExist
-				console.log(res)
+				offset++
+
+				for (const uuid of res.UUIDs) {
+					const md = await globalThis.nodeStore.filesMetadata.get(uuid)
+					if (!md) {
+						await this.storeFileFromServer(uuid)
+					}
+				}
 			}
 		} catch (e) {
 			console.error(e)
 			alert(e.message)
 		}
 	}// }}}
+	async storeFileFromServer(uuid) {// {{{
+		const headers = {}
+		const token = localStorage.getItem('token')
+		if (token) {
+			headers.Authorization = `Bearer ${token}`
+		}
+		const res = await fetch(`/file/uuid/${uuid}`, { headers })
+
+		const size = parseInt(res.headers.get('content-length'))
+		const fileData = {
+			UUID: uuid,
+			uploaded: 'on_server',
+			name: res.headers.get('x-file-name'),
+			size: res.headers.get('content-length'),
+			type: res.headers.get('content-type'),
+			modified: res.headers.get('x-file-modified'),
+		}
+
+		_mbus.dispatch('FILE_SYNC', {
+			direction: 'download',
+			name: fileData.name,
+		})
+
+		let prevPercent = 0
+		await globalThis.nodeStore.storeFile(fileData, res.body.getReader({ mode: 'byob' }), (downloaded) => {
+
+			const percent = Math.round((downloaded / size) * 100)
+			if (prevPercent != percent) {
+				_mbus.dispatch('FILE_DOWNLOAD_PROGRESS', {
+					uuid,
+					downloaded,
+					size,
+					percent,
+					done: false,
+				})
+				prevPercent = percent
+			}
+		})
+
+		_mbus.dispatch('FILE_DOWNLOAD_PROGRESS', {
+			uuid,
+			done: true,
+		})
+	}// }}}
 }
 
 export class N2SyncProgress extends CustomHTMLElement {