diff --git a/main.go b/main.go index aeba690..1466af1 100644 --- a/main.go +++ b/main.go @@ -27,7 +27,7 @@ import ( const VERSION = "v30" const CONTEXT_USER = 1 -const SYNC_PAGINATION = 200 +const SYNC_PAGINATION = 1 var ( FlagGenerate bool @@ -287,6 +287,9 @@ func pageSync(w http.ResponseWriter, r *http.Request) { // {{{ } // }}} func actionSyncFromServer(w http.ResponseWriter, r *http.Request) { // {{{ + // XXX - DELETE ME! + time.Sleep(time.Millisecond * 200) + // The purpose of the Client UUID is to avoid // sending nodes back once again to a client that // just created or modified it. @@ -551,6 +554,10 @@ func actionFile(w http.ResponseWriter, r *http.Request) { // {{{ return } + w.Header().Add("X-File-Name", md.Name) + w.Header().Add("X-File-Modified", fmt.Sprintf("%d", md.Modified.UnixMilli())) + w.Header().Add("Content-Type", md.Type) + http.ServeContent(w, r, md.Name, md.Modified, f) } // }}} func actionFileMetadata(w http.ResponseWriter, r *http.Request) { // {{{ diff --git a/node.go b/node.go index 15f277b..f95081c 100644 --- a/node.go +++ b/node.go @@ -129,6 +129,7 @@ func NodesCount(userID int, synced uint64, clientUUID string) (count int, err er FROM public.node WHERE + NOT special AND user_id = $1 AND client != $3 AND ( diff --git a/static/css/notes2.css b/static/css/notes2.css index 7fdea0b..4766167 100644 --- a/static/css/notes2.css +++ b/static/css/notes2.css @@ -178,6 +178,34 @@ button { border-right: 1px solid var(--line-color); n2-sidebar { + display: grid; + grid-template-rows: min-content min-content min-content 1fr; + + .el-sync-status { + display: grid; + grid-template-columns: min-content 1fr; + grid-gap: 0px 8px; + align-items: center; + padding: 8px 16px 8px 16px; + border-bottom: 1px solid var(--line-color); + + .el-sync-icon { + grid-column: 1; + grid-row: 1 / 3; + } + + .el-sync-label { + grid-column: 2; + grid-row: 1; + } + + progress { + grid-column: 2; + grid-row: 2; + width: 100%; + } + } + .el-treenodes { margin: 24px 32px 32px 32px; } diff --git a/static/js/node_store.mjs b/static/js/node_store.mjs index bf7c4e8..531626b 100644 --- a/static/js/node_store.mjs +++ b/static/js/node_store.mjs @@ -566,21 +566,32 @@ export class NodeStore { }) }//}}} - async storeFile(file) {// {{{ + async storeFile(file, reader, progressCallback) {// {{{ const metadata = new FileMetadata(file) + let processedBytes = 0 + let stream try { // Do potentially larger blocks. const BLOCKSIZE = 1048576 - const stream = file.stream() - const reader = stream.getReader({ mode: 'byob' }) let buffer = new Uint8Array(BLOCKSIZE) + // Files from server got through fetch() will bring its own + // reader without going through a stream. + if (!reader) { + stream = file.stream() + reader = stream.getReader({ mode: 'byob' }) + } + let seq = 0 while (true) { const { done, value } = await reader.read(buffer) if (done) break + processedBytes += value.length + if (progressCallback) + progressCallback(processedBytes) + // Block is added as a file segment. await this.fileSegments.add({ data: { @@ -895,7 +906,6 @@ class FileNodesLinkStore extends SimpleNodeStore { class FileMetadata { constructor(data) {// {{{ - console.log('md', data) this.UUID = data.UUID || uuidv7() this.uploaded = data.uploaded || 'only_on_device' diff --git a/static/js/page_storage.mjs b/static/js/page_storage.mjs index 96f50ac..f6f562f 100644 --- a/static/js/page_storage.mjs +++ b/static/js/page_storage.mjs @@ -20,9 +20,16 @@ export class N2PageStorage extends CustomHTMLElement { padding: 4px 8px; border-radius: 4px; } + + h2 { + margin-top: 24px; + margin-bottom: 8px; + }

Local storage

+ +

Nodes

Local nodes
@@ -34,11 +41,14 @@ export class N2PageStorage extends CustomHTMLElement {
-

Files

+

Files

Files on server
+
Files on device
+
+
Files to send
@@ -68,6 +78,8 @@ export class N2PageStorage extends CustomHTMLElement { this.elCountQueuedNodes.innerText = await globalThis.nodeStore.sendQueue.count() this.elCountHistoryNodes.innerText = await globalThis.nodeStore.nodesHistory.count() this.elCountQueuedFiles.innerText = await globalThis.nodeStore.filesMetadata.countToUpload() + + this.elCountFilesOnDevice.innerText = await globalThis.nodeStore.filesMetadata.count() } } customElements.define('n2-pagestorage', N2PageStorage) diff --git a/static/js/sidebar.mjs b/static/js/sidebar.mjs index 899e955..fa47110 100644 --- a/static/js/sidebar.mjs +++ b/static/js/sidebar.mjs @@ -84,7 +84,7 @@ export class N2Sidebar extends CustomHTMLElement { justify-content: start; margin-top: 16px; gap: 8px 16px; - padding-left: 40px; + padding-left: 16px; padding-bottom: 16px; border-bottom: 1px solid var(--line-color); @@ -112,6 +112,11 @@ export class N2Sidebar extends CustomHTMLElement { +
+ +
+ +
` }// }}} @@ -126,6 +131,8 @@ export class N2Sidebar extends CustomHTMLElement { this.expandedNodes = {} // keyed on UUID this.selectedNode = null this.rendered = false + this.nodesDownloadCount = 0 + this.nodesUploadCount = 0 new TreeExpansionHandler() @@ -134,7 +141,7 @@ export class N2Sidebar extends CustomHTMLElement { this.elSync.addEventListener('click', () => _sync.run()) this.elStorage.addEventListener('click', () => _mbus.dispatch('SHOW_PAGE', { page: 'storage' })) this.elLogo.addEventListener('click', () => _app.goToNode(ROOT_NODE, false, false)) - this.elSettings.addEventListener('click', ()=> _mbus.dispatch('SHOW_PAGE', { page: 'preferences' })) + this.elSettings.addEventListener('click', () => _mbus.dispatch('SHOW_PAGE', { page: 'preferences' })) this.elHideTree.addEventListener('click', event => { event.stopPropagation() _mbus.dispatch('TREE_EXPANSION', { expand: false }) @@ -151,6 +158,46 @@ export class N2Sidebar extends CustomHTMLElement { treenode.render(true) }) + // SYNC_DOWNLOAD_COUNT and SYNC_UPLOAD_COUNT will be sent first before progress messages comes for either. + _mbus.subscribe('SYNC_DOWNLOAD_COUNT', event => this.nodesDownloadCount = event.detail.data.count) + _mbus.subscribe('SYNC_UPLOAD_COUNT', event => this.nodesUploadCount = event.detail.data.count) + _mbus.subscribe('SYNC_DOWNLOADED', event => { + this.elSyncIcon.classList.add('colorize') + this.elSyncIcon.setAttribute('src', `/images/${_VERSION}/icon_node_download.svg`) + this.elSyncProgress.setAttribute('max', this.nodesDownloadCount) + this.elSyncProgress.value = event.detail.data.handled + }) + _mbus.subscribe('SYNC_UPLOADED', event => { + this.elSyncIcon.classList.add('colorize') + this.elSyncIcon.setAttribute('src', `/images/${_VERSION}/icon_node_upload.svg`) + this.elSyncProgress.setAttribute('max', this.nodesUploadCount) + this.elSyncProgress.value = event.detail.data.count + }) + _mbus.subscribe('SYNC_DONE', () => { + this.elSyncIcon.classList.remove('colorize') + this.elSyncIcon.setAttribute('src', `/images/${_VERSION}/icon_sync_done.svg`) + }) + + _mbus.subscribe('FILE_SYNC', event => { + const { direction, name } = event.detail.data + this.elSyncIcon.setAttribute('src', `/images/${_VERSION}/icon_file_${direction}.svg`) + this.elSyncLabel.innerText = name + }) + + _mbus.subscribe('FILE_DOWNLOAD_PROGRESS', event => { + const { uuid, size, downloaded, percent, done } = event.detail.data + console.log(percent) + + if (percent) { + this.elSyncProgress.value = percent + } + + if (done) { + this.elSyncLabel.innerText = '' + this.elSyncProgress.value = 0 + } + }) + /* XXX - set color */ let color = new Color(0x80, 0x00, 0x33) let solver = new Solver(color) diff --git a/static/js/sync.mjs b/static/js/sync.mjs index f68a09f..354fc54 100644 --- a/static/js/sync.mjs +++ b/static/js/sync.mjs @@ -10,8 +10,6 @@ export class Sync { async run() {//{{{ try { - let duration = 0 // in ms - // The latest sync node value is used to retrieve the changes // from the backend. const state = await nodeStore.getAppState('latest_sync_node') @@ -24,11 +22,12 @@ export class Sync { _mbus.dispatch('SYNC_DOWNLOAD_COUNT', { count: nodeCountDownload }) _mbus.dispatch('SYNC_UPLOAD_COUNT', { count: nodeCountUpload }) - await this.nodesFromServer(oldMax) - .then(durationNodes => { - duration = durationNodes // in ms - console.log(`Total time: ${Math.round(1000 * durationNodes) / 1000}s`) - }) + const durationNodes = await this.nodesFromServer(oldMax) + console.log(`Total time: ${Math.round(1000 * durationNodes) / 1000}s`) + + // Files are uploaded first since nodes can have references to them + // and the server requires linking a reference to the node and the file table. + await this.filesToServer() // Uploads of modified nodes to server. await this.nodesToServer() @@ -80,8 +79,9 @@ export class Sync { await this.handleNode(backendNode, objstore) handled++ - if (handled % 100 === 0) - _mbus.dispatch('SYNC_DOWNLOADED', { handled }) + // XXX - should this be back for performance or not? + // if (handled % 100 === 0) + _mbus.dispatch('SYNC_DOWNLOADED', { handled }) } } while (res.Continue) @@ -195,7 +195,6 @@ export class Sync { alert(e.message) } }// }}} - async filesFromServer() {// {{{ let offset = 0 let more = true @@ -204,13 +203,64 @@ export class Sync { while (more) { const res = await API.query('GET', `/file/server_uuids/${offset}`) more = res.MoreRowsExist - console.log(res) + offset++ + + for (const uuid of res.UUIDs) { + const md = await globalThis.nodeStore.filesMetadata.get(uuid) + if (!md) { + await this.storeFileFromServer(uuid) + } + } } } catch (e) { console.error(e) alert(e.message) } }// }}} + async storeFileFromServer(uuid) {// {{{ + const headers = {} + const token = localStorage.getItem('token') + if (token) { + headers.Authorization = `Bearer ${token}` + } + const res = await fetch(`/file/uuid/${uuid}`, { headers }) + + const size = parseInt(res.headers.get('content-length')) + const fileData = { + UUID: uuid, + uploaded: 'on_server', + name: res.headers.get('x-file-name'), + size: res.headers.get('content-length'), + type: res.headers.get('content-type'), + modified: res.headers.get('x-file-modified'), + } + + _mbus.dispatch('FILE_SYNC', { + direction: 'download', + name: fileData.name, + }) + + let prevPercent = 0 + await globalThis.nodeStore.storeFile(fileData, res.body.getReader({ mode: 'byob' }), (downloaded) => { + + const percent = Math.round((downloaded / size) * 100) + if (prevPercent != percent) { + _mbus.dispatch('FILE_DOWNLOAD_PROGRESS', { + uuid, + downloaded, + size, + percent, + done: false, + }) + prevPercent = percent + } + }) + + _mbus.dispatch('FILE_DOWNLOAD_PROGRESS', { + uuid, + done: true, + }) + }// }}} } export class N2SyncProgress extends CustomHTMLElement {