diff --git a/main.go b/main.go index 1466af1..ff935ad 100644 --- a/main.go +++ b/main.go @@ -27,7 +27,7 @@ import ( const VERSION = "v30" const CONTEXT_USER = 1 -const SYNC_PAGINATION = 1 +const SYNC_PAGINATION = 200 var ( FlagGenerate bool @@ -36,7 +36,6 @@ var ( FlagConfig string FlagCreateUser string FlagChangePassword string - FlagVersion bool Webengine HTMLTemplate.Engine config Config Log *slog.Logger @@ -67,7 +66,6 @@ func init() { // {{{ flag.BoolVar(&FlagLoremIpsum, "lorem-ipsum", false, "Replace all G- nodes with lorem ipsum paragraphs") flag.StringVar(&FlagCreateUser, "create-user", "", "Username for creating a new user") flag.StringVar(&FlagChangePassword, "change-password", "", "Change the password for the given username") - flag.BoolVar(&FlagVersion, "version", false, "Print version and exit") flag.Parse() RxpBearerToken = regexp.MustCompile("(?i)^\\s*Bearer\\s+(.*?)\\s*$") @@ -78,14 +76,7 @@ func initLog() { // {{{ Log = slog.New(slog.NewJSONHandler(os.Stdout, &opts)) } // }}} func main() { // {{{ - if FlagVersion { - fmt.Println(VERSION) - return - } - initLog() - Log.Info("application", "version", VERSION) - err := readConfig(FlagConfig) if err != nil { Log.Error("config", "error", err) @@ -287,9 +278,6 @@ func pageSync(w http.ResponseWriter, r *http.Request) { // {{{ } // }}} func actionSyncFromServer(w http.ResponseWriter, r *http.Request) { // {{{ - // XXX - DELETE ME! - time.Sleep(time.Millisecond * 200) - // The purpose of the Client UUID is to avoid // sending nodes back once again to a client that // just created or modified it. @@ -455,7 +443,7 @@ func actionSyncFileUpload(w http.ResponseWriter, r *http.Request) { // {{{ Log.Info("file_upload", "status", "done", "user", session.UserID, "uuid", uuid, "total", totalBytes) id := fileUUIDCache[uuid] delete(fileUUIDCache, uuid) - _, err = db.Exec(`UPDATE public.file SET upload_complete = true WHERE id = $1`, id) + _, err = db.Exec(`UPDATE public.file SET upload_complete = true WHERE id = $1`, id) default: id := fileUUIDCache[uuid] @@ -514,7 +502,7 @@ func actionUserSetPreferences(w http.ResponseWriter, r *http.Request) { // {{{ }) } // }}} -func actionFileCount(w http.ResponseWriter, r *http.Request) { // {{{ +func actionFileCount(w http.ResponseWriter, r *http.Request) {// {{{ session := getUserSession(r) count, err := FileCountForUser(session.UserID) if err != nil { @@ -524,11 +512,11 @@ func actionFileCount(w http.ResponseWriter, r *http.Request) { // {{{ } responseData(w, map[string]any{ - "OK": true, + "OK": true, "Count": count, }) -} // }}} -func actionFileServerUUIDs(w http.ResponseWriter, r *http.Request) { // {{{ +}// }}} +func actionFileServerUUIDs(w http.ResponseWriter, r *http.Request) {// {{{ session := getUserSession(r) offset, _ := strconv.Atoi(r.PathValue("offset")) uuids, more, err := FileServerUUIDsForUser(session.UserID, offset) @@ -539,12 +527,12 @@ func actionFileServerUUIDs(w http.ResponseWriter, r *http.Request) { // {{{ } responseData(w, map[string]any{ - "OK": true, - "UUIDs": uuids, + "OK": true, + "UUIDs": uuids, "MoreRowsExist": more, }) -} // }}} -func actionFile(w http.ResponseWriter, r *http.Request) { // {{{ +}// }}} +func actionFile(w http.ResponseWriter, r *http.Request) {// {{{ uuid := r.PathValue("uuid") session := getUserSession(r) f, md, err := FileForUser(session.UserID, uuid) @@ -554,13 +542,9 @@ func actionFile(w http.ResponseWriter, r *http.Request) { // {{{ return } - w.Header().Add("X-File-Name", md.Name) - w.Header().Add("X-File-Modified", fmt.Sprintf("%d", md.Modified.UnixMilli())) - w.Header().Add("Content-Type", md.Type) - http.ServeContent(w, r, md.Name, md.Modified, f) -} // }}} -func actionFileMetadata(w http.ResponseWriter, r *http.Request) { // {{{ +}// }}} +func actionFileMetadata(w http.ResponseWriter, r *http.Request) {// {{{ uuid := r.PathValue("uuid") session := getUserSession(r) md, err := FileMetadata(session.UserID, uuid) @@ -572,10 +556,10 @@ func actionFileMetadata(w http.ResponseWriter, r *http.Request) { // {{{ } responseData(w, map[string]any{ - "OK": true, + "OK": true, "Metadata": md, }) -} // }}} +}// }}} func createNewUser(username string) { // {{{ reader := bufio.NewReader(os.Stdin) diff --git a/node.go b/node.go index f95081c..15f277b 100644 --- a/node.go +++ b/node.go @@ -129,7 +129,6 @@ func NodesCount(userID int, synced uint64, clientUUID string) (count int, err er FROM public.node WHERE - NOT special AND user_id = $1 AND client != $3 AND ( diff --git a/static/css/notes2.css b/static/css/notes2.css index 4766167..7fdea0b 100644 --- a/static/css/notes2.css +++ b/static/css/notes2.css @@ -178,34 +178,6 @@ button { border-right: 1px solid var(--line-color); n2-sidebar { - display: grid; - grid-template-rows: min-content min-content min-content 1fr; - - .el-sync-status { - display: grid; - grid-template-columns: min-content 1fr; - grid-gap: 0px 8px; - align-items: center; - padding: 8px 16px 8px 16px; - border-bottom: 1px solid var(--line-color); - - .el-sync-icon { - grid-column: 1; - grid-row: 1 / 3; - } - - .el-sync-label { - grid-column: 2; - grid-row: 1; - } - - progress { - grid-column: 2; - grid-row: 2; - width: 100%; - } - } - .el-treenodes { margin: 24px 32px 32px 32px; } diff --git a/static/js/node_store.mjs b/static/js/node_store.mjs index 531626b..bf7c4e8 100644 --- a/static/js/node_store.mjs +++ b/static/js/node_store.mjs @@ -566,32 +566,21 @@ export class NodeStore { }) }//}}} - async storeFile(file, reader, progressCallback) {// {{{ + async storeFile(file) {// {{{ const metadata = new FileMetadata(file) - let processedBytes = 0 - let stream try { // Do potentially larger blocks. const BLOCKSIZE = 1048576 + const stream = file.stream() + const reader = stream.getReader({ mode: 'byob' }) let buffer = new Uint8Array(BLOCKSIZE) - // Files from server got through fetch() will bring its own - // reader without going through a stream. - if (!reader) { - stream = file.stream() - reader = stream.getReader({ mode: 'byob' }) - } - let seq = 0 while (true) { const { done, value } = await reader.read(buffer) if (done) break - processedBytes += value.length - if (progressCallback) - progressCallback(processedBytes) - // Block is added as a file segment. await this.fileSegments.add({ data: { @@ -906,6 +895,7 @@ class FileNodesLinkStore extends SimpleNodeStore { class FileMetadata { constructor(data) {// {{{ + console.log('md', data) this.UUID = data.UUID || uuidv7() this.uploaded = data.uploaded || 'only_on_device' diff --git a/static/js/page_storage.mjs b/static/js/page_storage.mjs index f6f562f..96f50ac 100644 --- a/static/js/page_storage.mjs +++ b/static/js/page_storage.mjs @@ -20,16 +20,9 @@ export class N2PageStorage extends CustomHTMLElement { padding: 4px 8px; border-radius: 4px; } - - h2 { - margin-top: 24px; - margin-bottom: 8px; - }

Local storage

- -

Nodes

Local nodes
@@ -41,14 +34,11 @@ export class N2PageStorage extends CustomHTMLElement {
-

Files

+

Files

Files on server
-
Files on device
-
-
Files to send
@@ -78,8 +68,6 @@ export class N2PageStorage extends CustomHTMLElement { this.elCountQueuedNodes.innerText = await globalThis.nodeStore.sendQueue.count() this.elCountHistoryNodes.innerText = await globalThis.nodeStore.nodesHistory.count() this.elCountQueuedFiles.innerText = await globalThis.nodeStore.filesMetadata.countToUpload() - - this.elCountFilesOnDevice.innerText = await globalThis.nodeStore.filesMetadata.count() } } customElements.define('n2-pagestorage', N2PageStorage) diff --git a/static/js/sidebar.mjs b/static/js/sidebar.mjs index fa47110..899e955 100644 --- a/static/js/sidebar.mjs +++ b/static/js/sidebar.mjs @@ -84,7 +84,7 @@ export class N2Sidebar extends CustomHTMLElement { justify-content: start; margin-top: 16px; gap: 8px 16px; - padding-left: 16px; + padding-left: 40px; padding-bottom: 16px; border-bottom: 1px solid var(--line-color); @@ -112,11 +112,6 @@ export class N2Sidebar extends CustomHTMLElement { -
- -
- -
` }// }}} @@ -131,8 +126,6 @@ export class N2Sidebar extends CustomHTMLElement { this.expandedNodes = {} // keyed on UUID this.selectedNode = null this.rendered = false - this.nodesDownloadCount = 0 - this.nodesUploadCount = 0 new TreeExpansionHandler() @@ -141,7 +134,7 @@ export class N2Sidebar extends CustomHTMLElement { this.elSync.addEventListener('click', () => _sync.run()) this.elStorage.addEventListener('click', () => _mbus.dispatch('SHOW_PAGE', { page: 'storage' })) this.elLogo.addEventListener('click', () => _app.goToNode(ROOT_NODE, false, false)) - this.elSettings.addEventListener('click', () => _mbus.dispatch('SHOW_PAGE', { page: 'preferences' })) + this.elSettings.addEventListener('click', ()=> _mbus.dispatch('SHOW_PAGE', { page: 'preferences' })) this.elHideTree.addEventListener('click', event => { event.stopPropagation() _mbus.dispatch('TREE_EXPANSION', { expand: false }) @@ -158,46 +151,6 @@ export class N2Sidebar extends CustomHTMLElement { treenode.render(true) }) - // SYNC_DOWNLOAD_COUNT and SYNC_UPLOAD_COUNT will be sent first before progress messages comes for either. - _mbus.subscribe('SYNC_DOWNLOAD_COUNT', event => this.nodesDownloadCount = event.detail.data.count) - _mbus.subscribe('SYNC_UPLOAD_COUNT', event => this.nodesUploadCount = event.detail.data.count) - _mbus.subscribe('SYNC_DOWNLOADED', event => { - this.elSyncIcon.classList.add('colorize') - this.elSyncIcon.setAttribute('src', `/images/${_VERSION}/icon_node_download.svg`) - this.elSyncProgress.setAttribute('max', this.nodesDownloadCount) - this.elSyncProgress.value = event.detail.data.handled - }) - _mbus.subscribe('SYNC_UPLOADED', event => { - this.elSyncIcon.classList.add('colorize') - this.elSyncIcon.setAttribute('src', `/images/${_VERSION}/icon_node_upload.svg`) - this.elSyncProgress.setAttribute('max', this.nodesUploadCount) - this.elSyncProgress.value = event.detail.data.count - }) - _mbus.subscribe('SYNC_DONE', () => { - this.elSyncIcon.classList.remove('colorize') - this.elSyncIcon.setAttribute('src', `/images/${_VERSION}/icon_sync_done.svg`) - }) - - _mbus.subscribe('FILE_SYNC', event => { - const { direction, name } = event.detail.data - this.elSyncIcon.setAttribute('src', `/images/${_VERSION}/icon_file_${direction}.svg`) - this.elSyncLabel.innerText = name - }) - - _mbus.subscribe('FILE_DOWNLOAD_PROGRESS', event => { - const { uuid, size, downloaded, percent, done } = event.detail.data - console.log(percent) - - if (percent) { - this.elSyncProgress.value = percent - } - - if (done) { - this.elSyncLabel.innerText = '' - this.elSyncProgress.value = 0 - } - }) - /* XXX - set color */ let color = new Color(0x80, 0x00, 0x33) let solver = new Solver(color) diff --git a/static/js/sync.mjs b/static/js/sync.mjs index 354fc54..f68a09f 100644 --- a/static/js/sync.mjs +++ b/static/js/sync.mjs @@ -10,6 +10,8 @@ export class Sync { async run() {//{{{ try { + let duration = 0 // in ms + // The latest sync node value is used to retrieve the changes // from the backend. const state = await nodeStore.getAppState('latest_sync_node') @@ -22,12 +24,11 @@ export class Sync { _mbus.dispatch('SYNC_DOWNLOAD_COUNT', { count: nodeCountDownload }) _mbus.dispatch('SYNC_UPLOAD_COUNT', { count: nodeCountUpload }) - const durationNodes = await this.nodesFromServer(oldMax) - console.log(`Total time: ${Math.round(1000 * durationNodes) / 1000}s`) - - // Files are uploaded first since nodes can have references to them - // and the server requires linking a reference to the node and the file table. - await this.filesToServer() + await this.nodesFromServer(oldMax) + .then(durationNodes => { + duration = durationNodes // in ms + console.log(`Total time: ${Math.round(1000 * durationNodes) / 1000}s`) + }) // Uploads of modified nodes to server. await this.nodesToServer() @@ -79,9 +80,8 @@ export class Sync { await this.handleNode(backendNode, objstore) handled++ - // XXX - should this be back for performance or not? - // if (handled % 100 === 0) - _mbus.dispatch('SYNC_DOWNLOADED', { handled }) + if (handled % 100 === 0) + _mbus.dispatch('SYNC_DOWNLOADED', { handled }) } } while (res.Continue) @@ -195,6 +195,7 @@ export class Sync { alert(e.message) } }// }}} + async filesFromServer() {// {{{ let offset = 0 let more = true @@ -203,64 +204,13 @@ export class Sync { while (more) { const res = await API.query('GET', `/file/server_uuids/${offset}`) more = res.MoreRowsExist - offset++ - - for (const uuid of res.UUIDs) { - const md = await globalThis.nodeStore.filesMetadata.get(uuid) - if (!md) { - await this.storeFileFromServer(uuid) - } - } + console.log(res) } } catch (e) { console.error(e) alert(e.message) } }// }}} - async storeFileFromServer(uuid) {// {{{ - const headers = {} - const token = localStorage.getItem('token') - if (token) { - headers.Authorization = `Bearer ${token}` - } - const res = await fetch(`/file/uuid/${uuid}`, { headers }) - - const size = parseInt(res.headers.get('content-length')) - const fileData = { - UUID: uuid, - uploaded: 'on_server', - name: res.headers.get('x-file-name'), - size: res.headers.get('content-length'), - type: res.headers.get('content-type'), - modified: res.headers.get('x-file-modified'), - } - - _mbus.dispatch('FILE_SYNC', { - direction: 'download', - name: fileData.name, - }) - - let prevPercent = 0 - await globalThis.nodeStore.storeFile(fileData, res.body.getReader({ mode: 'byob' }), (downloaded) => { - - const percent = Math.round((downloaded / size) * 100) - if (prevPercent != percent) { - _mbus.dispatch('FILE_DOWNLOAD_PROGRESS', { - uuid, - downloaded, - size, - percent, - done: false, - }) - prevPercent = percent - } - }) - - _mbus.dispatch('FILE_DOWNLOAD_PROGRESS', { - uuid, - done: true, - }) - }// }}} } export class N2SyncProgress extends CustomHTMLElement {