magnus/Notes2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Cleanup of old code, moved Node class, many improvements on file handling

Browse source
This commit is contained in:
Magnus Åhall 2026-06-29 10:43:21 +02:00
parent 65f8cd14a7
commit 16992db6b1
19 changed files with 485 additions and 3281 deletions
Download patch file Download diff file Expand all files Collapse all files

37
file.go
View file

@ -129,6 +129,43 @@ func FileCountForUser(userID int) (count int, err error) { // {{{
err = row.Scan(&count)
return
} // }}}
func FileServerUUIDsForUser(userID, page int) (uuids []string, more bool, err error) { // {{{
uuids = []string{}
var rows *sqlx.Rows
rows, err = db.Queryx(`
SELECT
uuid
FROM public.file
WHERE
user_id = $1
ORDER BY
uuid ASC
LIMIT $2 OFFSET $3
`,
userID,
SYNC_PAGINATION+1,
page*SYNC_PAGINATION,
)
if err != nil {
return
}
defer rows.Close()
var uuid string
for rows.Next() {
err = rows.Scan(&uuid)
if err != nil {
return
}
uuids = append(uuids, uuid)
}
more = len(uuids) > SYNC_PAGINATION
uuids = uuids[0:min(SYNC_PAGINATION, len(uuids))]
return
} // }}}
func FileMetadata(userID int, uuid string) (f File, err error) { // {{{
row := db.QueryRowx(`SELECT * FROM public.file WHERE user_id = $1 AND "uuid" = $2`, userID, uuid)
err = row.StructScan(&f)

21
main.go
View file

@ -149,8 +149,9 @@ func main() { // {{{
http.HandleFunc("/node/history/count/{uuid}", authenticated(actionNodeHistoryCount))
http.HandleFunc("/file/count", authenticated(actionFileCount))
http.HandleFunc("/file/{uuid}", authenticated(actionFile))
http.HandleFunc("/file/{uuid}/metadata", authenticated(actionFileMetadata))
http.HandleFunc("/file/server_uuids/{offset}", authenticated(actionFileServerUUIDs))
http.HandleFunc("/file/uuid/{uuid}", authenticated(actionFile))
http.HandleFunc("/file/uuid/{uuid}/metadata", authenticated(actionFileMetadata))
http.HandleFunc("/service_worker.js", pageServiceWorker)
@ -515,6 +516,22 @@ func actionFileCount(w http.ResponseWriter, r *http.Request) {// {{{
"Count": count,
})
}// }}}
func actionFileServerUUIDs(w http.ResponseWriter, r *http.Request) {// {{{
session := getUserSession(r)
offset, _ := strconv.Atoi(r.PathValue("offset"))
uuids, more, err := FileServerUUIDsForUser(session.UserID, offset)
if err != nil {
Log.Error("file_server_uuids", "error", err)
httpError(w, err)
return
}
responseData(w, map[string]any{
"OK": true,
"UUIDs": uuids,
"MoreRowsExist": more,
})
}// }}}
func actionFile(w http.ResponseWriter, r *http.Request) {// {{{
uuid := r.PathValue("uuid")
session := getUserSession(r)

61
node.go
View file

@ -57,67 +57,6 @@ type Node struct {
Special bool
}
func NodeTree(userID, offset int, synced uint64) (nodes []TreeNode, maxSeq uint64, moreRowsExist bool, err error) { // {{{
const LIMIT = 100
var rows *sqlx.Rows
rows, err = db.Queryx(`
SELECT
uuid,
COALESCE(parent_uuid, '') AS parent_uuid,
name,
created,
updated,
deleted IS NOT NULL AS deleted,
created_seq,
updated_seq,
deleted_seq
FROM
public.node
WHERE
user_id = $1 AND
(
created_seq > $4 OR
updated_seq > $4 OR
deleted_seq > $4
)
ORDER BY
created ASC
LIMIT $2 OFFSET $3
`,
userID,
LIMIT+1,
offset,
synced,
)
if err != nil {
return
}
defer rows.Close()
nodes = []TreeNode{}
numNodes := 0
for rows.Next() {
// Query selects up to one more row than the decided limit.
// Saves one SQL query for row counting.
// Thus if numNodes is larger than the limit, more rows exist for the next call.
numNodes++
if numNodes > LIMIT {
moreRowsExist = true
return
}
node := TreeNode{}
if err = rows.StructScan(&node); err != nil {
return
}
nodes = append(nodes, node)
// DeletedSeq will be 0 if invalid, and thus not be a problem for the max function.
maxSeq = max(maxSeq, node.CreatedSeq, node.UpdatedSeq, uint64(node.DeletedSeq.Int64))
}
return
} // }}}
func Nodes(userID, offset int, synced uint64, clientUUID string) (nodes []Node, maxSeq uint64, moreRowsExist bool, err error) { // {{{
var rows *sqlx.Rows
rows, err = db.Queryx(`

13
static/js/api.mjs
View file

@ -23,7 +23,6 @@ export class API {
throw new Error(json.Error, { cause: { type: 'application', application: json, } })
return json
} catch (err) {
// Catch any other errors from fetch.
throw new Error(err.message, { cause: { type: 'http', error: err, } })
@ -31,21 +30,21 @@ export class API {
}
// Sends a block of binary data to server.
static async upload(uuid, data, sentBytes, file) {
static async upload(uuid, data, sentBytes, metadata) {
try {
const path = `/sync/file/upload/${uuid}`
const headers = {
'X-File-Sent-Bytes': sentBytes,
'X-File-Total-Bytes': file.size,
'X-File-Total-Bytes': metadata.size,
}
// Metadata is needed for the database.
// No need to send it every block.
if (sentBytes === 0) {
headers['X-File-Name'] = file.name
headers['X-File-Size'] = file.size
headers['X-File-Type'] = file.type
headers['X-File-Modified'] = file.lastModified
headers['X-File-Name'] = metadata.name
headers['X-File-Size'] = metadata.size
headers['X-File-Type'] = metadata.type
headers['X-File-Modified'] = metadata.modified
}
// Authentication is done with a bearer token.

11
static/js/app.mjs
View file

@ -1,7 +1,6 @@
import { ROOT_NODE } from 'node_store'
import { ROOT_NODE, Node } from 'node_store'
import { CustomHTMLElement } from './lib/custom_html_element.mjs'
import { N2Sidebar } from 'sidebar'
import { Node } from 'node'
import { N2PreferenceSet } from './page_preferences.mjs'
export class App {
@ -68,8 +67,8 @@ export class App {
this.preferences = this.getPreferences()
})
window.addEventListener('keydown', event => this.keyHandler(event))
window.addEventListener('popstate', event => this.popState(event))
globalThis.addEventListener('keydown', event => this.keyHandler(event))
globalThis.addEventListener('popstate', event => this.popState(event))
document.getElementById('notes2').addEventListener('click', event => {
if (event.target.id === 'notes2')
document.getElementById('node-content')?.focus()
@ -80,12 +79,12 @@ export class App {
_mbus.dispatch('SHOW_PAGE', { page: 'node' })
window._sync = new Sync()
globalThis._sync = new Sync()
// I think it is uncomfortable having the sync running as soon as the page load.
// I haven't gotten the time to look at the page before stuff jumps around.
// There a slight delay to initiate sync seems reasonable.
setTimeout(() => window._sync.run(), 1000)
setTimeout(() => globalThis._sync.run(), 1000)
}// }}}
keyHandler(event) {//{{{
let handled = true

72
static/js/crypto.mjs
View file

@ -1,72 +0,0 @@
export default class Crypto {
constructor(key) {//{{{
if(key === null)
throw new Error("No key provided")
if(typeof key === 'string')
this.key = sjcl.codec.base64.toBits(base64_key)
else
this.key = key
this.aes = new sjcl.cipher.aes(this.key)
}//}}}
static generate_key() {//{{{
return sjcl.random.randomWords(8)
}//}}}
static pass_to_key(pass, salt = null) {//{{{
if(salt === null)
salt = sjcl.random.randomWords(4) // 128 bits (16 bytes)
let key = sjcl.misc.pbkdf2(pass, salt, 10000)
return {
salt,
key,
}
}//}}}
encrypt(plaintext_data_in_bits, counter, return_encoded = true) {//{{{
// 8 bytes of random data, (1 word = 4 bytes) * 2
// with 8 bytes of byte encoded counter is used as
// IV to guarantee a non-repeated IV (which is a catastrophe).
// Assumes counter value is kept unique. Counter is taken from
// Postgres sequence.
let random_bits = sjcl.random.randomWords(2)
let iv_bytes = sjcl.codec.bytes.fromBits(random_bits)
for (let i = 0; i < 8; ++i) {
let mask = 0xffn << BigInt(i*8)
let counter_i_byte = (counter & mask) >> BigInt(i*8)
iv_bytes[15-i] = Number(counter_i_byte)
}
let iv = sjcl.codec.bytes.toBits(iv_bytes)
let encrypted = sjcl.mode['ccm'].encrypt(
this.aes,
plaintext_data_in_bits,
iv,
)
// Returning 16 bytes (4 words) IV + encrypted data.
if(return_encoded)
return sjcl.codec.base64.fromBits(
iv.concat(encrypted)
)
else
return iv.concat(encrypted)
}//}}}
decrypt(encrypted_base64_data) {//{{{
try {
let encoded = sjcl.codec.base64.toBits(encrypted_base64_data)
let iv = encoded.slice(0, 4) // in words (4 bytes), not bytes
let encrypted_data = encoded.slice(4)
return sjcl.mode['ccm'].decrypt(this.aes, encrypted_data, iv)
} catch(err) {
if(err.message == `ccm: tag doesn't match`)
throw('Decryption failed')
else
throw(err)
}
}//}}}
}
// vim: foldmethod=marker

48
static/js/file.mjs
View file

@ -35,7 +35,7 @@ export class N2File extends CustomHTMLElement {
constructor() {// {{{
super(true)
this.file = null
this.streamLocalFile = false
this.uuid = null
this.addEventListener('click', event => {
@ -54,35 +54,32 @@ export class N2File extends CustomHTMLElement {
this.render()
}// }}}
openInBrowser(event) {
// this.file is only defined when file is stored in IndexedDB.
if (this.file)
openInBrowser(event) {// {{{
if (this.streamLocalFile)
window.open(
URL.createObjectURL(this.file),
`/stream-local?uuid=${this.metadata.UUID}`,
event.ctrlKey ? '_blank' : '_self',
)
else {
const jwt = localStorage.getItem('token')
window.open(
`/file/${this.uuid}?jwt=${jwt}`,
`/file/uuid/${this.uuid}?jwt=${jwt}`,
event.ctrlKey ? '_blank' : '_self',
)
}
}
}// }}}
async downloadToDisk() {// {{{
try {
// this.file is only defined when file is stored in IndexedDB.
// TODO: implement download of remote content.
if (this.file === null)
if (this.streamLocalFile === false)
throw new Error('This file is only available online. Download not implemented yet.')
const handle = await window.showSaveFilePicker({
suggestedName: this.file.name,
suggestedName: this.metadata.name,
})
const writable = await handle.createWritable()
const blobStream = this.file.stream()
const blobStream = await globalThis.nodeStore.fileSegments.getStream(this.uuid)
await blobStream.pipeTo(writable)
} catch (err) {
if (err.name == 'AbortError')
@ -101,18 +98,21 @@ export class N2File extends CustomHTMLElement {
// populateImg makes sure this returned img element exists and then populates it
// with the image from IndexedDB.
const fileUUID = src.slice(5)
this.uuid = fileUUID
const file = await globalThis.nodeStore.files.get(fileUUID)
if (file) {
this.renderLocalFile(file)
this.uuid = fileUUID
const md = await globalThis.nodeStore.filesMetadata.get(fileUUID)
if (md) {
this.metadata = md
this.streamLocalFile = true
this.renderLocalFile(md)
} else {
/* <file> can be undefined when a node is retrieved but files aren't synced.
* Try to show from server.
*
* Metadata is needed to know whether to display an image or the file representation. */
try {
const res = await API.query('GET', `/file/${fileUUID}/metadata`)
const res = await API.query('GET', `/file/uuid/${fileUUID}/metadata`)
this.renderRemoteFile(res.Metadata)
} catch (err) {
console.error(err)
@ -123,27 +123,25 @@ export class N2File extends CustomHTMLElement {
// Probably an external URL.
this.elImage.src = src
}// }}}
async renderLocalFile(file) {// {{{
this.file = file.file
if (file.file.type.startsWith('image/'))
this.elImage.src = URL.createObjectURL(file.file)
async renderLocalFile(fileMetadata) {// {{{
if (fileMetadata.type.startsWith('image/'))
this.elImage.src = `/stream-local?uuid=${fileMetadata.UUID}`
else {
// Check for and use an existing MIME type icon.
// Place them in static/images/file_icons/ and replace the slash with an underscore.
const url = `/images/${_VERSION}/file_icons/${file.file.type.replaceAll('/', '_')}.svg`
const url = `/images/${_VERSION}/file_icons/${fileMetadata.type.replaceAll('/', '_')}.svg`
const res = await fetch(url)
if (res.ok)
this.elImage.src = url
this.elFilename.innerText = file.file.name
this.elFilename.innerText = fileMetadata.name
this.elFilename.style.display = 'block'
}
}// }}}
async renderRemoteFile(md) {// {{{
if (md.Type.startsWith('image/')) {
const jwt = localStorage.getItem('token')
this.elImage.src = `/file/${md.UUID}?jwt=${jwt}`
this.elImage.src = `/file/uuid/${md.UUID}?jwt=${jwt}`
} else {
// Check for and use an existing MIME type icon.
// Place them in static/images/file_icons/ and replace the slash with an underscore.

240
static/js/key.mjs
View file

@ -1,240 +0,0 @@
import { h, Component } from 'preact'
import htm from 'htm'
import Crypto from 'crypto'
const html = htm.bind(h)
export class Keys extends Component {
constructor(props) {//{{{
super(props)
this.state = {
create: false,
}
props.nodeui.retrieveKeys()
}//}}}
render({ nodeui }, { create }) {//{{{
let keys = nodeui.keys.value
.sort((a,b)=>{
if(a.description < b.description) return -1
if(a.description > b.description) return 1
return 0
})
.map(key=>
html`<${KeyComponent} key=${`key-${key.ID}`} model=${key} />`
)
let createButton = ''
let createComponents = ''
if(create) {
createComponents = html`
<div id="key-create">
<h2>New key</h2>
<div class="fields">
<input type="text" id="key-description" placeholder="Name" />
<input type="password" id="key-pass1" placeholder="Password" />
<input type="password" id="key-pass2" placeholder="Repeat password" />
<textarea id="key-key" placeholder="Key"></textarea>
<div>
<button class="generate" onclick=${()=>this.generateKey()}>Generate</button>
<button class="create" onclick=${()=>this.createKey()}>Create</button>
</div>
</div>
</div>
`
} else {
createButton = html`<div style="margin-top: 16px;"><button onclick=${()=>this.setState({ create: true })}>Create new key</button></div>`
}
return html`
<div id="keys">
<h1>Encryption keys</h1>
<p>
Unlock a key by clicking its name. Lock it by clicking it again.
</p>
<p>
Copy the key and store it in a very secure place to have a way to access notes
in case the password is forgotten, or database is corrupted.
</p>
<p>Click "View key" after unlocking it.</p>
${createButton}
${createComponents}
<h2>Keys</h2>
<div class="key-list">
${keys}
</div>
</div>`
}//}}}
generateKey() {//{{{
let keyTextarea = document.getElementById('key-key')
let key = sjcl.codec.hex.fromBits(Crypto.generate_key()).replace(/(....)/g, '$1 ').trim()
keyTextarea.value = key
}//}}}
validateNewKey() {//{{{
let keyDescription = document.getElementById('key-description').value
let keyTextarea = document.getElementById('key-key').value
let pass1 = document.getElementById('key-pass1').value
let pass2 = document.getElementById('key-pass2').value
if(keyDescription.trim() == '')
throw new Error('The key has to have a description')
if(pass1.trim() == '' || pass1.length < 4)
throw new Error('The password has to be at least 4 characters long.')
if(pass1 != pass2)
throw new Error(`Passwords doesn't match`)
let cleanKey = keyTextarea.replace(/\s+/g, '')
if(!cleanKey.match(/^[0-9a-f]{64}$/i))
throw new Error('Invalid key - has to be 64 characters of 0-9 and A-F')
}//}}}
createKey() {//{{{
try {
this.validateNewKey()
let description = document.getElementById('key-description').value
let keyAscii = document.getElementById('key-key').value
let pass1 = document.getElementById('key-pass1').value
// Key in hex taken from user.
let actual_key = sjcl.codec.hex.toBits(keyAscii.replace(/\s+/g, ''))
// Key generated from password, used to encrypt the actual key.
let pass_gen = Crypto.pass_to_key(pass1)
let crypto = new Crypto(pass_gen.key)
let encrypted_actual_key = crypto.encrypt(actual_key, 0x1n, false)
// Database value is salt + actual key, needed to generate the same key from the password.
let db_encoded = sjcl.codec.hex.fromBits(
pass_gen.salt.concat(encrypted_actual_key)
)
// Create on server.
window._app.current.request('/key/create', {
description,
key: db_encoded,
})
.then(res=>{
let key = new Key(res.Key, this.props.nodeui.keyCounter)
this.props.nodeui.keys.value = this.props.nodeui.keys.value.concat(key)
})
.catch(window._app.current.responseError)
} catch(err) {
alert(err.message)
return
}
}//}}}
}
export class Key {
constructor(data, counter_callback) {//{{{
this.ID = data.ID
this.description = data.Description
this.encryptedKey = data.Key
this.key = null
this._counter_cbk = counter_callback
let hex_key = window.sessionStorage.getItem(`key-${this.ID}`)
if(hex_key)
this.key = sjcl.codec.hex.toBits(hex_key)
}//}}}
status() {//{{{
if(this.key === null)
return 'locked'
return 'unlocked'
}//}}}
lock() {//{{{
this.key = null
window.sessionStorage.removeItem(`key-${this.ID}`)
}//}}}
unlock(password) {//{{{
let db = sjcl.codec.hex.toBits(this.encryptedKey)
let salt = db.slice(0, 4)
let pass_key = Crypto.pass_to_key(password, salt)
let crypto = new Crypto(pass_key.key)
this.key = crypto.decrypt(sjcl.codec.base64.fromBits(db.slice(4)))
window.sessionStorage.setItem(`key-${this.ID}`, sjcl.codec.hex.fromBits(this.key))
}//}}}
async counter() {//{{{
return this._counter_cbk()
}//}}}
}
export class KeyComponent extends Component {
constructor({ model }) {//{{{
super({ model })
this.state = {
show_key: false,
}
}//}}}
render({ model }, { show_key }) {//{{{
let status = ''
switch(model.status()) {
case 'locked':
status = html`<div class="status locked"><img src="/images/${window._VERSION}/padlock-closed.svg" /></div>`
break
case 'unlocked':
status = html`<div class="status unlocked"><img src="/images/${window._VERSION}/padlock-open.svg" /></div>`
break
}
let hex_key = ''
if(show_key) {
if(model.status() == 'locked')
hex_key = html`<div class="hex-key">Unlock key first</div>`
else {
let key = sjcl.codec.hex.fromBits(model.key)
key = key.replace(/(....)/g, "$1 ").trim()
hex_key = html`<div class="hex-key">${key}</div>`
}
}
let unlocked = model.status()=='unlocked'
return html`
<div class="status" onclick=${()=>this.toggle()}>${status}</div>
<div class="description" onclick=${()=>this.toggle()}>${model.description}</div>
<div class="view" onclick=${()=>this.toggleViewKey()}>${unlocked ? 'View key' : ''}</div>
${hex_key}
`
}//}}}
toggle() {//{{{
if(this.props.model.status() == 'locked')
this.unlock()
else
this.lock()
}//}}}
lock() {//{{{
this.props.model.lock()
this.forceUpdate()
}//}}}
unlock() {//{{{
let pass = prompt("Password")
if(!pass)
return
try {
this.props.model.unlock(pass)
this.forceUpdate()
} catch(err) {
alert(err)
}
}//}}}
toggleViewKey() {//{{{
this.setState({ show_key: !this.state.show_key })
}//}}}
}
// vim: foldmethod=marker

2574
static/js/lib/sjcl.js
View file

File diff suppressed because it is too large Load diff

364
static/js/node_store.mjs
View file

@ -1,12 +1,218 @@
import { Node } from 'node'
export const ROOT_NODE = '00000000-0000-0000-0000-000000000000'
export const ORPHANED_NODE = '00000000-0000-0000-0000-000000000001'
export const DELETED_NODE = '00000000-0000-0000-0000-000000000002'
export class Node {
static sort(a, b) {//{{{
// Nodes with children ("folders") are sorted first.
if (a._has_children && !b._has_children) return -1
if (!a._has_children && b._has_children) return 1
// Otherwise sort by lowercased name.
const an = a.data.Name.toLowerCase()
const bn = b.data.Name.toLowerCase()
if (an < bn) return -1
if (an > bn) return 1
return 0
}//}}}
static create(name, parentUUID) {// {{{
const node = new Node({
UUID: uuidv7(),
Created: (new Date()).toISOString(),
Content: '',
Name: name,
ParentUUID: parentUUID,
Markdown: false,
})
// Newly created node (not constructed from existing data) is considered modified
// since node.save returns early if it isn't modified.
node._modified = true
return node
}// }}}
constructor(nodeData, level) {//{{{
this.Level = level
this.data = nodeData
this.UUID = nodeData.UUID
// Toplevel nodes are normalized to have the ROOT_NODE as parent.
if (nodeData.UUID !== ROOT_NODE && nodeData.ParentUUID === '') {
this.ParentUUID = ROOT_NODE
this.data.ParentUUID = ROOT_NODE
} else
this.ParentUUID = nodeData.ParentUUID
this._children_fetched = false
this._has_children = null // this will be set by nodeStore.getTreeNodes
this.Children = []
this.Ancestors = []
this._sibling_before = null
this._sibling_after = null
this._parent = null
this.reset()
}//}}}
reset() {// {{{
// this._content is the runtime content in the editor.
// this.data.Content is the original data.
this._content = this.data.Content
this._modified = false
}// }}}
get(prop) {//{{{
return this.data[prop]
}//}}}
updated() {//{{{
// '2024-12-17T17:33:48.85939Z
return new Date(Date.parse(this.data.Updated))
}//}}}
isModified() {// {{{
return this._modified
}// }}}
hasFetchedChildren() {//{{{
return this._children_fetched
}//}}}
async fetchChildren() {//{{{
this.Children = await globalThis.nodeStore.getTreeNodes(this.UUID, this.Level + 1)
this._children_fetched = true
// Children are sorted to allow for storing siblings befare and after.
// These are used with keyboard navigation in the tree.
this.Children.sort(Node.sort)
const numChildren = this.Children.length
this.setHasChildren(numChildren > 0)
for (let i = 0; i < numChildren; i++) {
if (i > 0)
this.Children[i]._sibling_before = this.Children[i - 1]
if (i < numChildren - 1)
this.Children[i]._sibling_after = this.Children[i + 1]
this.Children[i]._parent = this
}
return this.Children
}//}}}
setHasChildren(v) {// {{{
this._has_children = v
}// }}}
hasChildren() {//{{{
return this._has_children
}//}}}
getSiblingBefore() {// {{{
return this._sibling_before
}// }}}
getSiblingAfter() {// {{{
return this._sibling_after
}// }}}
getParent() {//{{{
return this._parent
}//}}}
moveToParent(newParentUUID) {// {{{
if (this.UUID === newParentUUID)
throw new Error("New parent UUID is the same as node UUID. Can't be your own parent.")
this.ParentUUID = newParentUUID
this.data.ParentUUID = newParentUUID
this._modified = true
}// }}}
isLastSibling() {//{{{
return this._sibling_after === null
}//}}}
isFirstSibling() {//{{{
return this._sibling_before === null
}//}}}
isSpecial() {// {{{
return this.data.Special
}// }}}
content() {//{{{
// TODO - implement crypto
return this._content
}//}}}
setContent(new_content) {//{{{
this._content = new_content
this._modified = true
_mbus.dispatch('NODE_MODIFIED', { node: this })
}//}}}
setName(new_name) {// {{{
if (new_name.trim() === '')
throw new Error(`The name can't be empty`)
this.data.Name = new_name
this._modified = true
_mbus.dispatch('NODE_MODIFIED', { node: this })
}// }}}
async save() {//{{{
try {
const dblink = /db:\/\/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})/gi
// Just safeguarding not using the root node,
// which sort of exist but isn't supposed to communicate to server.
if (this.UUID == ROOT_NODE)
return
// File links from the original, unmodified content are found to compare to links from the new content.
// All files (images) needs to be indexed to nodes in order to track orphaned files.
const before = new Set([...this.data.Content.matchAll(dblink)].map(m => m[1]))
const after = new Set([...this._content.matchAll(dblink)].map(m => m[1]))
const removed = before.difference(after)
const added = after.difference(before)
await globalThis.nodeStore.filesNodelink.removeFileUUIDs(this.UUID, removed)
await globalThis.nodeStore.filesNodelink.addFileUUIDs(this.UUID, added)
// Authoritative content and metadata is set to prepare it for IndexedDB update.
this.data.Content = this._content
this.data.Updated = new Date().toISOString()
this.data.HistoryUUID = uuidv7() // every time the node is saved a new history UUID identifies the changed node.
_mbus.dispatch('NODE_UNMODIFIED')
this._modified = false
// When stored into database and ancestry was changed,
// the ancestry path could be interesting.
/*
const ancestors = await nodeStore.getNodeAncestry(this)
this.data.Ancestors = ancestors.map(a => a.get('Name')).reverse()
*/
/* The node history is a local store for node history.
* This could be provisioned from the server or cleared if
* deemed unnecessary.
*
* The send queue is what will be sent back to the server
* to have a recorded history of the notes.
*
* A setting to be implemented in the future could be to
* not save the history locally at all. */
// Current node is added to history. It will be duplicated with the "nodes" store
// for simplicity, to hopefully avoid bugs.
const history = globalThis.nodeStore.nodesHistory.add(this)
// Updated node is added to the send queue to be stored on server.
const sendQueue = globalThis.nodeStore.sendQueue.add(this)
// Updated node is saved to the primary node store.
const nodeStoreAdding = globalThis.nodeStore.add([this])
await Promise.all([history, sendQueue, nodeStoreAdding])
} catch (err) {
console.error(err)
alert(err.message)
}
return
}//}}}
}
export class NodeStore {
constructor() {//{{{
if (!('indexedDB' in window)) {
if (!('indexedDB' in self)) {
throw 'Missing IndexedDB'
}
@ -21,7 +227,7 @@ export class NodeStore {
}//}}}
initializeDB() {//{{{
return new Promise((resolve, reject) => {
const req = indexedDB.open('notes', 12)
const req = indexedDB.open('notes', 14)
// Schema upgrades for IndexedDB.
// These can start from different points depending on updates to Notes2 since a device was online.
@ -31,7 +237,6 @@ export class NodeStore {
let sendQueue
let nodesHistory
let files
let filesNodelink
const db = event.target.result
const trx = event.target.transaction
@ -89,6 +294,14 @@ export class NodeStore {
case 12:
trx.objectStore('files_nodelink').createIndex('byNodeCount', 'NodeCount', { unique: false })
break
case 13:
db.createObjectStore('file_segments', { keyPath: ['UUID', 'Sequence'] })
break
case 14:
db.deleteObjectStore('files')
break
}
}
}
@ -98,6 +311,7 @@ export class NodeStore {
this.sendQueue = new SimpleNodeStore(this.db, 'send_queue')
this.nodesHistory = new NodeHistoryStore(this.db, 'nodes_history')
this.files = new SimpleNodeStore(this.db, 'files')
this.fileSegments = new FileSegmentsStore(this.db, 'file_segments')
this.filesMetadata = new FilesMetadataStore(this.db, 'files_metadata')
this.filesNodelink = new FileNodesLinkStore(this.db, 'files_nodelink')
resolve()
@ -353,9 +567,38 @@ export class NodeStore {
}//}}}
async storeFile(file) {// {{{
const metadata = new FileMetadata(file, {})
const metadata = new FileMetadata(file)
try {
// Do potentially larger blocks.
const BLOCKSIZE = 1048576
const stream = file.stream()
const reader = stream.getReader({ mode: 'byob' })
let buffer = new Uint8Array(BLOCKSIZE)
let seq = 0
while (true) {
const { done, value } = await reader.read(buffer)
if (done) break
// Block is added as a file segment.
await this.fileSegments.add({
data: {
UUID: metadata.UUID,
Sequence: seq,
Data: value,
}
})
buffer = new Uint8Array(value.buffer)
seq++
}
reader.releaseLock()
} catch (e) {
console.error(e)
alert(e.message)
}
await this.files.add({ data: { UUID: metadata.UUID, file } })
await this.filesMetadata.add({ data: metadata })
return metadata
@ -374,30 +617,11 @@ export class NodeStore {
// No more files to sync.
if (data === undefined) {
resolve({ file: null, metadata: null })
resolve(null)
return
}
// Retrieve file as well.
const filereq = this.db.
transaction('files', 'readonly')
.objectStore('files')
.get(data.UUID)
filereq.onsuccess = (e) => {
const f = e.target.result
// File should always be in IndexedDB when metadata is.
if (f === undefined) {
reject(new Error(`File with UUID '${data.UUID}' is missing.`))
return
}
resolve({
file: f.file,
metadata: new FileMetadata(f.file, data),
})
}
resolve(new FileMetadata(data))
}
})
}// }}}
@ -528,7 +752,7 @@ class NodeHistoryStore extends SimpleNodeStore {
}
req.onerror = (event) => {
console.log(event.target.error)
console.error(event.target.error)
reject(event.target.error)
}
})
@ -581,6 +805,37 @@ class NodeHistoryStore extends SimpleNodeStore {
}// }}}
}
class FileSegmentsStore extends SimpleNodeStore {
constructor(db, storeName) {//{{{
super(db, storeName)
}//}}}
async getStream(uuid) {// {{{
let seq = 0
const meself = this
return new ReadableStream({
type: 'bytes',
async pull(controller) {
try {
while (true) {
const chunk = await meself.get([uuid, seq])
if (!chunk) {
controller.close()
return
}
controller.enqueue(chunk.Data)
seq++
}
} catch (e) {
console.error(e)
}
}
})
}// }}}
}
class FilesMetadataStore extends SimpleNodeStore {
countToUpload() {// {{{
const store = this.db
@ -638,6 +893,32 @@ class FileNodesLinkStore extends SimpleNodeStore {
}// }}}
}
class FileMetadata {
constructor(data) {// {{{
console.log('md', data)
this.UUID = data.UUID || uuidv7()
this.uploaded = data.uploaded || 'only_on_device'
this.name = data.name
this.size = data.size
this.type = data.type || 'application/octet-stream'
this.modified = data.modified || data.lastModified || Date.now()
}// }}}
setUploaded() {// {{{
this.uploaded = 'on_server'
}// }}}
get data() {// {{{
return {
UUID: this.UUID,
uploaded: this.uploaded,
name: this.name,
size: this.size,
type: this.type,
modified: this.modified,
}
}// }}}
}
export function uuidv7() {// {{{
// random bytes
const value = new Uint8Array(16)
@ -664,29 +945,4 @@ export function uuidv7() {// {{{
return `${str.slice(0, 8)}-${str.slice(8, 12)}-${str.slice(12, 16)}-${str.slice(16, 20)}-${str.slice(20)}`
}// }}}
class FileMetadata {
constructor(file, data) {// {{{
this.UUID = data.UUID || uuidv7()
this.uploaded = data.uploaded || 'only_on_device'
this.name = file.name
this.size = file.size
this.type = file.type || 'application/octet-stream'
this.modified = file.lastModified
}// }}}
setUploaded() {// {{{
this.uploaded = 'on_server'
}// }}}
get data() {// {{{
return {
UUID: this.UUID,
uploaded: this.uploaded,
name: this.name,
size: this.size,
type: this.type,
modified: this.modified,
}
}// }}}
}
// vim: foldmethod=marker

2
static/js/page_history.mjs
View file

@ -1,5 +1,5 @@
import { CustomHTMLElement } from './lib/custom_html_element.mjs'
import { Node } from './page_node.mjs'
import { Node } from 'node_store'
import { MarkedPosition } from './marked_position.mjs'

210
static/js/page_node.mjs
View file

@ -1,4 +1,3 @@
import { ROOT_NODE, uuidv7 } from 'node_store'
import { CustomHTMLElement } from './lib/custom_html_element.mjs'
import { MarkedPosition } from './marked_position.mjs'
@ -398,213 +397,4 @@ export class N2PageNodeUI extends CustomHTMLElement {
}
customElements.define('n2-nodeui', N2PageNodeUI)
export class Node {
static sort(a, b) {//{{{
// Nodes with children ("folders") are sorted first.
if (a._has_children && !b._has_children) return -1
if (!a._has_children && b._has_children) return 1
// Otherwise sort by lowercased name.
const an = a.data.Name.toLowerCase()
const bn = b.data.Name.toLowerCase()
if (an < bn) return -1
if (an > bn) return 1
return 0
}//}}}
static create(name, parentUUID) {// {{{
const node = new Node({
UUID: uuidv7(),
Created: (new Date()).toISOString(),
Content: '',
Name: name,
ParentUUID: parentUUID,
Markdown: false,
})
// Newly created node (not constructed from existing data) is considered modified
// since node.save returns early if it isn't modified.
node._modified = true
return node
}// }}}
constructor(nodeData, level) {//{{{
this.Level = level
this.data = nodeData
this.UUID = nodeData.UUID
// Toplevel nodes are normalized to have the ROOT_NODE as parent.
if (nodeData.UUID !== ROOT_NODE && nodeData.ParentUUID === '') {
this.ParentUUID = ROOT_NODE
this.data.ParentUUID = ROOT_NODE
} else
this.ParentUUID = nodeData.ParentUUID
this._children_fetched = false
this._has_children = null // this will be set by nodeStore.getTreeNodes
this.Children = []
this.Ancestors = []
this._sibling_before = null
this._sibling_after = null
this._parent = null
this.reset()
}//}}}
reset() {// {{{
// this._content is the runtime content in the editor.
// this.data.Content is the original data.
this._content = this.data.Content
this._modified = false
}// }}}
get(prop) {//{{{
return this.data[prop]
}//}}}
updated() {//{{{
// '2024-12-17T17:33:48.85939Z
return new Date(Date.parse(this.data.Updated))
}//}}}
isModified() {// {{{
return this._modified
}// }}}
hasFetchedChildren() {//{{{
return this._children_fetched
}//}}}
async fetchChildren() {//{{{
this.Children = await nodeStore.getTreeNodes(this.UUID, this.Level + 1)
this._children_fetched = true
// Children are sorted to allow for storing siblings befare and after.
// These are used with keyboard navigation in the tree.
this.Children.sort(Node.sort)
const numChildren = this.Children.length
this.setHasChildren(numChildren > 0)
for (let i = 0; i < numChildren; i++) {
if (i > 0)
this.Children[i]._sibling_before = this.Children[i - 1]
if (i < numChildren - 1)
this.Children[i]._sibling_after = this.Children[i + 1]
this.Children[i]._parent = this
}
return this.Children
}//}}}
setHasChildren(v) {// {{{
this._has_children = v
}// }}}
hasChildren() {//{{{
return this._has_children
}//}}}
getSiblingBefore() {// {{{
return this._sibling_before
}// }}}
getSiblingAfter() {// {{{
return this._sibling_after
}// }}}
getParent() {//{{{
return this._parent
}//}}}
moveToParent(newParentUUID) {// {{{
if (this.UUID === newParentUUID)
throw new Error("New parent UUID is the same as node UUID. Can't be your own parent.")
this.ParentUUID = newParentUUID
this.data.ParentUUID = newParentUUID
this._modified = true
}// }}}
isLastSibling() {//{{{
return this._sibling_after === null
}//}}}
isFirstSibling() {//{{{
return this._sibling_before === null
}//}}}
isSpecial() {// {{{
return this.data.Special
}// }}}
content() {//{{{
// TODO - implement crypto
return this._content
}//}}}
setContent(new_content) {//{{{
this._content = new_content
this._modified = true
_mbus.dispatch('NODE_MODIFIED', { node: this })
}//}}}
setName(new_name) {// {{{
if (new_name.trim() === '')
throw new Error(`The name can't be empty`)
this.data.Name = new_name
this._modified = true
_mbus.dispatch('NODE_MODIFIED', { node: this })
}// }}}
async save() {//{{{
try {
const dblink = /db:\/\/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})/gi
// Just safeguarding not using the root node,
// which sort of exist but isn't supposed to communicate to server.
if (this.UUID == ROOT_NODE)
return
// File links from the original, unmodified content are found to compare to links from the new content.
// All files (images) needs to be indexed to nodes in order to track orphaned files.
const before = new Set([...this.data.Content.matchAll(dblink)].map(m => m[1]))
const after = new Set([...this._content.matchAll(dblink)].map(m => m[1]))
const removed = before.difference(after)
const added = after.difference(before)
await nodeStore.filesNodelink.removeFileUUIDs(this.UUID, removed)
await nodeStore.filesNodelink.addFileUUIDs(this.UUID, added)
// Authoritative content and metadata is set to prepare it for IndexedDB update.
this.data.Content = this._content
this.data.Updated = new Date().toISOString()
this.data.HistoryUUID = uuidv7() // every time the node is saved a new history UUID identifies the changed node.
_mbus.dispatch('NODE_UNMODIFIED')
this._modified = false
// When stored into database and ancestry was changed,
// the ancestry path could be interesting.
/*
const ancestors = await nodeStore.getNodeAncestry(this)
this.data.Ancestors = ancestors.map(a => a.get('Name')).reverse()
*/
/* The node history is a local store for node history.
* This could be provisioned from the server or cleared if
* deemed unnecessary.
*
* The send queue is what will be sent back to the server
* to have a recorded history of the notes.
*
* A setting to be implemented in the future could be to
* not save the history locally at all. */
// Current node is added to history. It will be duplicated with the "nodes" store
// for simplicity, to hopefully avoid bugs.
const history = nodeStore.nodesHistory.add(this)
// Updated node is added to the send queue to be stored on server.
const sendQueue = nodeStore.sendQueue.add(this)
// Updated node is saved to the primary node store.
const nodeStoreAdding = nodeStore.add([this])
await Promise.all([history, sendQueue, nodeStoreAdding])
} catch (err) {
console.error(err)
alert(err.message)
}
return
}//}}}
}
// vim: foldmethod=marker

3
static/js/sidebar.mjs
View file

@ -1,5 +1,4 @@
import { ROOT_NODE, ORPHANED_NODE, DELETED_NODE } from 'node_store'
import { Node } from 'node'
import { ROOT_NODE, ORPHANED_NODE, DELETED_NODE, Node } from 'node_store'
import { CustomHTMLElement } from './lib/custom_html_element.mjs'
import { Color, Solver } from './lib/css_colorize.mjs'

27
static/js/sync.mjs
View file

@ -1,5 +1,5 @@
import { API } from 'api'
import { Node } from 'node'
import { Node } from 'node_store'
import { CustomHTMLElement } from './lib/custom_html_element.mjs'
export class Sync {
@ -169,18 +169,18 @@ export class Sync {
const BLOCKSIZE = 128 * 1024
let sentBytes = 0
const { file, metadata } = await nodeStore.nextFileToSync()
if (file === null)
const metadata = await nodeStore.nextFileToSync()
if (metadata === null)
return
const stream = file.stream()
const stream = await globalThis.nodeStore.fileSegments.getStream(metadata.UUID)
const reader = stream.getReader({ mode: 'byob' }) // Bring Your Own Buffer
let buffer = new Uint8Array(BLOCKSIZE)
while (true) {
const { value, done } = await reader.read(buffer)
await API.upload(metadata.UUID, value, sentBytes, file)
await API.upload(metadata.UUID, value, sentBytes, metadata)
metadata.setUploaded()
nodeStore.filesMetadata.add(metadata)
@ -189,6 +189,23 @@ export class Sync {
sentBytes += value.length
buffer = new Uint8Array(value.buffer)
}
reader.releaseLock()
} catch (e) {
console.error(e)
alert(e.message)
}
}// }}}
async filesFromServer() {// {{{
let offset = 0
let more = true
try {
while (more) {
const res = await API.query('GET', `/file/server_uuids/${offset}`)
more = res.MoreRowsExist
console.log(res)
}
} catch (e) {
console.error(e)
alert(e.message)

50
static/service_worker.js
View file

@ -1,3 +1,5 @@
import { NodeStore } from '/js/{{ .VERSION }}/node_store.mjs'
const CACHE_NAME = 'notes2-{{ .VERSION }}'
const CACHED_ASSETS = [
'/',
@ -33,14 +35,11 @@ const CACHED_ASSETS = [
'/js/{{ .VERSION }}/api.mjs',
'/js/{{ .VERSION }}/app.mjs',
'/js/{{ .VERSION }}/checklist.mjs',
'/js/{{ .VERSION }}/crypto.mjs',
'/js/{{ .VERSION }}/file.mjs',
'/js/{{ .VERSION }}/key.mjs',
'/js/{{ .VERSION }}/lib/css_colorize.mjs',
'/js/{{ .VERSION }}/lib/custom_html_element.mjs',
'/js/{{ .VERSION }}/lib/node_modules/marked/lib/marked.esm.js',
'/js/{{ .VERSION }}/lib/node_modules/marked-token-position/lib/index.esm.js',
'/js/{{ .VERSION }}/lib/sjcl.js',
'/js/{{ .VERSION }}/marked_position.mjs',
'/js/{{ .VERSION }}/mbus.mjs',
'/js/{{ .VERSION }}/node_store.mjs',
@ -126,14 +125,55 @@ self.addEventListener('activate', event => {
})
self.addEventListener('fetch', event => {
const url = new URL(event.request.url)
// The fetch event is also seeing requests to other domains.
// Just let the browser handle those for itself.
const ourDomain = event.request.url.startsWith(self.location.origin)
const ourDomain = (url.host == self.location.host)
if (!ourDomain)
return event
if (url.pathname == '/stream-local') {
event.respondWith(indexeddbFile(event))
return
}
if (`{{ .DevMode }}` == 'true')
return event
else
event.respondWith(fetchAsset(event))
})
let nodestore = null
async function getNodestore() {
if (nodestore)
return nodestore
// When setting nodestore directly, then waiting on initializeDB,
// a second call for getNodestore can return the nodestore before
// initialized.
const initializingNodeStore = new NodeStore()
await initializingNodeStore.initializeDB()
nodestore = initializingNodeStore
return nodestore
}
async function indexeddbFile(event) {
const url = new URL(event.request.url)
const uuid = url.searchParams.get('uuid')
try {
const nodeStore = await getNodestore()
const md = await nodeStore.filesMetadata.get(uuid)
const stream = await nodeStore.fileSegments.getStream(uuid)
return new Response(stream, {
headers: {
'Content-Type': md.type,
}
})
} catch (err) {
console.error(err)
}
}

8
views/layouts/main.gotmpl
View file

@ -10,9 +10,7 @@
"imports": {
"api": "/js/{{ .VERSION }}/api.mjs",
"sync": "/js/{{ .VERSION }}/sync.mjs",
"key": "/js/{{ .VERSION }}/key.mjs",
"checklist": "/js/{{ .VERSION }}/checklist.mjs",
"crypto": "/js/{{ .VERSION }}/crypto.mjs",
"node_store": "/js/{{ .VERSION }}/node_store.mjs",
"node": "/js/{{ .VERSION }}/page_node.mjs",
"sidebar": "/js/{{ .VERSION }}/sidebar.mjs"
@ -20,14 +18,14 @@
}
</script>
<script>
window._VERSION = "{{ .VERSION }}"
globalThis._VERSION = "{{ .VERSION }}"
if (navigator.serviceWorker)
navigator.serviceWorker.register('/service_worker.js')
navigator.serviceWorker.register('/service_worker.js', { type: 'module' })
</script>
<script type="module" defer>
import { MessageBus } from '/js/{{ .VERSION }}/mbus.mjs'
window._mbus = new MessageBus()
globalThis._mbus = new MessageBus()
</script>
</head>
<body>

4
views/pages/login.gotmpl
View file

@ -4,7 +4,7 @@
<img id="logo" src="/images/v1/logo.svg">
<input type="text" id="username" placeholder="Username">
<input type="password" id="password" placeholder="Password">
<button onclick=window._login.authenticate()>Log in</button>
<button onclick="globalThis._login.authenticate()">Log in</button>
<div id="error"></div>
</div>
@ -42,6 +42,6 @@ class Login {
}
}
window._login = new Login()
globalThis._login = new Login()
</script>
{{ end }}

11
views/pages/notes2.gotmpl
View file

@ -7,7 +7,7 @@
<!-- page-node -->
<div id="notes2" class="page-node">
<div id="tree-expander" onclick="window._mbus.dispatch('TREE_EXPANSION', { expand: true })">&gt;</div>
<div id="tree-expander" onclick="globalThis._mbus.dispatch('TREE_EXPANSION', { expand: true })">&gt;</div>
<div id="tree" tabindex=0></div>
<div id="main-page">
@ -47,20 +47,21 @@
import {App} from "/js/{{ .VERSION }}/app.mjs"
import {API} from 'api'
import {Sync} from 'sync'
import { } from '/js/{{ .VERSION }}/page_node.mjs'
import { } from '/js/{{ .VERSION }}/page_preferences.mjs'
import { } from '/js/{{ .VERSION }}/page_storage.mjs'
import { } from '/js/{{ .VERSION }}/page_history.mjs'
import { } from '/js/{{ .VERSION }}/file.mjs'
window.Sync = Sync
globalThis.Sync = Sync
if (!API.hasAuthenticationToken()) {
location.href = '/login'
} else {
try {
window.nodeStore = new NodeStore()
window.nodeStore.initializeDB().then(() => {
window._app = new App()
globalThis.nodeStore = new NodeStore()
globalThis.nodeStore.initializeDB().then(() => {
globalThis._app = new App()
})
} catch (e) {
alert(e)

6
views/pages/sync.gotmpl
View file

@ -5,9 +5,9 @@
<script type="module">
import { NodeStore } from 'node_store'
import { Sync } from 'sync'
window.Sync = Sync
window.nodeStore = new NodeStore()
window.nodeStore.initializeDB().then(()=>{
globalThis.Sync = Sync
globalThis.nodeStore = new NodeStore()
globalThis.nodeStore.initializeDB().then(()=>{
Sync.tree()
})
</script>