diff --git a/config.go b/config.go index 71d5f5d..8d89255 100644 --- a/config.go +++ b/config.go @@ -24,6 +24,10 @@ type Config struct { Secret string ExpireDays int } + + Storage struct { + Files string + } } func readConfig(fname string) (err error) { diff --git a/file.go b/file.go index 9ad8457..6dd1c75 100644 --- a/file.go +++ b/file.go @@ -5,35 +5,47 @@ import ( "github.com/jmoiron/sqlx" // Standard + "fmt" + "os" + "path" "time" ) type File struct { - ID int - UserID int `db:"user_id"` - NodeID int `db:"node_id"` - Filename string - Size int64 - MIME string - MD5 string - Uploaded time.Time + ID int + UUID string + UserID int `db:"user_id"` + Name string `db:"name"` + Size int + Type string `db:"type"` + Modified time.Time + UploadComplete bool `db:"upload_complete"` } -func AddFile(userID int, file *File) (err error) { // {{{ +var ( + fileUUIDCache map[string]int +) + +func init() { + fileUUIDCache = make(map[string]int) +} + +func AddFileToDb(userID int, file *File) (err error) { // {{{ file.UserID = userID var rows *sqlx.Rows rows, err = db.Queryx(` - INSERT INTO file(user_id, node_id, filename, size, mime, md5) - VALUES($1, $2, $3, $4, $5, $6) + INSERT INTO public.file(user_id, uuid, name, size, type, modified, upload_complete) + VALUES($1, $2, $3, $4, $5, $6, false) + ON CONFLICT (user_id, uuid) DO UPDATE set name = public.file.name RETURNING id `, file.UserID, - file.NodeID, - file.Filename, + file.UUID, + file.Name, file.Size, - file.MIME, - file.MD5, + file.Type, + file.Modified, ) if err != nil { return @@ -76,5 +88,101 @@ func Files(userID int, nodeUUID string, fileID int) (files []File, err error) { return } // }}} +func FileIDToPath(id int) (string, string) { // {{{ + str := fmt.Sprintf("%012d", id) + dir := path.Join(config.Storage.Files, str[0:4], str[4:8]) + fpath := path.Join(config.Storage.Files, str[0:4], str[4:8], str[8:12]) + return dir, fpath +} // }}} + +func FileWriteData(userID int, dbID int, data []byte, create bool) (err error) { // {{{ + dir, fpath := FileIDToPath(dbID) + err = os.MkdirAll(dir, 0700) + if err != nil { + Log.Error("file", "error", err) + return + } + + var f *os.File + if create { + f, err = os.OpenFile(fpath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) + } else { + f, err = os.OpenFile(fpath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) + } + if err != nil { + Log.Error("file", "error", err) + return + } + defer f.Close() + + _, err = f.Write(data) + if err != nil { + Log.Error("file", "error", err) + return + } + + return +} // }}} + +func FileCountForUser(userID int) (count int, err error) { // {{{ + row := db.QueryRowx(`SELECT COUNT(*) FROM public.file WHERE user_id = $1`, userID) + err = row.Scan(&count) + return +} // }}} +func FileServerUUIDsForUser(userID, page int) (uuids []string, more bool, err error) { // {{{ + uuids = []string{} + + var rows *sqlx.Rows + rows, err = db.Queryx(` + SELECT + uuid + FROM public.file + WHERE + user_id = $1 + ORDER BY + uuid ASC + LIMIT $2 OFFSET $3 + `, + userID, + SYNC_PAGINATION+1, + page*SYNC_PAGINATION, + ) + if err != nil { + return + } + defer rows.Close() + + var uuid string + for rows.Next() { + err = rows.Scan(&uuid) + if err != nil { + return + } + uuids = append(uuids, uuid) + } + + more = len(uuids) > SYNC_PAGINATION + uuids = uuids[0:min(SYNC_PAGINATION, len(uuids))] + + return +} // }}} +func FileMetadata(userID int, uuid string) (f File, err error) { // {{{ + row := db.QueryRowx(`SELECT * FROM public.file WHERE user_id = $1 AND "uuid" = $2`, userID, uuid) + err = row.StructScan(&f) + return + +} // }}} +func FileForUser(userID int, uuid string) (f *os.File, md File, err error) { // {{{ + row := db.QueryRowx(`SELECT * FROM public.file WHERE user_id = $1 AND "uuid" = $2`, userID, uuid) + err = row.StructScan(&md) + if err != nil { + return + } + + _, fpath := FileIDToPath(md.ID) + f, err = os.OpenFile(fpath, os.O_RDONLY, 0600) + return + +} // }}} // vim: foldmethod=marker diff --git a/main.go b/main.go index 6e3cf94..ff935ad 100644 --- a/main.go +++ b/main.go @@ -5,7 +5,6 @@ import ( "notes2/authentication" "notes2/html_template" appUser "notes2/user" - "os" // Standard "bufio" @@ -17,14 +16,16 @@ import ( "io" "log/slog" "net/http" + "os" "path" "regexp" "strconv" "strings" "text/template" + "time" ) -const VERSION = "v29" +const VERSION = "v30" const CONTEXT_USER = 1 const SYNC_PAGINATION = 200 @@ -71,7 +72,7 @@ func init() { // {{{ } // }}} func initLog() { // {{{ opts := slog.HandlerOptions{} - opts.Level = slog.LevelDebug + opts.Level = slog.LevelInfo Log = slog.New(slog.NewJSONHandler(os.Stdout, &opts)) } // }}} func main() { // {{{ @@ -141,11 +142,17 @@ func main() { // {{{ http.HandleFunc("/sync/from_server/count/{sequence}", authenticated(actionSyncFromServerCount)) http.HandleFunc("/sync/from_server/{sequence}/{offset}", authenticated(actionSyncFromServer)) http.HandleFunc("/sync/to_server", authenticated(actionSyncToServer)) + http.HandleFunc("/sync/file/upload/{uuid}", authenticated(actionSyncFileUpload)) http.HandleFunc("/node/retrieve/{uuid}", authenticated(actionNodeRetrieve)) http.HandleFunc("/node/history/retrieve/{uuid}/{offset}", authenticated(actionNodeHistoryRetrieve)) http.HandleFunc("/node/history/count/{uuid}", authenticated(actionNodeHistoryCount)) + http.HandleFunc("/file/count", authenticated(actionFileCount)) + http.HandleFunc("/file/server_uuids/{offset}", authenticated(actionFileServerUUIDs)) + http.HandleFunc("/file/uuid/{uuid}", authenticated(actionFile)) + http.HandleFunc("/file/uuid/{uuid}/metadata", authenticated(actionFileMetadata)) + http.HandleFunc("/service_worker.js", pageServiceWorker) listen := fmt.Sprintf("%s:%d", config.Network.Address, config.Network.Port) @@ -166,6 +173,11 @@ func authenticated(fn func(http.ResponseWriter, *http.Request)) func(http.Respon // The Bearer token is extracted. authHeader := r.Header.Get("Authorization") + + if authHeader == "" { + authHeader = "Bearer " + r.URL.Query().Get("jwt") + } + authParts := RxpBearerToken.FindStringSubmatch(authHeader) if len(authParts) != 2 { failed(fmt.Errorf("Authorization missing or invalid")) @@ -391,6 +403,63 @@ func actionSyncToServer(w http.ResponseWriter, r *http.Request) { // {{{ "OK": true, }) } // }}} +func actionSyncFileUpload(w http.ResponseWriter, r *http.Request) { // {{{ + uuid := r.PathValue("uuid") + session := getUserSession(r) + + sentBytes, _ := strconv.Atoi(r.Header.Get("X-File-Sent-Bytes")) + totalBytes, _ := strconv.Atoi(r.Header.Get("X-File-Total-Bytes")) + + // First block comes with file metadata. + file := File{} + if sentBytes == 0 { + file.UUID = uuid + file.Name = r.Header.Get("X-File-Name") + file.Size, _ = strconv.Atoi(r.Header.Get("X-File-Size")) + file.Type = r.Header.Get("X-File-Type") + modded, _ := strconv.ParseInt(r.Header.Get("X-File-Modified"), 10, 64) + file.Modified = time.UnixMilli(modded) + err := AddFileToDb(session.UserID, &file) + if err != nil { + Log.Error("upload", "op", "AddFileToDB", "error", err) + httpError(w, err) + return + } + fileUUIDCache[uuid] = file.ID + } + + data, _ := io.ReadAll(r.Body) + var err error + + switch sentBytes { + case 0: + // First block of data. + Log.Info("file_upload", "status", "new", "user", session.UserID, "uuid", uuid, "total", totalBytes) + id := fileUUIDCache[uuid] + err = FileWriteData(session.UserID, id, data, true) + + case totalBytes: + // Last block of data - sent with 0 length data. + Log.Info("file_upload", "status", "done", "user", session.UserID, "uuid", uuid, "total", totalBytes) + id := fileUUIDCache[uuid] + delete(fileUUIDCache, uuid) + _, err = db.Exec(`UPDATE public.file SET upload_complete = true WHERE id = $1`, id) + + default: + id := fileUUIDCache[uuid] + err = FileWriteData(session.UserID, id, data, false) + } + + if err != nil { + Log.Error("upload", "error", err) + httpError(w, err) + return + } + + responseData(w, map[string]any{ + "OK": true, + }) +} // }}} func actionUserGetPreferences(w http.ResponseWriter, r *http.Request) { // {{{ user := getUserSession(r) @@ -433,6 +502,65 @@ func actionUserSetPreferences(w http.ResponseWriter, r *http.Request) { // {{{ }) } // }}} +func actionFileCount(w http.ResponseWriter, r *http.Request) {// {{{ + session := getUserSession(r) + count, err := FileCountForUser(session.UserID) + if err != nil { + Log.Error("file_count", "error", err) + httpError(w, err) + return + } + + responseData(w, map[string]any{ + "OK": true, + "Count": count, + }) +}// }}} +func actionFileServerUUIDs(w http.ResponseWriter, r *http.Request) {// {{{ + session := getUserSession(r) + offset, _ := strconv.Atoi(r.PathValue("offset")) + uuids, more, err := FileServerUUIDsForUser(session.UserID, offset) + if err != nil { + Log.Error("file_server_uuids", "error", err) + httpError(w, err) + return + } + + responseData(w, map[string]any{ + "OK": true, + "UUIDs": uuids, + "MoreRowsExist": more, + }) +}// }}} +func actionFile(w http.ResponseWriter, r *http.Request) {// {{{ + uuid := r.PathValue("uuid") + session := getUserSession(r) + f, md, err := FileForUser(session.UserID, uuid) + if err != nil { + Log.Error("file_metadata", "error", err) + httpError(w, err) + return + } + + http.ServeContent(w, r, md.Name, md.Modified, f) +}// }}} +func actionFileMetadata(w http.ResponseWriter, r *http.Request) {// {{{ + uuid := r.PathValue("uuid") + session := getUserSession(r) + md, err := FileMetadata(session.UserID, uuid) + + if err != nil { + Log.Error("file_metadata", "error", err) + httpError(w, err) + return + } + + responseData(w, map[string]any{ + "OK": true, + "Metadata": md, + }) +}// }}} + func createNewUser(username string) { // {{{ reader := bufio.NewReader(os.Stdin) diff --git a/node.go b/node.go index a25c771..15f277b 100644 --- a/node.go +++ b/node.go @@ -57,67 +57,6 @@ type Node struct { Special bool } -func NodeTree(userID, offset int, synced uint64) (nodes []TreeNode, maxSeq uint64, moreRowsExist bool, err error) { // {{{ - const LIMIT = 100 - var rows *sqlx.Rows - rows, err = db.Queryx(` - SELECT - uuid, - COALESCE(parent_uuid, '') AS parent_uuid, - name, - created, - updated, - deleted IS NOT NULL AS deleted, - created_seq, - updated_seq, - deleted_seq - FROM - public.node - WHERE - user_id = $1 AND - ( - created_seq > $4 OR - updated_seq > $4 OR - deleted_seq > $4 - ) - ORDER BY - created ASC - LIMIT $2 OFFSET $3 - `, - userID, - LIMIT+1, - offset, - synced, - ) - if err != nil { - return - } - defer rows.Close() - - nodes = []TreeNode{} - numNodes := 0 - for rows.Next() { - // Query selects up to one more row than the decided limit. - // Saves one SQL query for row counting. - // Thus if numNodes is larger than the limit, more rows exist for the next call. - numNodes++ - if numNodes > LIMIT { - moreRowsExist = true - return - } - - node := TreeNode{} - if err = rows.StructScan(&node); err != nil { - return - } - nodes = append(nodes, node) - - // DeletedSeq will be 0 if invalid, and thus not be a problem for the max function. - maxSeq = max(maxSeq, node.CreatedSeq, node.UpdatedSeq, uint64(node.DeletedSeq.Int64)) - } - - return -} // }}} func Nodes(userID, offset int, synced uint64, clientUUID string) (nodes []Node, maxSeq uint64, moreRowsExist bool, err error) { // {{{ var rows *sqlx.Rows rows, err = db.Queryx(` diff --git a/sql/00011.sql b/sql/00011.sql new file mode 100644 index 0000000..f9efdbb --- /dev/null +++ b/sql/00011.sql @@ -0,0 +1,13 @@ +CREATE TABLE public.file ( + id serial NOT NULL, + user_id int4 NOT NULL, + uuid uuid NOT NULL, + name varchar DEFAULT '' NOT NULL, + "size" int DEFAULT 0 NOT NULL, + "type" varchar DEFAULT 'application/octet-stream' NOT NULL, + modified timestamptz DEFAULT NOW() NOT NULL, + CONSTRAINT file_pk PRIMARY KEY (id), + CONSTRAINT file_user_fk FOREIGN KEY (user_id) REFERENCES public."user"(id) ON DELETE RESTRICT ON UPDATE RESTRICT +); + +ALTER TABLE public.file ADD CONSTRAINT file_user_uuid_unique UNIQUE (user_id,"uuid"); diff --git a/sql/00012.sql b/sql/00012.sql new file mode 100644 index 0000000..422d2cf --- /dev/null +++ b/sql/00012.sql @@ -0,0 +1 @@ +ALTER TABLE public.file ADD upload_complete bool DEFAULT false NOT NULL; diff --git a/sql/00013.sql b/sql/00013.sql new file mode 100644 index 0000000..ef7e72c --- /dev/null +++ b/sql/00013.sql @@ -0,0 +1,171 @@ +-- file_node_links Keeps tracks of the files referenced by each node. +CREATE TABLE public.file_node_links ( + file_uuid uuid NOT NULL, + node_uuid uuid NOT NULL, + user_id int4 NOT NULL, + CONSTRAINT file_node_pk PRIMARY KEY (file_uuid,node_uuid,user_id), + CONSTRAINT file_node_user_fk FOREIGN KEY (user_id) REFERENCES public."user"(id) ON DELETE CASCADE ON UPDATE CASCADE, + CONSTRAINT file_node_node_fk FOREIGN KEY (user_id,node_uuid) REFERENCES public.node(user_id,"uuid") ON DELETE CASCADE ON UPDATE CASCADE, + CONSTRAINT file_node_file_fk FOREIGN KEY (user_id,file_uuid) REFERENCES public.file(user_id,"uuid") ON DELETE CASCADE ON UPDATE CASCADE +); + +-- add_nodes needs to call the new manage_node_file_links procedure. +CREATE OR REPLACE PROCEDURE public.add_nodes(IN p_user_id integer, IN p_client_uuid uuid, IN p_nodes jsonb) + LANGUAGE plpgsql +AS $procedure$ + +DECLARE + node_data jsonb; + node_updated timestamptz; + db_updated timestamptz; + db_uuid uuid; + db_client uuid; + db_history_uuid uuid; + node_uuid uuid; + node_parent_uuid uuid; + node_history_uuid uuid; + +BEGIN + FOR node_data IN SELECT * FROM jsonb_array_elements(p_nodes) + LOOP + node_uuid = (node_data->>'UUID')::uuid; + node_history_uuid = (node_data->>'HistoryUUID')::uuid; + node_updated = (node_data->>'Updated')::timestamptz; + + + + -- Frontend is using an all-zero UUID to define the root node. + -- Database is using NULL. + IF node_data->>'ParentUUID' = '00000000-0000-0000-0000-000000000000' OR node_data->>'ParentUUID' = '' THEN + node_parent_uuid = NULL; + ELSE + node_parent_uuid = (node_data->>'ParentUUID')::uuid; + END IF; + + -- Safeguard against being your own parent. + IF node_uuid = node_parent_uuid THEN + RAISE EXCEPTION 'Node UUID is same as node parent UUID.' USING ERRCODE = 'XPRNT'; + END IF; + + + -- Every jode has a new history UUID to keep the history entry uniquely identifiable + -- across clients. A history entry could potentially be sent again, but should be + -- safe to ignore as every change to a node should have a new history UUID. + -- + -- The current node is also stored as history. + INSERT INTO node_history( + user_id, "uuid", "history_uuid", parents, created, updated, + "name", "content", "content_encrypted", + client + ) + VALUES( + p_user_id, -- combined key + node_uuid, -- combined key + node_history_uuid, -- combined key + (jsonb_populate_record(null::json_ancestor_array, node_data))."Ancestors", + COALESCE((node_data->>'Created')::timestamptz, NOW()), + COALESCE((node_data->>'Updated')::timestamptz, NOW()), + (node_data->>'Name')::varchar, + (node_data->>'Content')::text, + '', /* content_encrypted */ + p_client_uuid + ) + ON CONFLICT ("user_id", "uuid", "history_uuid") + DO NOTHING; + + + + -- Retrieve the current modified timestamp for this node from the database. + SELECT + uuid, updated, client + INTO + db_uuid, db_updated, db_client + FROM public."node" + WHERE + user_id = p_user_id AND + uuid::uuid = node_uuid::uuid; + + + + -- Is the node not in database? It needs to be created. + IF db_uuid IS NULL THEN + RAISE NOTICE '01 New node %', node_uuid; + + INSERT INTO public."node" ( + user_id, "uuid", parent_uuid, created, updated, + "name", "content", "content_encrypted", + client + ) + VALUES( + p_user_id, + node_uuid, + node_parent_uuid, + COALESCE((node_data->>'Created')::timestamptz, NOW()), + COALESCE((node_data->>'Updated')::timestamptz, NOW()), + (node_data->>'Name')::varchar, + (node_data->>'Content')::text, + '', /* content_encrypted */ + p_client_uuid + ); + + -- Remove/insert file links for the newest content of this node. + CALL manage_node_file_links(p_user_id, node_uuid, (node_data->>'Content')::text); + + CONTINUE; + END IF; + + + + -- Update the public node as well if it was older than incoming node. + IF node_updated > db_updated THEN + UPDATE public."node" + SET + updated = (node_data->>'Updated')::timestamptz, + updated_seq = nextval('node_updates'), + parent_uuid = node_parent_uuid, + name = (node_data->>'Name')::varchar, + content = (node_data->>'Content')::text, + client = p_client_uuid + WHERE + user_id = p_user_id AND + uuid::uuid = node_uuid::uuid; + + -- Remove/insert file links for the newest content of this node. + CALL manage_node_file_links(p_user_id, node_uuid, (node_data->>'Content')::text); + END IF; + + END LOOP; +END +$procedure$ +; + +CREATE OR REPLACE PROCEDURE public.manage_node_file_links(IN p_user_id integer, IN p_node_uuid uuid, IN p_node_content text) +LANGUAGE plpgsql +AS $fn$ +BEGIN + + -- Old file links are purged to get rid of deleted ones. + -- New links will be added from latest node update. + DELETE FROM file_node_links + WHERE + user_id = p_user_id AND + node_uuid = p_node_uuid; + + -- db:// URLs are extracted from the content to manage them in a separate table. + WITH file_uuids AS ( + SELECT regexp_matches( + p_node_content, + 'db://([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})', + 'g' + ) AS file_uuid + ) + INSERT INTO file_node_links(user_id, node_uuid, file_uuid) + SELECT + p_user_id, + p_node_uuid, + (file_uuid[1])::uuid + FROM file_uuids; + +END +$fn$ +; diff --git a/static/images/icon_storage.svg b/static/images/icon_storage.svg new file mode 100644 index 0000000..563d355 --- /dev/null +++ b/static/images/icon_storage.svg @@ -0,0 +1,49 @@ + + + + + + + + database + + + diff --git a/static/js/api.mjs b/static/js/api.mjs index 26a19de..9b53cbf 100644 --- a/static/js/api.mjs +++ b/static/js/api.mjs @@ -15,18 +15,61 @@ export class API { const res = await fetch(path, { method, headers, body }) // An HTTP communication level error occured. if (!res.ok || res.status != 200) - throw new Error('HTTP error', { cause: { type: 'http', error: res, }}) + throw new Error('HTTP error', { cause: { type: 'http', error: res, } }) // Application level response are handled here. const json = await res.json() if (!json.OK) - throw new Error(json.Error, { cause: { type: 'application', application: json, }}) + throw new Error(json.Error, { cause: { type: 'application', application: json, } }) + + return json + } catch (err) { + // Catch any other errors from fetch. + throw new Error(err.message, { cause: { type: 'http', error: err, } }) + } + } + + // Sends a block of binary data to server. + static async upload(uuid, data, sentBytes, metadata) { + try { + const path = `/sync/file/upload/${uuid}` + const headers = { + 'X-File-Sent-Bytes': sentBytes, + 'X-File-Total-Bytes': metadata.size, + } + + // Metadata is needed for the database. + // No need to send it every block. + if (sentBytes === 0) { + headers['X-File-Name'] = metadata.name + headers['X-File-Size'] = metadata.size + headers['X-File-Type'] = metadata.type + headers['X-File-Modified'] = metadata.modified + } + + // Authentication is done with a bearer token. + // Here provided to the backend if set. + const token = localStorage.getItem('token') + if (token) { + headers.Authorization = `Bearer ${token}` + } + + const res = await fetch(path, { method: 'POST', headers, body: data }) + // An HTTP communication level error occured. + if (!res.ok || res.status != 200) + throw new Error('HTTP error', { cause: { type: 'http', error: res, } }) + + // Application level response are handled here. + const json = await res.json() + if (!json.OK) + throw new Error(json.Error, { cause: { type: 'application', application: json, } }) return json } catch (err) { // Catch any other errors from fetch. - throw new Error(err.message, { cause: { type: 'http', error: err, }}) + console.error(err) + throw new Error(err, { cause: { type: 'http', error: err } }) } } diff --git a/static/js/app.mjs b/static/js/app.mjs index 90bad39..e44eabd 100644 --- a/static/js/app.mjs +++ b/static/js/app.mjs @@ -1,7 +1,6 @@ -import { ROOT_NODE } from 'node_store' +import { ROOT_NODE, Node } from 'node_store' import { CustomHTMLElement } from './lib/custom_html_element.mjs' import { N2Sidebar } from 'sidebar' -import { Node } from 'node' import { N2PreferenceSet } from './page_preferences.mjs' export class App { @@ -66,11 +65,10 @@ export class App { _mbus.subscribe('DEVICE_PREFERENCE_SET_UPDATED', ()=>{ this.preferences = this.getPreferences() - console.log(this.preferences.data) }) - window.addEventListener('keydown', event => this.keyHandler(event)) - window.addEventListener('popstate', event => this.popState(event)) + globalThis.addEventListener('keydown', event => this.keyHandler(event)) + globalThis.addEventListener('popstate', event => this.popState(event)) document.getElementById('notes2').addEventListener('click', event => { if (event.target.id === 'notes2') document.getElementById('node-content')?.focus() @@ -81,12 +79,12 @@ export class App { _mbus.dispatch('SHOW_PAGE', { page: 'node' }) - window._sync = new Sync() + globalThis._sync = new Sync() // I think it is uncomfortable having the sync running as soon as the page load. // I haven't gotten the time to look at the page before stuff jumps around. // There a slight delay to initiate sync seems reasonable. - setTimeout(() => window._sync.run(), 1000) + setTimeout(() => globalThis._sync.run(), 1000) }// }}} keyHandler(event) {//{{{ let handled = true diff --git a/static/js/crypto.mjs b/static/js/crypto.mjs deleted file mode 100644 index 9aab255..0000000 --- a/static/js/crypto.mjs +++ /dev/null @@ -1,72 +0,0 @@ -export default class Crypto { - constructor(key) {//{{{ - if(key === null) - throw new Error("No key provided") - - if(typeof key === 'string') - this.key = sjcl.codec.base64.toBits(base64_key) - else - this.key = key - - this.aes = new sjcl.cipher.aes(this.key) - }//}}} - - static generate_key() {//{{{ - return sjcl.random.randomWords(8) - }//}}} - static pass_to_key(pass, salt = null) {//{{{ - if(salt === null) - salt = sjcl.random.randomWords(4) // 128 bits (16 bytes) - let key = sjcl.misc.pbkdf2(pass, salt, 10000) - - return { - salt, - key, - } - }//}}} - - encrypt(plaintext_data_in_bits, counter, return_encoded = true) {//{{{ - // 8 bytes of random data, (1 word = 4 bytes) * 2 - // with 8 bytes of byte encoded counter is used as - // IV to guarantee a non-repeated IV (which is a catastrophe). - // Assumes counter value is kept unique. Counter is taken from - // Postgres sequence. - let random_bits = sjcl.random.randomWords(2) - let iv_bytes = sjcl.codec.bytes.fromBits(random_bits) - for (let i = 0; i < 8; ++i) { - let mask = 0xffn << BigInt(i*8) - let counter_i_byte = (counter & mask) >> BigInt(i*8) - iv_bytes[15-i] = Number(counter_i_byte) - } - let iv = sjcl.codec.bytes.toBits(iv_bytes) - - let encrypted = sjcl.mode['ccm'].encrypt( - this.aes, - plaintext_data_in_bits, - iv, - ) - - // Returning 16 bytes (4 words) IV + encrypted data. - if(return_encoded) - return sjcl.codec.base64.fromBits( - iv.concat(encrypted) - ) - else - return iv.concat(encrypted) - }//}}} - decrypt(encrypted_base64_data) {//{{{ - try { - let encoded = sjcl.codec.base64.toBits(encrypted_base64_data) - let iv = encoded.slice(0, 4) // in words (4 bytes), not bytes - let encrypted_data = encoded.slice(4) - return sjcl.mode['ccm'].decrypt(this.aes, encrypted_data, iv) - } catch(err) { - if(err.message == `ccm: tag doesn't match`) - throw('Decryption failed') - else - throw(err) - } - }//}}} -} - -// vim: foldmethod=marker diff --git a/static/js/file.mjs b/static/js/file.mjs index 4322218..8e3bb94 100644 --- a/static/js/file.mjs +++ b/static/js/file.mjs @@ -1,7 +1,8 @@ import { CustomHTMLElement } from "./lib/custom_html_element.mjs"; +import { API } from 'api' export class N2File extends CustomHTMLElement { - static { + static {// {{{ this.tmpl = document.createElement('template') this.tmpl.innerHTML = ` +

Local storage

-
-
-
+
+
Local nodes
+
+ +
Queued to sync
+
+ +
History nodes
+
+
+ +

Files

+
+
Files on server
+
+ +
Files to send
+
+
` } constructor() { - super() + super(true) window._mbus.subscribe('SHOW_PAGE', event => { if (event.detail.data?.page == 'storage') @@ -19,13 +53,21 @@ export class N2PageStorage extends CustomHTMLElement { }) } async render() { - const countNodes = await globalThis.nodeStore.nodeCount() - const countQueuedNodes = await globalThis.nodeStore.sendQueue.count() - const countHistoryNodes = await globalThis.nodeStore.nodesHistory.count() + API.query('GET', '/file/count') + .then(res => { + this.elCountFilesOnServer.innerText = res.Count + }) + .catch(err => { + this.elCountFilesOnServer.classList.add('error') + console.error(err) + this.elCountFilesOnServer.innerText = err.message + }) - this.elCountNodes.innerText = countNodes - this.elCountQueuedNodes.innerText = countQueuedNodes - this.elCountHistoryNodes.innerText = countHistoryNodes + + this.elCountNodes.innerText = await globalThis.nodeStore.nodeCount() + this.elCountQueuedNodes.innerText = await globalThis.nodeStore.sendQueue.count() + this.elCountHistoryNodes.innerText = await globalThis.nodeStore.nodesHistory.count() + this.elCountQueuedFiles.innerText = await globalThis.nodeStore.filesMetadata.countToUpload() } } customElements.define('n2-pagestorage', N2PageStorage) diff --git a/static/js/sidebar.mjs b/static/js/sidebar.mjs index 6cd5814..899e955 100644 --- a/static/js/sidebar.mjs +++ b/static/js/sidebar.mjs @@ -1,5 +1,4 @@ -import { ROOT_NODE, ORPHANED_NODE, DELETED_NODE } from 'node_store' -import { Node } from 'node' +import { ROOT_NODE, ORPHANED_NODE, DELETED_NODE, Node } from 'node_store' import { CustomHTMLElement } from './lib/custom_html_element.mjs' import { Color, Solver } from './lib/css_colorize.mjs' @@ -82,11 +81,16 @@ export class N2Sidebar extends CustomHTMLElement { .icons { display: flex; - justify-content: center; + justify-content: start; margin-top: 16px; - gap: 8px; + gap: 8px 16px; + padding-left: 40px; padding-bottom: 16px; border-bottom: 1px solid var(--line-color); + + img { + cursor: pointer; + } } @@ -104,6 +108,7 @@ export class N2Sidebar extends CustomHTMLElement {
+
@@ -127,6 +132,7 @@ export class N2Sidebar extends CustomHTMLElement { this.addEventListener('keydown', event => this.keyHandler(event)) this.elSearch.addEventListener('click', () => _mbus.dispatch('op-search')) this.elSync.addEventListener('click', () => _sync.run()) + this.elStorage.addEventListener('click', () => _mbus.dispatch('SHOW_PAGE', { page: 'storage' })) this.elLogo.addEventListener('click', () => _app.goToNode(ROOT_NODE, false, false)) this.elSettings.addEventListener('click', ()=> _mbus.dispatch('SHOW_PAGE', { page: 'preferences' })) this.elHideTree.addEventListener('click', event => { diff --git a/static/js/sync.mjs b/static/js/sync.mjs index daa603f..f68a09f 100644 --- a/static/js/sync.mjs +++ b/static/js/sync.mjs @@ -1,5 +1,5 @@ import { API } from 'api' -import { Node } from 'node' +import { Node } from 'node_store' import { CustomHTMLElement } from './lib/custom_html_element.mjs' export class Sync { @@ -164,6 +164,53 @@ export class Sync { } } }//}}} + async filesToServer() {// {{{ + try { + const BLOCKSIZE = 128 * 1024 + let sentBytes = 0 + + const metadata = await nodeStore.nextFileToSync() + if (metadata === null) + return + + const stream = await globalThis.nodeStore.fileSegments.getStream(metadata.UUID) + const reader = stream.getReader({ mode: 'byob' }) // Bring Your Own Buffer + let buffer = new Uint8Array(BLOCKSIZE) + + while (true) { + const { value, done } = await reader.read(buffer) + + await API.upload(metadata.UUID, value, sentBytes, metadata) + metadata.setUploaded() + nodeStore.filesMetadata.add(metadata) + + if (done) + break + sentBytes += value.length + buffer = new Uint8Array(value.buffer) + } + reader.releaseLock() + } catch (e) { + console.error(e) + alert(e.message) + } + }// }}} + + async filesFromServer() {// {{{ + let offset = 0 + let more = true + + try { + while (more) { + const res = await API.query('GET', `/file/server_uuids/${offset}`) + more = res.MoreRowsExist + console.log(res) + } + } catch (e) { + console.error(e) + alert(e.message) + } + }// }}} } export class N2SyncProgress extends CustomHTMLElement { diff --git a/static/service_worker.js b/static/service_worker.js index 8522b20..7cf268e 100644 --- a/static/service_worker.js +++ b/static/service_worker.js @@ -1,3 +1,5 @@ +import { NodeStore } from '/js/{{ .VERSION }}/node_store.mjs' + const CACHE_NAME = 'notes2-{{ .VERSION }}' const CACHED_ASSETS = [ '/', @@ -11,16 +13,21 @@ const CACHED_ASSETS = [ '/images/{{ .VERSION }}/collapsed.svg', '/images/{{ .VERSION }}/expanded.svg', + '/images/{{ .VERSION }}/icon_back.svg', '/images/{{ .VERSION }}/icon_history.svg', '/images/{{ .VERSION }}/icon_home.svg', '/images/{{ .VERSION }}/icon_markdown_hollow.svg', '/images/{{ .VERSION }}/icon_markdown.svg', + '/images/{{ .VERSION }}/icon_menu.svg', + '/images/{{ .VERSION }}/icon_new_document.svg', '/images/{{ .VERSION }}/icon_refresh.svg', '/images/{{ .VERSION }}/icon_save_disabled.svg', '/images/{{ .VERSION }}/icon_save.svg', '/images/{{ .VERSION }}/icon_search.svg', '/images/{{ .VERSION }}/icon_settings.svg', + '/images/{{ .VERSION }}/icon_storage.svg', '/images/{{ .VERSION }}/icon_table.svg', + '/images/{{ .VERSION }}/icon_transfer.svg', '/images/{{ .VERSION }}/leaf.svg', '/images/{{ .VERSION }}/logo_small.svg', '/images/{{ .VERSION }}/logo.svg', @@ -28,20 +35,18 @@ const CACHED_ASSETS = [ '/js/{{ .VERSION }}/api.mjs', '/js/{{ .VERSION }}/app.mjs', '/js/{{ .VERSION }}/checklist.mjs', - '/js/{{ .VERSION }}/crypto.mjs', '/js/{{ .VERSION }}/file.mjs', - '/js/{{ .VERSION }}/key.mjs', '/js/{{ .VERSION }}/lib/css_colorize.mjs', '/js/{{ .VERSION }}/lib/custom_html_element.mjs', '/js/{{ .VERSION }}/lib/node_modules/marked/lib/marked.esm.js', '/js/{{ .VERSION }}/lib/node_modules/marked-token-position/lib/index.esm.js', - '/js/{{ .VERSION }}/lib/sjcl.js', '/js/{{ .VERSION }}/marked_position.mjs', '/js/{{ .VERSION }}/mbus.mjs', '/js/{{ .VERSION }}/node_store.mjs', '/js/{{ .VERSION }}/notes2.mjs', '/js/{{ .VERSION }}/page_history.mjs', '/js/{{ .VERSION }}/page_node.mjs', + '/js/{{ .VERSION }}/page_preferences.mjs', '/js/{{ .VERSION }}/page_storage.mjs', '/js/{{ .VERSION }}/sidebar.mjs', '/js/{{ .VERSION }}/sync.mjs', @@ -120,14 +125,55 @@ self.addEventListener('activate', event => { }) self.addEventListener('fetch', event => { + const url = new URL(event.request.url) + // The fetch event is also seeing requests to other domains. // Just let the browser handle those for itself. - const ourDomain = event.request.url.startsWith(self.location.origin) + const ourDomain = (url.host == self.location.host) if (!ourDomain) return event + if (url.pathname == '/stream-local') { + event.respondWith(indexeddbFile(event)) + return + } + if (`{{ .DevMode }}` == 'true') return event - - event.respondWith(fetchAsset(event)) + else + event.respondWith(fetchAsset(event)) }) + + +let nodestore = null +async function getNodestore() { + if (nodestore) + return nodestore + + // When setting nodestore directly, then waiting on initializeDB, + // a second call for getNodestore can return the nodestore before + // initialized. + const initializingNodeStore = new NodeStore() + await initializingNodeStore.initializeDB() + nodestore = initializingNodeStore + return nodestore +} + +async function indexeddbFile(event) { + const url = new URL(event.request.url) + const uuid = url.searchParams.get('uuid') + + try { + const nodeStore = await getNodestore() + const md = await nodeStore.filesMetadata.get(uuid) + const stream = await nodeStore.fileSegments.getStream(uuid) + + return new Response(stream, { + headers: { + 'Content-Type': md.type, + } + }) + } catch (err) { + console.error(err) + } +} diff --git a/views/layouts/main.gotmpl b/views/layouts/main.gotmpl index c5fbead..13a0bcf 100644 --- a/views/layouts/main.gotmpl +++ b/views/layouts/main.gotmpl @@ -9,10 +9,8 @@ { "imports": { "api": "/js/{{ .VERSION }}/api.mjs", - "sync": "/js/{{ .VERSION }}/sync.mjs", - "key": "/js/{{ .VERSION }}/key.mjs", + "sync": "/js/{{ .VERSION }}/sync.mjs", "checklist": "/js/{{ .VERSION }}/checklist.mjs", - "crypto": "/js/{{ .VERSION }}/crypto.mjs", "node_store": "/js/{{ .VERSION }}/node_store.mjs", "node": "/js/{{ .VERSION }}/page_node.mjs", "sidebar": "/js/{{ .VERSION }}/sidebar.mjs" @@ -20,14 +18,14 @@ } diff --git a/views/pages/login.gotmpl b/views/pages/login.gotmpl index 3e2235f..8e824aa 100644 --- a/views/pages/login.gotmpl +++ b/views/pages/login.gotmpl @@ -4,7 +4,7 @@ - +
@@ -42,6 +42,6 @@ class Login { } } -window._login = new Login() +globalThis._login = new Login() {{ end }} diff --git a/views/pages/notes2.gotmpl b/views/pages/notes2.gotmpl index 2755aea..c5df63e 100644 --- a/views/pages/notes2.gotmpl +++ b/views/pages/notes2.gotmpl @@ -7,7 +7,7 @@
-
>
+
>
@@ -47,20 +47,21 @@ import {App} from "/js/{{ .VERSION }}/app.mjs" import {API} from 'api' import {Sync} from 'sync' + import { } from '/js/{{ .VERSION }}/page_node.mjs' import { } from '/js/{{ .VERSION }}/page_preferences.mjs' import { } from '/js/{{ .VERSION }}/page_storage.mjs' import { } from '/js/{{ .VERSION }}/page_history.mjs' import { } from '/js/{{ .VERSION }}/file.mjs' - window.Sync = Sync + globalThis.Sync = Sync if (!API.hasAuthenticationToken()) { location.href = '/login' } else { try { - window.nodeStore = new NodeStore() - window.nodeStore.initializeDB().then(() => { - window._app = new App() + globalThis.nodeStore = new NodeStore() + globalThis.nodeStore.initializeDB().then(() => { + globalThis._app = new App() }) } catch (e) { alert(e) diff --git a/views/pages/sync.gotmpl b/views/pages/sync.gotmpl index eb79c77..c9fa870 100644 --- a/views/pages/sync.gotmpl +++ b/views/pages/sync.gotmpl @@ -5,9 +5,9 @@