Manual server file uploads implemented

config.go

@@ -24,6 +24,10 @@ type Config struct {
 		Secret     string
 		ExpireDays int
 	}
+
+	Storage struct {
+		Files string
+	}
 }
 
 func readConfig(fname string) (err error) {

file.go

@@ -5,35 +5,46 @@ import (
 	"github.com/jmoiron/sqlx"
 
 	// Standard
+	"fmt"
+	"os"
+	"path"
 	"time"
 )
 
 type File struct {
 	ID       int
+	UUID     string
 	UserID   int `db:"user_id"`
-	NodeID   int `db:"node_id"`
 	Filename string
-	Size     int64
+	Size     int
 	MIME     string
-	MD5      string
-	Uploaded time.Time
+	Modified time.Time
 }
 
-func AddFile(userID int, file *File) (err error) { // {{{
+var (
+	fileUUIDCache map[string]int
+)
+
+func init() {
+	fileUUIDCache = make(map[string]int)
+}
+
+func AddFileToDb(userID int, file *File) (err error) { // {{{
 	file.UserID = userID
 
 	var rows *sqlx.Rows
 	rows, err = db.Queryx(`
-		INSERT INTO file(user_id, node_id, filename, size, mime, md5)
-		VALUES($1, $2, $3, $4, $5, $6)
+		INSERT INTO public.file(user_id, uuid, name, size, type, modified, upload_complete)
+		VALUES($1, $2, $3, $4, $5, $6, false)
+		ON CONFLICT (user_id, uuid) DO UPDATE set name = public.file.name
 		RETURNING id
 		`,
 		file.UserID,
-		file.NodeID,
+		file.UUID,
 		file.Filename,
 		file.Size,
 		file.MIME,
-		file.MD5,
+		file.Modified,
 	)
 	if err != nil {
 		return
@@ -76,5 +87,40 @@ func Files(userID int, nodeUUID string, fileID int) (files []File, err error) {
 
 	return
 } // }}}
+func FileIDToPath(id int) (string, string) { // {{{
+	str := fmt.Sprintf("%012d", id)
+	dir := path.Join(config.Storage.Files, str[0:4], str[4:8])
+	fpath := path.Join(config.Storage.Files, str[0:4], str[4:8], str[8:12])
+	return dir, fpath
+} // }}}
+
+func FileWriteData(userID int, dbID int, data []byte, create bool) (err error) {// {{{
+	dir, fpath := FileIDToPath(dbID)
+	err = os.MkdirAll(dir, 0700)
+	if err != nil {
+		Log.Error("file", "error", err)
+		return
+	}
+
+	var f *os.File
+	if create {
+		f, err = os.OpenFile(fpath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
+	} else {
+		f, err = os.OpenFile(fpath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
+	}
+	if err != nil {
+		Log.Error("file", "error", err)
+		return
+	}
+	defer f.Close()
+
+	_, err = f.Write(data)
+	if err != nil {
+		Log.Error("file", "error", err)
+		return
+	}
+
+	return
+}// }}}
 
 // vim: foldmethod=marker

main.go

@@ -5,7 +5,6 @@ import (
 	"notes2/authentication"
 	"notes2/html_template"
 	appUser "notes2/user"
-	"os"
 
 	// Standard
 	"bufio"
@@ -17,11 +16,13 @@ import (
 	"io"
 	"log/slog"
 	"net/http"
+	"os"
 	"path"
 	"regexp"
 	"strconv"
 	"strings"
 	"text/template"
+	"time"
 )
 
 const VERSION = "v29"
@@ -71,7 +72,7 @@ func init() { // {{{
 } // }}}
 func initLog() { // {{{
 	opts := slog.HandlerOptions{}
-	opts.Level = slog.LevelDebug
+	opts.Level = slog.LevelInfo
 	Log = slog.New(slog.NewJSONHandler(os.Stdout, &opts))
 } // }}}
 func main() { // {{{
@@ -141,6 +142,7 @@ func main() { // {{{
 	http.HandleFunc("/sync/from_server/count/{sequence}", authenticated(actionSyncFromServerCount))
 	http.HandleFunc("/sync/from_server/{sequence}/{offset}", authenticated(actionSyncFromServer))
 	http.HandleFunc("/sync/to_server", authenticated(actionSyncToServer))
+	http.HandleFunc("/sync/file/upload/{uuid}", authenticated(actionSyncFileUpload))
 
 	http.HandleFunc("/node/retrieve/{uuid}", authenticated(actionNodeRetrieve))
 	http.HandleFunc("/node/history/retrieve/{uuid}/{offset}", authenticated(actionNodeHistoryRetrieve))
@@ -391,6 +393,63 @@ func actionSyncToServer(w http.ResponseWriter, r *http.Request) { // {{{
 		"OK": true,
 	})
 } // }}}
+func actionSyncFileUpload(w http.ResponseWriter, r *http.Request) { // {{{
+	uuid := r.PathValue("uuid")
+	session := getUserSession(r)
+
+	sentBytes, _ := strconv.Atoi(r.Header.Get("X-File-Sent-Bytes"))
+	totalBytes, _ := strconv.Atoi(r.Header.Get("X-File-Total-Bytes"))
+
+	// First block comes with file metadata.
+	file := File{}
+	if sentBytes == 0 {
+		file.UUID = uuid
+		file.Filename = r.Header.Get("X-File-Name")
+		file.Size, _ = strconv.Atoi(r.Header.Get("X-File-Size"))
+		file.MIME = r.Header.Get("X-File-Type")
+		modded, _ := strconv.ParseInt(r.Header.Get("X-File-Modified"), 10, 64)
+		file.Modified = time.UnixMilli(modded)
+		err := AddFileToDb(session.UserID, &file)
+		if err != nil {
+			Log.Error("upload", "op", "AddFileToDB", "error", err)
+			httpError(w, err)
+			return
+		}
+		fileUUIDCache[uuid] = file.ID
+	}
+
+	data, _ := io.ReadAll(r.Body)
+	var err error
+
+	switch sentBytes {
+	case 0:
+		// First block of data.
+		Log.Info("file_upload", "status", "new", "user", session.UserID, "uuid", uuid, "total", totalBytes)
+		id := fileUUIDCache[uuid]
+		err = FileWriteData(session.UserID, id, data, true)
+
+	case totalBytes:
+		// Last block of data - sent with 0 length data.
+		Log.Info("file_upload", "status", "done", "user", session.UserID, "uuid", uuid, "total", totalBytes)
+		id := fileUUIDCache[uuid]
+		delete(fileUUIDCache, uuid)
+		_, err = db.Exec(`UPDATE public.file SET upload_complete = true WHERE id = $1`, id) 
+
+	default:
+		id := fileUUIDCache[uuid]
+		err = FileWriteData(session.UserID, id, data, false)
+	}
+
+	if err != nil {
+		Log.Error("upload", "error", err)
+		httpError(w, err)
+		return
+	}
+
+	responseData(w, map[string]any{
+		"OK": true,
+	})
+} // }}}
 
 func actionUserGetPreferences(w http.ResponseWriter, r *http.Request) { // {{{
 	user := getUserSession(r)

sql/00011.sql

@@ -0,0 +1,13 @@
+CREATE TABLE public.file (
+	id serial NOT NULL,
+	user_id int4 NOT NULL,
+	uuid uuid NOT NULL,
+	name varchar DEFAULT '' NOT NULL,
+	"size" int DEFAULT 0 NOT NULL,
+	"type" varchar DEFAULT 'application/octet-stream' NOT NULL,
+	modified timestamptz DEFAULT NOW() NOT NULL,
+	CONSTRAINT file_pk PRIMARY KEY (id),
+	CONSTRAINT file_user_fk FOREIGN KEY (user_id) REFERENCES public."user"(id) ON DELETE RESTRICT ON UPDATE RESTRICT
+);
+
+ALTER TABLE public.file ADD CONSTRAINT file_user_uuid_unique UNIQUE (user_id,"uuid");

sql/00012.sql

@@ -0,0 +1 @@
+ALTER TABLE public.file ADD upload_complete bool DEFAULT false NOT NULL;

static/js/api.mjs

@@ -30,6 +30,50 @@ export class API {
 		}
 	}
 
+	// Sends a block of binary data to server.
+	static async upload(uuid, data, sentBytes, file) {
+		try {
+			const path = `/sync/file/upload/${uuid}`
+			const headers = {
+				'X-File-Sent-Bytes': sentBytes,
+				'X-File-Total-Bytes': file.size,
+			}
+
+			// Metadata is needed for the database.
+			// No need to send it every block.
+			if (sentBytes === 0) {
+				headers['X-File-Name'] = file.name
+				headers['X-File-Size'] = file.size
+				headers['X-File-Type'] = file.type
+				headers['X-File-Modified'] = file.lastModified
+			}
+
+			// Authentication is done with a bearer token.
+			// Here provided to the backend if set.
+			const token = localStorage.getItem('token')
+			if (token) {
+				headers.Authorization = `Bearer ${token}`
+			}
+
+			const res = await fetch(path, { method: 'POST', headers, body: data })
+			// An HTTP communication level error occured.
+			if (!res.ok || res.status != 200)
+				throw new Error('HTTP error', { cause: { type: 'http', error: res, } })
+
+			// Application level response are handled here.
+			const json = await res.json()
+			if (!json.OK)
+				throw new Error(json.Error, { cause: { type: 'application', application: json, } })
+
+			return json
+
+		} catch (err) {
+			// Catch any other errors from fetch.
+			console.error(err)
+			throw new Error(err, { cause: { type: 'http', error: err } })
+		}
+	}
+
 	static hasAuthenticationToken() {//{{{
 		const token = localStorage.getItem('token')
 		return token !== null && token !== ''

static/js/file.mjs

@@ -1,7 +1,7 @@
 import { CustomHTMLElement } from "./lib/custom_html_element.mjs";
 
 export class N2File extends CustomHTMLElement {
-	static {
+	static {// {{{
 		this.tmpl = document.createElement('template')
 		this.tmpl.innerHTML = `
 			<style>
@@ -30,24 +30,46 @@ export class N2File extends CustomHTMLElement {
 			<img data-el="image" src="/images/${_VERSION}/file_icons/generic.svg">
 			<div data-el="filename"></div>
 		`
-	}
-	constructor() {
+	}// }}}
+	constructor() {// {{{
 		super(true)
 
 		this.addEventListener('click', event => {
 			event.preventDefault()
 			event.stopPropagation()
 
+			// Download to disk.
+			if (event.shiftKey) {
+				this.downloadToDisk()
+				return
+			}
+
+			// Open in browser.
 			window.open(
 				URL.createObjectURL(this.file),
-				(event.ctrlKey || event.shiftKey) ? '_blank' : '_self',
+				event.ctrlKey ? '_blank' : '_self',
 			)
 		})
 
 		this.render()
-	}
+	}// }}}
+	async downloadToDisk() {// {{{
+		try {
+			const handle = await window.showSaveFilePicker({
+				suggestedName: this.file.name,
+			})
 
-	async render() {
+			const writable = await handle.createWritable()
+			const blobStream = this.file.stream()
+			await blobStream.pipeTo(writable)
+		} catch (err) {
+			if (err.name == 'AbortError')
+				return
+			console.error(err)
+			alert(err.message)
+		}
+	}// }}}
+	async render() {// {{{
 		const src = this.getAttribute('src')
 
 		// N2's db:// URLs are fetched from IndexedDB.
@@ -76,6 +98,6 @@ export class N2File extends CustomHTMLElement {
 			}
 		} else
 			this.elImage.src = src
-	}
+	}// }}}
 }
 customElements.define('n2-file', N2File)

static/js/node_store.mjs

@@ -15,12 +15,13 @@ export class NodeStore {
 		this.sendQueue = null
 		this.nodesHistory = null
 		this.files = null
+		this.filesMetadata = null
 
 		this.initializeSpecialNodes()
 	}//}}}
 	initializeDB() {//{{{
 		return new Promise((resolve, reject) => {
-			const req = indexedDB.open('notes', 8)
+			const req = indexedDB.open('notes', 10)
 
 			// Schema upgrades for IndexedDB.
 			// These can start from different points depending on updates to Notes2 since a device was online.
@@ -71,6 +72,14 @@ export class NodeStore {
 						case 8:
 							files = db.createObjectStore('files', { keyPath: 'UUID' })
 							break
+
+						case 9:
+							db.createObjectStore('files_metadata', { keyPath: 'UUID' })
+							break
+
+						case 10:
+							trx.objectStore('files_metadata').createIndex('byUploaded', 'uploaded', { unique: false })
+							break
 					}
 				}
 			}
@@ -80,6 +89,7 @@ export class NodeStore {
 				this.sendQueue = new SimpleNodeStore(this.db, 'send_queue')
 				this.nodesHistory = new NodeHistoryStore(this.db, 'nodes_history')
 				this.files = new SimpleNodeStore(this.db, 'files')
+				this.filesMetadata = new SimpleNodeStore(this.db, 'files_metadata')
 				resolve()
 			}
 
@@ -332,6 +342,56 @@ export class NodeStore {
 			}
 		})
 	}//}}}
+
+	async storeFile(file) {// {{{
+		const metadata = new FileMetadata(file, {})
+
+		await this.files.add({ data: { UUID: metadata.UUID, file } })
+		await this.filesMetadata.add({ data: metadata })
+
+		return metadata
+	}// }}}
+	nextFileToSync() {// {{{
+		return new Promise((resolve, reject) => {
+			const req = this.db
+				.transaction('files_metadata', 'readonly')
+				.objectStore('files_metadata')
+				.index('byUploaded')
+				.get('only_on_device')
+
+			req.onerror = (e) => reject(e)
+			req.onsuccess = (e) => {
+				const data = e.target.result
+
+				// No more files to sync.
+				if (data === undefined) {
+					resolve({ file: null, metadata: null })
+					return
+				}
+
+				// Retrieve file as well.
+				const filereq = this.db.
+					transaction('files', 'readonly')
+					.objectStore('files')
+					.get(data.UUID)
+
+				filereq.onsuccess = (e) => {
+					const f = e.target.result
+
+					// File should always be in IndexedDB when metadata is.
+					if (f === undefined) {
+						reject(new Error(`File with UUID '${data.UUID}' is missing.`))
+						return
+					}
+
+					resolve({
+						file: f.file,
+						metadata: new FileMetadata(f.file, data),
+					})
+				}
+			}
+		})
+	}// }}}
 }
 
 class SimpleNodeStore {
@@ -351,7 +411,6 @@ class SimpleNodeStore {
 			// Node to be moved is first stored in the new queue.
 			const req = store.put(node.data)
 			req.onsuccess = () => {
-				console.log('here')
 				resolve()
 			}
 			req.onerror = (event) => {
@@ -539,4 +598,29 @@ export function uuidv7() {// {{{
 	return `${str.slice(0, 8)}-${str.slice(8, 12)}-${str.slice(12, 16)}-${str.slice(16, 20)}-${str.slice(20)}`
 }// }}}
 
+class FileMetadata {
+	constructor(file, data) {// {{{
+		this.UUID = data.UUID || uuidv7()
+		this.uploaded = data.uploaded || 'only_on_device'
+
+		this.name = file.name
+		this.size = file.size
+		this.type = file.type || 'application/octet-stream'
+		this.modified = file.lastModified
+	}// }}}
+	setUploaded() {// {{{
+		this.uploaded = 'on_server'
+	}// }}}
+	get data() {// {{{
+		return {
+			UUID: this.UUID,
+			uploaded: this.uploaded,
+			name: this.name,
+			size: this.size,
+			type: this.type,
+			modified: this.modified,
+		}
+	}// }}}
+}
+
 // vim: foldmethod=marker

static/js/page_node.mjs

@@ -267,11 +267,11 @@ export class N2PageNodeUI extends CustomHTMLElement {
 					const file = item.getAsFile()
 					if (!file)
 						throw new Error("Couldn't convert image to file object.")
-					const uuid = uuidv7()
-					await globalThis.nodeStore.files.add({ data: { UUID: uuid, file: file } })
+
+					const fileMetadata = await globalThis.nodeStore.storeFile(file)
 
 					const [start, end] = [this.elNodeContent.selectionStart, this.elNodeContent.selectionEnd]
-					this.elNodeContent.setRangeText(`![${file.name}](db://${uuid})`, start, end, 'select');
+					this.elNodeContent.setRangeText(`![${file.name}](db://${fileMetadata.UUID})`, start, end, 'select');
 
 					// Editing the textarea programatically doesn't generate the events it usually gets when edited interactively.
 					this.node.setContent(this.elNodeContent.value)

static/js/sync.mjs

@@ -164,6 +164,36 @@ export class Sync {
 			}
 		}
 	}//}}}
+	async filesToServer() {
+		try {
+			const BLOCKSIZE = 128*1024
+			let sentBytes = 0
+
+			const { file, metadata } = await nodeStore.nextFileToSync()
+			if (file === null)
+				return
+
+			const stream = file.stream()
+			const reader = stream.getReader({ mode: 'byob' }) // Bring Your Own Buffer
+			let buffer = new Uint8Array(BLOCKSIZE)
+
+			while (true) {
+				const { value, done } = await reader.read(buffer)
+
+				await API.upload(metadata.UUID, value, sentBytes, file)
+				metadata.setUploaded()
+				nodeStore.filesMetadata.add(metadata)
+
+				if (done)
+					break
+				sentBytes += value.length
+				buffer = new Uint8Array(value.buffer)
+			}
+		} catch (e) {
+			console.error(e)
+			alert(e.message)
+		}
+	}
 }
 
 export class N2SyncProgress extends CustomHTMLElement {