magnus/Notes2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Manual server file uploads implemented

Browse source
This commit is contained in:
Magnus Åhall 2026-06-26 08:31:54 +02:00
parent c8308664d3
commit 70b5285e16
10 changed files with 329 additions and 26 deletions
Download patch file Download diff file Expand all files Collapse all files

4
config.go
View file

@ -24,6 +24,10 @@ type Config struct {
Secret string
ExpireDays int
}
Storage struct {
Files string
}
}
func readConfig(fname string) (err error) {

64
file.go
View file

@ -5,35 +5,46 @@ import (
"github.com/jmoiron/sqlx"
// Standard
"fmt"
"os"
"path"
"time"
)
type File struct {
ID int
UUID string
UserID int `db:"user_id"`
NodeID int `db:"node_id"`
Filename string
Size int64
Size int
MIME string
MD5 string
Uploaded time.Time
Modified time.Time
}
func AddFile(userID int, file *File) (err error) { // {{{
var (
fileUUIDCache map[string]int
)
func init() {
fileUUIDCache = make(map[string]int)
}
func AddFileToDb(userID int, file *File) (err error) { // {{{
file.UserID = userID
var rows *sqlx.Rows
rows, err = db.Queryx(`
INSERT INTO file(user_id, node_id, filename, size, mime, md5)
VALUES($1, $2, $3, $4, $5, $6)
INSERT INTO public.file(user_id, uuid, name, size, type, modified, upload_complete)
VALUES($1, $2, $3, $4, $5, $6, false)
ON CONFLICT (user_id, uuid) DO UPDATE set name = public.file.name
RETURNING id
`,
file.UserID,
file.NodeID,
file.UUID,
file.Filename,
file.Size,
file.MIME,
file.MD5,
file.Modified,
)
if err != nil {
return
@ -76,5 +87,40 @@ func Files(userID int, nodeUUID string, fileID int) (files []File, err error) {
return
} // }}}
func FileIDToPath(id int) (string, string) { // {{{
str := fmt.Sprintf("%012d", id)
dir := path.Join(config.Storage.Files, str[0:4], str[4:8])
fpath := path.Join(config.Storage.Files, str[0:4], str[4:8], str[8:12])
return dir, fpath
} // }}}
func FileWriteData(userID int, dbID int, data []byte, create bool) (err error) {// {{{
dir, fpath := FileIDToPath(dbID)
err = os.MkdirAll(dir, 0700)
if err != nil {
Log.Error("file", "error", err)
return
}
var f *os.File
if create {
f, err = os.OpenFile(fpath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
} else {
f, err = os.OpenFile(fpath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
}
if err != nil {
Log.Error("file", "error", err)
return
}
defer f.Close()
_, err = f.Write(data)
if err != nil {
Log.Error("file", "error", err)
return
}
return
}// }}}
// vim: foldmethod=marker

63
main.go
View file

@ -5,7 +5,6 @@ import (
"notes2/authentication"
"notes2/html_template"
appUser "notes2/user"
"os"
// Standard
"bufio"
@ -17,11 +16,13 @@ import (
"io"
"log/slog"
"net/http"
"os"
"path"
"regexp"
"strconv"
"strings"
"text/template"
"time"
)
const VERSION = "v29"
@ -71,7 +72,7 @@ func init() { // {{{
} // }}}
func initLog() { // {{{
opts := slog.HandlerOptions{}
opts.Level = slog.LevelDebug
opts.Level = slog.LevelInfo
Log = slog.New(slog.NewJSONHandler(os.Stdout, &opts))
} // }}}
func main() { // {{{
@ -141,6 +142,7 @@ func main() { // {{{
http.HandleFunc("/sync/from_server/count/{sequence}", authenticated(actionSyncFromServerCount))
http.HandleFunc("/sync/from_server/{sequence}/{offset}", authenticated(actionSyncFromServer))
http.HandleFunc("/sync/to_server", authenticated(actionSyncToServer))
http.HandleFunc("/sync/file/upload/{uuid}", authenticated(actionSyncFileUpload))
http.HandleFunc("/node/retrieve/{uuid}", authenticated(actionNodeRetrieve))
http.HandleFunc("/node/history/retrieve/{uuid}/{offset}", authenticated(actionNodeHistoryRetrieve))
@ -391,6 +393,63 @@ func actionSyncToServer(w http.ResponseWriter, r *http.Request) { // {{{
"OK": true,
})
} // }}}
func actionSyncFileUpload(w http.ResponseWriter, r *http.Request) { // {{{
uuid := r.PathValue("uuid")
session := getUserSession(r)
sentBytes, _ := strconv.Atoi(r.Header.Get("X-File-Sent-Bytes"))
totalBytes, _ := strconv.Atoi(r.Header.Get("X-File-Total-Bytes"))
// First block comes with file metadata.
file := File{}
if sentBytes == 0 {
file.UUID = uuid
file.Filename = r.Header.Get("X-File-Name")
file.Size, _ = strconv.Atoi(r.Header.Get("X-File-Size"))
file.MIME = r.Header.Get("X-File-Type")
modded, _ := strconv.ParseInt(r.Header.Get("X-File-Modified"), 10, 64)
file.Modified = time.UnixMilli(modded)
err := AddFileToDb(session.UserID, &file)
if err != nil {
Log.Error("upload", "op", "AddFileToDB", "error", err)
httpError(w, err)
return
}
fileUUIDCache[uuid] = file.ID
}
data, _ := io.ReadAll(r.Body)
var err error
switch sentBytes {
case 0:
// First block of data.
Log.Info("file_upload", "status", "new", "user", session.UserID, "uuid", uuid, "total", totalBytes)
id := fileUUIDCache[uuid]
err = FileWriteData(session.UserID, id, data, true)
case totalBytes:
// Last block of data - sent with 0 length data.
Log.Info("file_upload", "status", "done", "user", session.UserID, "uuid", uuid, "total", totalBytes)
id := fileUUIDCache[uuid]
delete(fileUUIDCache, uuid)
_, err = db.Exec(`UPDATE public.file SET upload_complete = true WHERE id = $1`, id)
default:
id := fileUUIDCache[uuid]
err = FileWriteData(session.UserID, id, data, false)
}
if err != nil {
Log.Error("upload", "error", err)
httpError(w, err)
return
}
responseData(w, map[string]any{
"OK": true,
})
} // }}}
func actionUserGetPreferences(w http.ResponseWriter, r *http.Request) { // {{{
user := getUserSession(r)

13
sql/00011.sql Normal file
View file

@ -0,0 +1,13 @@
CREATE TABLE public.file (
id serial NOT NULL,
user_id int4 NOT NULL,
uuid uuid NOT NULL,
name varchar DEFAULT '' NOT NULL,
"size" int DEFAULT 0 NOT NULL,
"type" varchar DEFAULT 'application/octet-stream' NOT NULL,
modified timestamptz DEFAULT NOW() NOT NULL,
CONSTRAINT file_pk PRIMARY KEY (id),
CONSTRAINT file_user_fk FOREIGN KEY (user_id) REFERENCES public."user"(id) ON DELETE RESTRICT ON UPDATE RESTRICT
);
ALTER TABLE public.file ADD CONSTRAINT file_user_uuid_unique UNIQUE (user_id,"uuid");

1
sql/00012.sql Normal file
View file

@ -0,0 +1 @@
ALTER TABLE public.file ADD upload_complete bool DEFAULT false NOT NULL;

50
static/js/api.mjs
View file

@ -15,18 +15,62 @@ export class API {
const res = await fetch(path, { method, headers, body })
// An HTTP communication level error occured.
if (!res.ok || res.status != 200)
throw new Error('HTTP error', { cause: { type: 'http', error: res, }})
throw new Error('HTTP error', { cause: { type: 'http', error: res, } })
// Application level response are handled here.
const json = await res.json()
if (!json.OK)
throw new Error(json.Error, { cause: { type: 'application', application: json, }})
throw new Error(json.Error, { cause: { type: 'application', application: json, } })
return json
} catch (err) {
// Catch any other errors from fetch.
throw new Error(err.message, { cause: { type: 'http', error: err, }})
throw new Error(err.message, { cause: { type: 'http', error: err, } })
}
}
// Sends a block of binary data to server.
static async upload(uuid, data, sentBytes, file) {
try {
const path = `/sync/file/upload/${uuid}`
const headers = {
'X-File-Sent-Bytes': sentBytes,
'X-File-Total-Bytes': file.size,
}
// Metadata is needed for the database.
// No need to send it every block.
if (sentBytes === 0) {
headers['X-File-Name'] = file.name
headers['X-File-Size'] = file.size
headers['X-File-Type'] = file.type
headers['X-File-Modified'] = file.lastModified
}
// Authentication is done with a bearer token.
// Here provided to the backend if set.
const token = localStorage.getItem('token')
if (token) {
headers.Authorization = `Bearer ${token}`
}
const res = await fetch(path, { method: 'POST', headers, body: data })
// An HTTP communication level error occured.
if (!res.ok || res.status != 200)
throw new Error('HTTP error', { cause: { type: 'http', error: res, } })
// Application level response are handled here.
const json = await res.json()
if (!json.OK)
throw new Error(json.Error, { cause: { type: 'application', application: json, } })
return json
} catch (err) {
// Catch any other errors from fetch.
console.error(err)
throw new Error(err, { cause: { type: 'http', error: err } })
}
}

36
static/js/file.mjs
View file

@ -1,7 +1,7 @@
import { CustomHTMLElement } from "./lib/custom_html_element.mjs";
export class N2File extends CustomHTMLElement {
static {
static {// {{{
this.tmpl = document.createElement('template')
this.tmpl.innerHTML = `
<style>
@ -30,24 +30,46 @@ export class N2File extends CustomHTMLElement {
<img data-el="image" src="/images/${_VERSION}/file_icons/generic.svg">
<div data-el="filename"></div>
`
}
constructor() {
}// }}}
constructor() {// {{{
super(true)
this.addEventListener('click', event => {
event.preventDefault()
event.stopPropagation()
// Download to disk.
if (event.shiftKey) {
this.downloadToDisk()
return
}
// Open in browser.
window.open(
URL.createObjectURL(this.file),
(event.ctrlKey || event.shiftKey) ? '_blank' : '_self',
event.ctrlKey ? '_blank' : '_self',
)
})
this.render()
}
}// }}}
async downloadToDisk() {// {{{
try {
const handle = await window.showSaveFilePicker({
suggestedName: this.file.name,
})
async render() {
const writable = await handle.createWritable()
const blobStream = this.file.stream()
await blobStream.pipeTo(writable)
} catch (err) {
if (err.name == 'AbortError')
return
console.error(err)
alert(err.message)
}
}// }}}
async render() {// {{{
const src = this.getAttribute('src')
// N2's db:// URLs are fetched from IndexedDB.
@ -76,6 +98,6 @@ export class N2File extends CustomHTMLElement {
}
} else
this.elImage.src = src
}
}// }}}
}
customElements.define('n2-file', N2File)

88
static/js/node_store.mjs
View file

@ -15,12 +15,13 @@ export class NodeStore {
this.sendQueue = null
this.nodesHistory = null
this.files = null
this.filesMetadata = null
this.initializeSpecialNodes()
}//}}}
initializeDB() {//{{{
return new Promise((resolve, reject) => {
const req = indexedDB.open('notes', 8)
const req = indexedDB.open('notes', 10)
// Schema upgrades for IndexedDB.
// These can start from different points depending on updates to Notes2 since a device was online.
@ -71,6 +72,14 @@ export class NodeStore {
case 8:
files = db.createObjectStore('files', { keyPath: 'UUID' })
break
case 9:
db.createObjectStore('files_metadata', { keyPath: 'UUID' })
break
case 10:
trx.objectStore('files_metadata').createIndex('byUploaded', 'uploaded', { unique: false })
break
}
}
}
@ -80,6 +89,7 @@ export class NodeStore {
this.sendQueue = new SimpleNodeStore(this.db, 'send_queue')
this.nodesHistory = new NodeHistoryStore(this.db, 'nodes_history')
this.files = new SimpleNodeStore(this.db, 'files')
this.filesMetadata = new SimpleNodeStore(this.db, 'files_metadata')
resolve()
}
@ -332,6 +342,56 @@ export class NodeStore {
}
})
}//}}}
async storeFile(file) {// {{{
const metadata = new FileMetadata(file, {})
await this.files.add({ data: { UUID: metadata.UUID, file } })
await this.filesMetadata.add({ data: metadata })
return metadata
}// }}}
nextFileToSync() {// {{{
return new Promise((resolve, reject) => {
const req = this.db
.transaction('files_metadata', 'readonly')
.objectStore('files_metadata')
.index('byUploaded')
.get('only_on_device')
req.onerror = (e) => reject(e)
req.onsuccess = (e) => {
const data = e.target.result
// No more files to sync.
if (data === undefined) {
resolve({ file: null, metadata: null })
return
}
// Retrieve file as well.
const filereq = this.db.
transaction('files', 'readonly')
.objectStore('files')
.get(data.UUID)
filereq.onsuccess = (e) => {
const f = e.target.result
// File should always be in IndexedDB when metadata is.
if (f === undefined) {
reject(new Error(`File with UUID '${data.UUID}' is missing.`))
return
}
resolve({
file: f.file,
metadata: new FileMetadata(f.file, data),
})
}
}
})
}// }}}
}
class SimpleNodeStore {
@ -351,7 +411,6 @@ class SimpleNodeStore {
// Node to be moved is first stored in the new queue.
const req = store.put(node.data)
req.onsuccess = () => {
console.log('here')
resolve()
}
req.onerror = (event) => {
@ -539,4 +598,29 @@ export function uuidv7() {// {{{
return `${str.slice(0, 8)}-${str.slice(8, 12)}-${str.slice(12, 16)}-${str.slice(16, 20)}-${str.slice(20)}`
}// }}}
class FileMetadata {
constructor(file, data) {// {{{
this.UUID = data.UUID || uuidv7()
this.uploaded = data.uploaded || 'only_on_device'
this.name = file.name
this.size = file.size
this.type = file.type || 'application/octet-stream'
this.modified = file.lastModified
}// }}}
setUploaded() {// {{{
this.uploaded = 'on_server'
}// }}}
get data() {// {{{
return {
UUID: this.UUID,
uploaded: this.uploaded,
name: this.name,
size: this.size,
type: this.type,
modified: this.modified,
}
}// }}}
}
// vim: foldmethod=marker

6
static/js/page_node.mjs
View file

@ -267,11 +267,11 @@ export class N2PageNodeUI extends CustomHTMLElement {
const file = item.getAsFile()
if (!file)
throw new Error("Couldn't convert image to file object.")
const uuid = uuidv7()
await globalThis.nodeStore.files.add({ data: { UUID: uuid, file: file } })
const fileMetadata = await globalThis.nodeStore.storeFile(file)
const [start, end] = [this.elNodeContent.selectionStart, this.elNodeContent.selectionEnd]
this.elNodeContent.setRangeText(`![${file.name}](db://${uuid})`, start, end, 'select');
this.elNodeContent.setRangeText(`![${file.name}](db://${fileMetadata.UUID})`, start, end, 'select');
// Editing the textarea programatically doesn't generate the events it usually gets when edited interactively.
this.node.setContent(this.elNodeContent.value)

30
static/js/sync.mjs
View file

@ -164,6 +164,36 @@ export class Sync {
}
}
}//}}}
async filesToServer() {
try {
const BLOCKSIZE = 128*1024
let sentBytes = 0
const { file, metadata } = await nodeStore.nextFileToSync()
if (file === null)
return
const stream = file.stream()
const reader = stream.getReader({ mode: 'byob' }) // Bring Your Own Buffer
let buffer = new Uint8Array(BLOCKSIZE)
while (true) {
const { value, done } = await reader.read(buffer)
await API.upload(metadata.UUID, value, sentBytes, file)
metadata.setUploaded()
nodeStore.filesMetadata.add(metadata)
if (done)
break
sentBytes += value.length
buffer = new Uint8Array(value.buffer)
}
} catch (e) {
console.error(e)
alert(e.message)
}
}
}
export class N2SyncProgress extends CustomHTMLElement {