diff --git a/config.go b/config.go index 71d5f5d..8d89255 100644 --- a/config.go +++ b/config.go @@ -24,6 +24,10 @@ type Config struct { Secret string ExpireDays int } + + Storage struct { + Files string + } } func readConfig(fname string) (err error) { diff --git a/file.go b/file.go index 9ad8457..14a01d7 100644 --- a/file.go +++ b/file.go @@ -5,35 +5,46 @@ import ( "github.com/jmoiron/sqlx" // Standard + "fmt" + "os" + "path" "time" ) type File struct { ID int + UUID string UserID int `db:"user_id"` - NodeID int `db:"node_id"` Filename string - Size int64 + Size int MIME string - MD5 string - Uploaded time.Time + Modified time.Time } -func AddFile(userID int, file *File) (err error) { // {{{ +var ( + fileUUIDCache map[string]int +) + +func init() { + fileUUIDCache = make(map[string]int) +} + +func AddFileToDb(userID int, file *File) (err error) { // {{{ file.UserID = userID var rows *sqlx.Rows rows, err = db.Queryx(` - INSERT INTO file(user_id, node_id, filename, size, mime, md5) - VALUES($1, $2, $3, $4, $5, $6) + INSERT INTO public.file(user_id, uuid, name, size, type, modified, upload_complete) + VALUES($1, $2, $3, $4, $5, $6, false) + ON CONFLICT (user_id, uuid) DO UPDATE set name = public.file.name RETURNING id `, file.UserID, - file.NodeID, + file.UUID, file.Filename, file.Size, file.MIME, - file.MD5, + file.Modified, ) if err != nil { return @@ -76,5 +87,40 @@ func Files(userID int, nodeUUID string, fileID int) (files []File, err error) { return } // }}} +func FileIDToPath(id int) (string, string) { // {{{ + str := fmt.Sprintf("%012d", id) + dir := path.Join(config.Storage.Files, str[0:4], str[4:8]) + fpath := path.Join(config.Storage.Files, str[0:4], str[4:8], str[8:12]) + return dir, fpath +} // }}} + +func FileWriteData(userID int, dbID int, data []byte, create bool) (err error) {// {{{ + dir, fpath := FileIDToPath(dbID) + err = os.MkdirAll(dir, 0700) + if err != nil { + Log.Error("file", "error", err) + return + } + + var f *os.File + if create { + f, err = os.OpenFile(fpath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) + } else { + f, err = os.OpenFile(fpath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) + } + if err != nil { + Log.Error("file", "error", err) + return + } + defer f.Close() + + _, err = f.Write(data) + if err != nil { + Log.Error("file", "error", err) + return + } + + return +}// }}} // vim: foldmethod=marker diff --git a/main.go b/main.go index 6e3cf94..0a10092 100644 --- a/main.go +++ b/main.go @@ -5,7 +5,6 @@ import ( "notes2/authentication" "notes2/html_template" appUser "notes2/user" - "os" // Standard "bufio" @@ -17,11 +16,13 @@ import ( "io" "log/slog" "net/http" + "os" "path" "regexp" "strconv" "strings" "text/template" + "time" ) const VERSION = "v29" @@ -71,7 +72,7 @@ func init() { // {{{ } // }}} func initLog() { // {{{ opts := slog.HandlerOptions{} - opts.Level = slog.LevelDebug + opts.Level = slog.LevelInfo Log = slog.New(slog.NewJSONHandler(os.Stdout, &opts)) } // }}} func main() { // {{{ @@ -141,6 +142,7 @@ func main() { // {{{ http.HandleFunc("/sync/from_server/count/{sequence}", authenticated(actionSyncFromServerCount)) http.HandleFunc("/sync/from_server/{sequence}/{offset}", authenticated(actionSyncFromServer)) http.HandleFunc("/sync/to_server", authenticated(actionSyncToServer)) + http.HandleFunc("/sync/file/upload/{uuid}", authenticated(actionSyncFileUpload)) http.HandleFunc("/node/retrieve/{uuid}", authenticated(actionNodeRetrieve)) http.HandleFunc("/node/history/retrieve/{uuid}/{offset}", authenticated(actionNodeHistoryRetrieve)) @@ -391,6 +393,63 @@ func actionSyncToServer(w http.ResponseWriter, r *http.Request) { // {{{ "OK": true, }) } // }}} +func actionSyncFileUpload(w http.ResponseWriter, r *http.Request) { // {{{ + uuid := r.PathValue("uuid") + session := getUserSession(r) + + sentBytes, _ := strconv.Atoi(r.Header.Get("X-File-Sent-Bytes")) + totalBytes, _ := strconv.Atoi(r.Header.Get("X-File-Total-Bytes")) + + // First block comes with file metadata. + file := File{} + if sentBytes == 0 { + file.UUID = uuid + file.Filename = r.Header.Get("X-File-Name") + file.Size, _ = strconv.Atoi(r.Header.Get("X-File-Size")) + file.MIME = r.Header.Get("X-File-Type") + modded, _ := strconv.ParseInt(r.Header.Get("X-File-Modified"), 10, 64) + file.Modified = time.UnixMilli(modded) + err := AddFileToDb(session.UserID, &file) + if err != nil { + Log.Error("upload", "op", "AddFileToDB", "error", err) + httpError(w, err) + return + } + fileUUIDCache[uuid] = file.ID + } + + data, _ := io.ReadAll(r.Body) + var err error + + switch sentBytes { + case 0: + // First block of data. + Log.Info("file_upload", "status", "new", "user", session.UserID, "uuid", uuid, "total", totalBytes) + id := fileUUIDCache[uuid] + err = FileWriteData(session.UserID, id, data, true) + + case totalBytes: + // Last block of data - sent with 0 length data. + Log.Info("file_upload", "status", "done", "user", session.UserID, "uuid", uuid, "total", totalBytes) + id := fileUUIDCache[uuid] + delete(fileUUIDCache, uuid) + _, err = db.Exec(`UPDATE public.file SET upload_complete = true WHERE id = $1`, id) + + default: + id := fileUUIDCache[uuid] + err = FileWriteData(session.UserID, id, data, false) + } + + if err != nil { + Log.Error("upload", "error", err) + httpError(w, err) + return + } + + responseData(w, map[string]any{ + "OK": true, + }) +} // }}} func actionUserGetPreferences(w http.ResponseWriter, r *http.Request) { // {{{ user := getUserSession(r) diff --git a/sql/00011.sql b/sql/00011.sql new file mode 100644 index 0000000..f9efdbb --- /dev/null +++ b/sql/00011.sql @@ -0,0 +1,13 @@ +CREATE TABLE public.file ( + id serial NOT NULL, + user_id int4 NOT NULL, + uuid uuid NOT NULL, + name varchar DEFAULT '' NOT NULL, + "size" int DEFAULT 0 NOT NULL, + "type" varchar DEFAULT 'application/octet-stream' NOT NULL, + modified timestamptz DEFAULT NOW() NOT NULL, + CONSTRAINT file_pk PRIMARY KEY (id), + CONSTRAINT file_user_fk FOREIGN KEY (user_id) REFERENCES public."user"(id) ON DELETE RESTRICT ON UPDATE RESTRICT +); + +ALTER TABLE public.file ADD CONSTRAINT file_user_uuid_unique UNIQUE (user_id,"uuid"); diff --git a/sql/00012.sql b/sql/00012.sql new file mode 100644 index 0000000..422d2cf --- /dev/null +++ b/sql/00012.sql @@ -0,0 +1 @@ +ALTER TABLE public.file ADD upload_complete bool DEFAULT false NOT NULL; diff --git a/static/js/api.mjs b/static/js/api.mjs index 26a19de..01005e2 100644 --- a/static/js/api.mjs +++ b/static/js/api.mjs @@ -15,18 +15,62 @@ export class API { const res = await fetch(path, { method, headers, body }) // An HTTP communication level error occured. if (!res.ok || res.status != 200) - throw new Error('HTTP error', { cause: { type: 'http', error: res, }}) + throw new Error('HTTP error', { cause: { type: 'http', error: res, } }) // Application level response are handled here. const json = await res.json() if (!json.OK) - throw new Error(json.Error, { cause: { type: 'application', application: json, }}) + throw new Error(json.Error, { cause: { type: 'application', application: json, } }) return json } catch (err) { // Catch any other errors from fetch. - throw new Error(err.message, { cause: { type: 'http', error: err, }}) + throw new Error(err.message, { cause: { type: 'http', error: err, } }) + } + } + + // Sends a block of binary data to server. + static async upload(uuid, data, sentBytes, file) { + try { + const path = `/sync/file/upload/${uuid}` + const headers = { + 'X-File-Sent-Bytes': sentBytes, + 'X-File-Total-Bytes': file.size, + } + + // Metadata is needed for the database. + // No need to send it every block. + if (sentBytes === 0) { + headers['X-File-Name'] = file.name + headers['X-File-Size'] = file.size + headers['X-File-Type'] = file.type + headers['X-File-Modified'] = file.lastModified + } + + // Authentication is done with a bearer token. + // Here provided to the backend if set. + const token = localStorage.getItem('token') + if (token) { + headers.Authorization = `Bearer ${token}` + } + + const res = await fetch(path, { method: 'POST', headers, body: data }) + // An HTTP communication level error occured. + if (!res.ok || res.status != 200) + throw new Error('HTTP error', { cause: { type: 'http', error: res, } }) + + // Application level response are handled here. + const json = await res.json() + if (!json.OK) + throw new Error(json.Error, { cause: { type: 'application', application: json, } }) + + return json + + } catch (err) { + // Catch any other errors from fetch. + console.error(err) + throw new Error(err, { cause: { type: 'http', error: err } }) } } diff --git a/static/js/file.mjs b/static/js/file.mjs index 4322218..47e79d6 100644 --- a/static/js/file.mjs +++ b/static/js/file.mjs @@ -1,7 +1,7 @@ import { CustomHTMLElement } from "./lib/custom_html_element.mjs"; export class N2File extends CustomHTMLElement { - static { + static {// {{{ this.tmpl = document.createElement('template') this.tmpl.innerHTML = `