diff --git a/apps/dashboard-api/src/__tests__/routes.projects.storage.test.js b/apps/dashboard-api/src/__tests__/routes.projects.storage.test.js new file mode 100644 index 00000000..014b9752 --- /dev/null +++ b/apps/dashboard-api/src/__tests__/routes.projects.storage.test.js @@ -0,0 +1,117 @@ +'use strict'; + +jest.mock('../middlewares/authMiddleware', () => + jest.fn((req, _res, next) => { + req.user = { _id: 'dev1' }; + next(); + }), +); + +jest.mock('../middlewares/planEnforcement', () => ({ + attachDeveloper: jest.fn((_req, _res, next) => next()), + checkProjectLimit: jest.fn((_req, _res, next) => next()), + checkCollectionLimit: jest.fn((_req, _res, next) => next()), + checkByokGate: jest.fn((_req, _res, next) => next()), +})); + +jest.mock('@urbackend/common', () => ({ + verifyEmail: jest.fn((_req, _res, next) => next()), + checkAuthEnabled: jest.fn((_req, _res, next) => next()), + loadProjectForAdmin: jest.fn((_req, _res, next) => next()), +})); + +jest.mock('../controllers/userAuth.controller', () => ({ + createAdminUser: jest.fn((_req, res) => res.json({ ok: true })), + resetPassword: jest.fn((_req, res) => res.json({ ok: true })), + getUserDetails: jest.fn((_req, res) => res.json({ ok: true })), + updateAdminUser: jest.fn((_req, res) => res.json({ ok: true })), + listUserSessions: jest.fn((_req, res) => res.json({ ok: true })), + revokeUserSession: jest.fn((_req, res) => res.json({ ok: true })), +})); + +jest.mock('../controllers/project.controller', () => { + const ok = (_req, res) => res.json({ ok: true }); + return { + createProject: jest.fn(ok), + getAllProject: jest.fn(ok), + getSingleProject: jest.fn(ok), + regenerateApiKey: jest.fn(ok), + createCollection: jest.fn(ok), + deleteCollection: jest.fn(ok), + getData: jest.fn(ok), + deleteRow: jest.fn(ok), + insertData: jest.fn(ok), + editRow: jest.fn(ok), + listFiles: jest.fn(ok), + deleteFile: jest.fn(ok), + deleteAllFiles: jest.fn(ok), + deleteProject: jest.fn(ok), + updateProject: jest.fn(ok), + updateExternalConfig: jest.fn(ok), + deleteExternalDbConfig: jest.fn(ok), + deleteExternalStorageConfig: jest.fn(ok), + analytics: jest.fn(ok), + updateAllowedDomains: jest.fn(ok), + toggleAuth: jest.fn(ok), + updateAuthProviders: jest.fn(ok), + updateCollectionRls: jest.fn(ok), + listMailTemplates: jest.fn(ok), + listGlobalMailTemplates: jest.fn(ok), + getMailTemplate: jest.fn(ok), + createMailTemplate: jest.fn(ok), + updateMailTemplate: jest.fn(ok), + deleteMailTemplate: jest.fn(ok), + requestUpload: jest.fn(ok), + confirmUpload: jest.fn(ok), + }; +}); + +const express = require('express'); +const request = require('supertest'); +const projectsRouter = require('../routes/projects'); +const projectController = require('../controllers/project.controller'); +const authMiddleware = require('../middlewares/authMiddleware'); +const { verifyEmail, loadProjectForAdmin } = require('@urbackend/common'); + +let app; + +beforeEach(() => { + jest.clearAllMocks(); + app = express(); + app.use(express.json()); + app.use('/api/projects', projectsRouter); +}); + +describe('projects storage presigned routes', () => { + test('legacy proxy upload route is removed', async () => { + const res = await request(app) + .post('/api/projects/project1/storage/upload') + .send({}); + + expect(res.status).toBe(404); + }); + + test('upload-request route is wired and protected', async () => { + const res = await request(app) + .post('/api/projects/project1/storage/upload-request') + .send({ filename: 'a.txt', contentType: 'text/plain', size: 10 }); + + expect(res.status).toBe(200); + expect(authMiddleware).toHaveBeenCalled(); + expect(verifyEmail).toHaveBeenCalled(); + expect(loadProjectForAdmin).toHaveBeenCalled(); + expect(projectController.requestUpload).toHaveBeenCalledTimes(1); + }); + + test('upload-confirm route is wired and protected', async () => { + const res = await request(app) + .post('/api/projects/project1/storage/upload-confirm') + .send({ filePath: 'project1/a.txt', size: 10 }); + + expect(res.status).toBe(200); + expect(authMiddleware).toHaveBeenCalled(); + expect(verifyEmail).toHaveBeenCalled(); + expect(loadProjectForAdmin).toHaveBeenCalled(); + expect(projectController.confirmUpload).toHaveBeenCalledTimes(1); + }); +}); diff --git a/apps/dashboard-api/src/__tests__/storage.presigned.controller.test.js b/apps/dashboard-api/src/__tests__/storage.presigned.controller.test.js new file mode 100644 index 00000000..54145c8b --- /dev/null +++ b/apps/dashboard-api/src/__tests__/storage.presigned.controller.test.js @@ -0,0 +1,315 @@ +'use strict'; + +jest.mock('crypto', () => { + const actual = jest.requireActual('crypto'); + return { + ...actual, + randomUUID: jest.fn(() => 'mocked-uuid'), + }; +}); + +jest.mock('@urbackend/common', () => { + class AppError extends Error { + constructor(statusCode, message) { + super(message); + this.statusCode = statusCode; + this.status = `${statusCode}`.startsWith('4') ? 'fail' : 'error'; + } + } + + const mockStorageFrom = { + getPublicUrl: jest.fn(), + remove: jest.fn(), + }; + + const mockSupabaseStorage = { + from: jest.fn(() => mockStorageFrom), + }; + + return { + Project: { + findOne: jest.fn(), + updateOne: jest.fn(), + }, + getStorage: jest.fn(() => ({ + storage: mockSupabaseStorage, + })), + getPresignedUploadUrl: jest.fn(), + verifyUploadedFile: jest.fn(), + isProjectStorageExternal: jest.fn(), + getBucket: jest.fn(() => 'dev-files'), + sanitizeObjectId: jest.fn((value) => { + if (typeof value !== 'string') return null; + const normalized = value.trim(); + return /^[a-fA-F0-9]{24}$/.test(normalized) ? normalized : null; + }), + sanitizeNonEmptyString: jest.fn((value, options = {}) => { + if (typeof value !== 'string') return null; + const normalized = value.trim(); + if (!normalized) return null; + if (normalized.length > (options.maxLength || 1024)) return null; + return normalized; + }), + AppError, + __mockStorageFrom: mockStorageFrom, + }; +}); + +const { + Project, + getPresignedUploadUrl, + verifyUploadedFile, + isProjectStorageExternal, + __mockStorageFrom: mockStorageFrom, +} = require('@urbackend/common'); + +const controller = require('../controllers/project.controller'); +const PROJECT_ID = '507f1f77bcf86cd799439011'; + +const makeRes = () => { + const res = { status: jest.fn(), json: jest.fn() }; + res.status.mockReturnValue(res); + res.json.mockReturnValue(res); + return res; +}; + +const makeNext = () => jest.fn(); + +const makeProject = (overrides = {}) => ({ + _id: PROJECT_ID, + owner: 'dev1', + storageUsed: 100, + storageLimit: 1024 * 1024, + resources: { storage: { isExternal: false } }, + ...overrides, +}); + +const mockFindOneSelect = (project) => { + Project.findOne.mockReturnValue({ + select: jest.fn().mockResolvedValue(project), + }); +}; + +describe('dashboard project.controller presigned upload', () => { + beforeEach(() => { + jest.clearAllMocks(); + process.env.NODE_ENV = 'test'; + }); + + describe('requestUpload', () => { + test('returns 400 for invalid input', async () => { + const req = { + params: { projectId: PROJECT_ID }, + body: { filename: 'a.txt', contentType: 'text/plain', size: 'abc' }, + user: { _id: 'dev1' }, + }; + const res = makeRes(); + const next = makeNext(); + + await controller.requestUpload(req, res, next); + + expect(next).toHaveBeenCalledWith(expect.objectContaining({ + statusCode: 400, + message: 'projectId, filename, contentType, and size are required.', + })); + }); + + test('returns 403 when internal quota is exceeded', async () => { + mockFindOneSelect(makeProject({ storageUsed: 900, storageLimit: 1000 })); + isProjectStorageExternal.mockReturnValue(false); + + const req = { + params: { projectId: PROJECT_ID }, + body: { filename: 'a.txt', contentType: 'text/plain', size: 200 }, + user: { _id: 'dev1' }, + }; + const res = makeRes(); + const next = makeNext(); + + await controller.requestUpload(req, res, next); + + expect(next).toHaveBeenCalledWith(expect.objectContaining({ + statusCode: 403, + message: 'Internal storage limit exceeded.', + })); + }); + + test('returns signed URL payload on success', async () => { + mockFindOneSelect(makeProject()); + isProjectStorageExternal.mockReturnValue(false); + getPresignedUploadUrl.mockResolvedValue({ + signedUrl: 'https://signed.example/upload', + token: 't1', + }); + + const req = { + params: { projectId: PROJECT_ID }, + body: { filename: 'my file.txt', contentType: 'text/plain', size: 1234 }, + user: { _id: 'dev1' }, + }; + const res = makeRes(); + const next = makeNext(); + + await controller.requestUpload(req, res, next); + + expect(getPresignedUploadUrl).toHaveBeenCalledWith( + expect.objectContaining({ _id: PROJECT_ID }), + `${PROJECT_ID}/mocked-uuid_my_file.txt`, + 'text/plain', + 1234, + ); + expect(res.status).toHaveBeenCalledWith(200); + expect(res.json).toHaveBeenCalledWith({ + success: true, + data: { + signedUrl: 'https://signed.example/upload', + token: 't1', + filePath: `${PROJECT_ID}/mocked-uuid_my_file.txt`, + }, + message: 'Upload URL generated successfully.', + }); + expect(next).not.toHaveBeenCalled(); + }); + }); + + describe('confirmUpload', () => { + test('returns 403 when project path does not match', async () => { + mockFindOneSelect(makeProject()); + isProjectStorageExternal.mockReturnValue(false); + + const req = { + params: { projectId: PROJECT_ID }, + body: { filePath: '507f1f77bcf86cd799439012/file.txt', size: 200 }, + user: { _id: 'dev1' }, + }; + const res = makeRes(); + const next = makeNext(); + + await controller.confirmUpload(req, res, next); + + expect(next).toHaveBeenCalledWith(expect.objectContaining({ + statusCode: 403, + message: 'Access denied.', + })); + }); + + test('returns 409 when uploaded file is not visible yet', async () => { + mockFindOneSelect(makeProject()); + isProjectStorageExternal.mockReturnValue(false); + verifyUploadedFile.mockRejectedValue(new Error('File not found after upload')); + mockStorageFrom.remove.mockResolvedValue({ data: null, error: null }); + + const req = { + params: { projectId: PROJECT_ID }, + body: { filePath: `${PROJECT_ID}/file.txt`, size: 200 }, + user: { _id: 'dev1' }, + }; + const res = makeRes(); + const next = makeNext(); + + await controller.confirmUpload(req, res, next); + + expect(mockStorageFrom.remove).toHaveBeenCalledWith([`${PROJECT_ID}/file.txt`]); + expect(next).toHaveBeenCalledWith(expect.objectContaining({ + statusCode: 409, + message: 'Uploaded file is not visible yet. Please retry confirmation.', + })); + }); + + test('returns 400 when declared size mismatches actual size', async () => { + mockFindOneSelect(makeProject()); + isProjectStorageExternal.mockReturnValue(false); + verifyUploadedFile.mockResolvedValue(1024); + mockStorageFrom.remove.mockResolvedValue({ data: null, error: null }); + + const req = { + params: { projectId: PROJECT_ID }, + body: { filePath: `${PROJECT_ID}/file.txt`, size: 900 }, + user: { _id: 'dev1' }, + }; + const res = makeRes(); + const next = makeNext(); + + await controller.confirmUpload(req, res, next); + + expect(next).toHaveBeenCalledWith(expect.objectContaining({ + statusCode: 400, + message: 'Declared file size does not match uploaded file size.', + })); + expect(mockStorageFrom.remove).toHaveBeenCalledWith([`${PROJECT_ID}/file.txt`]); + }); + + test('charges quota and returns success on internal storage', async () => { + mockFindOneSelect(makeProject()); + isProjectStorageExternal.mockReturnValue(false); + verifyUploadedFile.mockResolvedValue(1024); + Project.updateOne.mockResolvedValue({ matchedCount: 1 }); + mockStorageFrom.getPublicUrl.mockReturnValue({ data: { publicUrl: 'https://cdn.example/p/project1/file.txt' } }); + + const req = { + params: { projectId: PROJECT_ID }, + body: { filePath: `${PROJECT_ID}/file.txt`, size: 1024 }, + user: { _id: 'dev1' }, + }; + const res = makeRes(); + const next = makeNext(); + + await controller.confirmUpload(req, res, next); + + expect(Project.updateOne).toHaveBeenCalledWith( + { + _id: PROJECT_ID, + $or: [ + { storageLimit: -1 }, + { $expr: { $lte: [{ $add: ['$storageUsed', 1024] }, '$storageLimit'] } }, + ], + }, + { $inc: { storageUsed: 1024 } }, + ); + expect(res.status).toHaveBeenCalledWith(200); + expect(res.json).toHaveBeenCalledWith({ + success: true, + data: { + message: 'Upload confirmed', + path: `${PROJECT_ID}/file.txt`, + provider: 'internal', + url: 'https://cdn.example/p/project1/file.txt', + }, + message: 'Upload confirmed.', + }); + expect(next).not.toHaveBeenCalled(); + }); + + test('skips quota charge for external storage', async () => { + mockFindOneSelect(makeProject({ resources: { storage: { isExternal: true } } })); + isProjectStorageExternal.mockReturnValue(true); + verifyUploadedFile.mockResolvedValue(512); + mockStorageFrom.getPublicUrl.mockReturnValue({ data: { publicUrl: null } }); + + const req = { + params: { projectId: PROJECT_ID }, + body: { filePath: `${PROJECT_ID}/file.txt`, size: 512 }, + user: { _id: 'dev1' }, + }; + const res = makeRes(); + const next = makeNext(); + + await controller.confirmUpload(req, res, next); + + expect(Project.updateOne).not.toHaveBeenCalled(); + expect(res.status).toHaveBeenCalledWith(200); + expect(res.json).toHaveBeenCalledWith({ + success: true, + data: { + message: 'Upload confirmed', + path: `${PROJECT_ID}/file.txt`, + provider: 'external', + url: null, + warning: 'Upload confirmed, but a public URL is unavailable.', + }, + message: 'Upload confirmed.', + }); + expect(next).not.toHaveBeenCalled(); + }); + }); +}); diff --git a/apps/dashboard-api/src/controllers/project.controller.js b/apps/dashboard-api/src/controllers/project.controller.js index 1029dd15..e7d9b2c2 100644 --- a/apps/dashboard-api/src/controllers/project.controller.js +++ b/apps/dashboard-api/src/controllers/project.controller.js @@ -9,15 +9,19 @@ const { createCollectionSchema, updateExternalConfigSchema, updateAuthProvidersSchema, + sanitizeObjectId, + sanitizeNonEmptyString, } = require("@urbackend/common"); const { generateApiKey, hashApiKey } = require("@urbackend/common"); const { z } = require("zod"); const { encrypt } = require("@urbackend/common"); const { URL } = require("url"); +const path = require("path"); const { getConnection } = require("@urbackend/common"); const { getCompiledModel } = require("@urbackend/common"); const { QueryEngine } = require("@urbackend/common"); const { storageRegistry } = require("@urbackend/common"); +const { AppError } = require("@urbackend/common"); const { deleteProjectByApiKeyCache, setProjectById, @@ -25,10 +29,16 @@ const { deleteProjectById, } = require("@urbackend/common"); const { isProjectStorageExternal, getBucket } = require("@urbackend/common"); +const { getPresignedUploadUrl } = require("@urbackend/common"); +const { verifyUploadedFile } = require("@urbackend/common"); const { getPublicIp } = require("@urbackend/common"); const { clearCompiledModel } = require("@urbackend/common"); const { createUniqueIndexes } = require("@urbackend/common"); +const MAX_FILE_SIZE = 10 * 1024 * 1024; +const SAFETY_MAX_BYTES = 100 * 1024 * 1024; +const CONFIRM_UPLOAD_SIZE_TOLERANCE_BYTES = 64; + const validateUsersSchema = (schema) => { if (!Array.isArray(schema)) return false; @@ -185,6 +195,54 @@ const sanitizeProjectResponse = (projectObj) => { return projectObj; }; +const parsePositiveSize = (size) => { + const numericSize = Number(size); + if (!Number.isFinite(numericSize) || numericSize <= 0) { + return null; + } + return numericSize; +}; + +const normalizeProjectPath = (projectId, inputPath) => { + if (typeof inputPath !== "string") { + return null; + } + + let decodedPath = inputPath; + try { + decodedPath = decodeURIComponent(inputPath); + } catch { + return null; + } + + const normalizedPath = path.posix.normalize(decodedPath).replace(/^\/+/, ""); + const segments = normalizedPath.split("/").filter(Boolean); + + if (segments.length < 2) { + return null; + } + + if (segments[0] !== String(projectId)) { + return null; + } + + if (segments.some((segment) => segment === "." || segment === "..")) { + return null; + } + + return normalizedPath; +}; + +const bestEffortDeleteUploadedObject = async (project, filePath) => { + try { + const supabase = await getStorage(project); + const bucket = getBucket(project); + await supabase.storage.from(bucket).remove([filePath]); + } catch { + // ignore cleanup failures; the primary response should still be returned + } +}; + module.exports.createProject = async (req, res) => { const session = await mongoose.startSession(); session.startTransaction(); @@ -197,7 +255,7 @@ module.exports.createProject = async (req, res) => { if (req.projectLimit !== undefined) { const currentCount = await Project.countDocuments( { owner: req.user._id }, - { session } + { session }, ); if (currentCount >= req.projectLimit) { @@ -245,6 +303,7 @@ module.exports.createProject = async (req, res) => { if (err instanceof z.ZodError) { return res.status(400).json({ error: err.issues }); } + res.status(500).json({ error: err.message }); } }; @@ -667,7 +726,6 @@ module.exports.createCollection = async (req, res) => { } catch (err) { await session.abortTransaction(); session.endSession(); - try { if (connection && compiledCollectionName) { clearCompiledModel(connection, compiledCollectionName); @@ -1021,55 +1079,6 @@ module.exports.listFiles = async (req, res) => { } }; -module.exports.uploadFile = async (req, res) => { - try { - const { projectId } = req.params; - const file = req.file; - - if (!file) return res.status(400).json({ error: "No file uploaded" }); - - const project = await Project.findOne({ - _id: projectId, - owner: req.user._id, - }).select( - "+resources.storage.config.encrypted +resources.storage.config.iv +resources.storage.config.tag resources.storage.isExternal storageUsed storageLimit", - ); - if (!project) return res.status(404).json({ error: "Project not found" }); - - const external = isProjectStorageExternal(project); - - if (!external) { - if (project.storageUsed + file.size > project.storageLimit) { - return res.status(403).json({ error: "Storage limit exceeded" }); - } - } - - const supabase = await getStorage(project); - const bucket = getBucket(project); - - const safeName = file.originalname.replace(/\s+/g, "_"); - const path = `${projectId}/${randomUUID()}_${safeName}`; - - const { error } = await supabase.storage - .from(bucket) - .upload(path, file.buffer, { - contentType: file.mimetype, - upsert: false, - }); - - if (error) throw error; - - if (!external) { - project.storageUsed += file.size; - await project.save(); - } - - res.json({ success: true, path }); - } catch (err) { - res.status(500).json({ error: err }); - } -}; - module.exports.deleteFile = async (req, res) => { try { const { projectId } = req.params; @@ -1164,6 +1173,184 @@ module.exports.deleteAllFiles = async (req, res) => { } }; +module.exports.requestUpload = async (req, res, next) => { + try { + const projectId = sanitizeObjectId(req.params?.projectId); + const { filename, contentType, size } = req.body; + const sanitizedFilename = sanitizeNonEmptyString(filename, { + maxLength: 255, + }); + const sanitizedContentType = sanitizeNonEmptyString(contentType, { + maxLength: 255, + }); + const numericSize = parsePositiveSize(size); + + if (!projectId || !sanitizedFilename || !sanitizedContentType || numericSize === null) { + return next( + new AppError(400, "projectId, filename, contentType, and size are required."), + ); + } + + if (numericSize > MAX_FILE_SIZE) { + return next(new AppError(413, "File size exceeds limit.")); + } + + const project = await Project.findOne({ + _id: projectId, + owner: req.user._id, + }).select( + "+resources.storage.config.encrypted +resources.storage.config.iv +resources.storage.config.tag resources.storage.isExternal storageUsed storageLimit", + ); + + if (!project) return next(new AppError(404, "Project not found")); + + const external = isProjectStorageExternal(project); + + // Pre-check quota only; actual storage usage is charged after confirmUpload verifies object existence and size. + if (!external) { + const storageLimit = + typeof project.storageLimit === "number" + ? project.storageLimit + : 20 * 1024 * 1024; + const quotaLimit = + storageLimit === -1 ? SAFETY_MAX_BYTES : storageLimit; + + if ((project.storageUsed || 0) + numericSize > quotaLimit) { + return next(new AppError(403, "Internal storage limit exceeded.")); + } + } + + const safeName = sanitizedFilename.replace(/\s+/g, "_"); + const filePath = `${projectId}/${randomUUID()}_${safeName}`; + + const { signedUrl, token } = await getPresignedUploadUrl( + project, + filePath, + sanitizedContentType, + numericSize, + ); + + return res.status(200).json({ + success: true, + data: { signedUrl, token, filePath }, + message: "Upload URL generated successfully.", + }); + } catch (err) { + if (err instanceof AppError) return next(err); + return next(new AppError(500, "Could not generate upload URL")); + } +}; + +module.exports.confirmUpload = async (req, res, next) => { + try { + const projectId = sanitizeObjectId(req.params?.projectId); + const { filePath, size } = req.body; + const sanitizedFilePath = sanitizeNonEmptyString(filePath, { + maxLength: 1024, + }); + const declaredSize = parsePositiveSize(size); + + if (!projectId || !sanitizedFilePath || declaredSize === null) { + return next(new AppError(400, "projectId, filePath, and size are required.")); + } + + const project = await Project.findOne({ + _id: projectId, + owner: req.user._id, + }).select( + "+resources.storage.config.encrypted +resources.storage.config.iv +resources.storage.config.tag resources.storage.isExternal storageUsed storageLimit", + ); + + if (!project) return next(new AppError(404, "Project not found")); + + const external = isProjectStorageExternal(project); + const normalizedPath = normalizeProjectPath(projectId, sanitizedFilePath); + + // make sure client isn't confirming someone else's file + if (!normalizedPath) { + return next(new AppError(403, "Access denied.")); + } + + // verify file actually exists on cloud before touching quota + let actualSize; + try { + actualSize = await verifyUploadedFile(project, normalizedPath); + } catch (err) { + if (err?.message === "File not found after upload") { + await bestEffortDeleteUploadedObject(project, normalizedPath); + return next( + new AppError( + 409, + "Uploaded file is not visible yet. Please retry confirmation.", + ), + ); + } + throw err; + } + + if (!Number.isFinite(actualSize) || actualSize <= 0) { + await bestEffortDeleteUploadedObject(project, normalizedPath); + return next(new AppError(500, "Uploaded file size could not be determined")); + } + + if (Math.abs(actualSize - declaredSize) > CONFIRM_UPLOAD_SIZE_TOLERANCE_BYTES) { + await bestEffortDeleteUploadedObject(project, normalizedPath); + return next( + new AppError(400, "Declared file size does not match uploaded file size."), + ); + } + + // now it's safe to charge quota + if (!external) { + const result = await Project.updateOne( + { + _id: project._id, + $or: [ + { storageLimit: -1 }, + { $expr: { $lte: [{ $add: ["$storageUsed", actualSize] }, "$storageLimit"] } }, + ], + }, + { $inc: { storageUsed: actualSize } }, + ); + + if (result.matchedCount === 0) { + await bestEffortDeleteUploadedObject(project, normalizedPath); + return next(new AppError(403, "Internal storage limit exceeded.")); + } + } + + const supabase = await getStorage(project); + const bucket = getBucket(project); + const { data: publicUrlData, error: apiError } = supabase.storage + .from(bucket) + .getPublicUrl(normalizedPath); + + const publicUrl = publicUrlData?.publicUrl; + const provider = publicUrl ? (external ? "external" : "internal") : "external"; + + const responseData = { + message: "Upload confirmed", + path: normalizedPath, + provider, + url: publicUrl ?? null, + }; + + if (!publicUrl) { + responseData.warning = + apiError || "Upload confirmed, but a public URL is unavailable."; + } + + return res.status(200).json({ + success: true, + data: responseData, + message: "Upload confirmed.", + }); + } catch (err) { + if (err instanceof AppError) return next(err); + return next(new AppError(500, "Upload confirmation failed")); + } +}; + module.exports.updateProject = async (req, res) => { try { const { name, siteUrl, resendApiKey, resendFromEmail } = req.body; diff --git a/apps/dashboard-api/src/routes/projects.js b/apps/dashboard-api/src/routes/projects.js index 301a45cb..dbcae35d 100644 --- a/apps/dashboard-api/src/routes/projects.js +++ b/apps/dashboard-api/src/routes/projects.js @@ -2,9 +2,7 @@ const express = require('express'); const router = express.Router(); const authMiddleware = require('../middlewares/authMiddleware'); const { attachDeveloper, checkProjectLimit, checkCollectionLimit, checkByokGate } = require('../middlewares/planEnforcement'); -const {verifyEmail} = require('@urbackend/common') -const multer = require('multer'); -const storage = multer.memoryStorage(); +const { verifyEmail, checkAuthEnabled, loadProjectForAdmin } = require('@urbackend/common'); const { createProject, @@ -17,7 +15,6 @@ const { deleteRow, insertData, editRow, - uploadFile, listFiles, deleteFile, deleteAllFiles, @@ -36,13 +33,13 @@ const { getMailTemplate, createMailTemplate, updateMailTemplate, - deleteMailTemplate + deleteMailTemplate, + requestUpload, + confirmUpload } = require("../controllers/project.controller") const { createAdminUser, resetPassword, getUserDetails, updateAdminUser, listUserSessions, revokeUserSession } = require('../controllers/userAuth.controller'); -const upload = multer({ storage: storage, limits: { fileSize: 10 * 1024 * 1024 } }); // 10MB Limit - // POST REQ FOR CREATE PROJECT router.post('/', authMiddleware, attachDeveloper, verifyEmail, checkProjectLimit, createProject); @@ -74,12 +71,14 @@ router.patch('/:projectId/collections/:collectionName/data/:id', authMiddleware, // GET REQ FOR FILES router.get('/:projectId/storage/files', authMiddleware, listFiles); -// POST REQ FOR UPLOAD FILE -router.post('/:projectId/storage/upload', authMiddleware, verifyEmail, upload.single('file'), uploadFile); - // POST REQ FOR DELETE FILE router.post('/:projectId/storage/delete', authMiddleware, verifyEmail, deleteFile); +//SIGNED URL +router.post('/:projectId/storage/upload-request', authMiddleware, verifyEmail, loadProjectForAdmin, requestUpload); +//UPLOAD URL +router.post('/:projectId/storage/upload-confirm', authMiddleware, verifyEmail, loadProjectForAdmin, confirmUpload); + // DELETE REQ FOR PROJECT router.delete('/:projectId', authMiddleware, verifyEmail, deleteProject); @@ -125,8 +124,7 @@ router.patch('/:projectId/auth/providers', authMiddleware, verifyEmail, updateAu router.patch('/:projectId/collections/:collectionName/rls', authMiddleware, verifyEmail, updateCollectionRls); // ADMIN AUTH ROUTES -const {checkAuthEnabled} = require('@urbackend/common'); -const {loadProjectForAdmin} = require('@urbackend/common'); + router.post('/:projectId/admin/users', authMiddleware, loadProjectForAdmin, checkAuthEnabled, createAdminUser); router.patch('/:projectId/admin/users/:userId/password', authMiddleware, loadProjectForAdmin, checkAuthEnabled, resetPassword); diff --git a/apps/web-dashboard/src/pages/Storage.jsx b/apps/web-dashboard/src/pages/Storage.jsx index 61c303fb..afbf50ba 100644 --- a/apps/web-dashboard/src/pages/Storage.jsx +++ b/apps/web-dashboard/src/pages/Storage.jsx @@ -39,18 +39,45 @@ export default function Storage() { toast.error("Account Verification Required. Please verify in Settings."); return; } - const formData = new FormData(); - formData.append('file', file); + setUploading(true); const toastId = toast.loading("Uploading..."); + try { - await api.post(`/api/projects/${projectId}/storage/upload`, formData, { - headers: { 'Content-Type': 'multipart/form-data' } + const requestRes = await api.post(`/api/projects/${projectId}/storage/upload-request`, { + filename: file.name, + contentType: file.type || 'application/octet-stream', + size: file.size, + }); + + const signedUrl = requestRes?.data?.data?.signedUrl; + const filePath = requestRes?.data?.data?.filePath; + + if (!signedUrl || !filePath) { + throw new Error('Could not get upload URL'); + } + + const uploadRes = await fetch(signedUrl, { + method: 'PUT', + headers: { + 'Content-Type': file.type || 'application/octet-stream', + }, + body: file, }); + + if (!uploadRes.ok) { + throw new Error('Direct upload failed'); + } + + await api.post(`/api/projects/${projectId}/storage/upload-confirm`, { + filePath, + size: file.size, + }); + toast.success("File uploaded!", { id: toastId }); fetchFiles(); } catch (err) { - toast.error(err.response?.data?.error || "Upload failed", { id: toastId }); + toast.error(err.response?.data?.message || err.message || "Upload failed", { id: toastId }); } finally { setUploading(false); if (fileInputRef.current) fileInputRef.current.value = ''; diff --git a/docs-legacy/storage.md b/docs-legacy/storage.md index 467b4351..2dd5cd39 100644 --- a/docs-legacy/storage.md +++ b/docs-legacy/storage.md @@ -66,3 +66,36 @@ If `path` is invalid or already removed, API returns `404`. - `400 Bad Request`: usually missing `file` in multipart form. - `401 Unauthorized`: missing/invalid API key. - `413 Payload Too Large`: file exceeds max size limit. + +## Presigned Upload (Dashboard/Public API) + +For the newer upload flow, file bytes are uploaded directly from the browser to storage. + +Typical flow: + +1. Call backend `POST /api/storage/upload-request` with `filename`, `contentType`, and `size`. +2. Receive `signedUrl` and `filePath`. +3. Browser uploads file to `signedUrl` using `PUT`. +4. Call backend `POST /api/storage/upload-confirm` to verify file existence and charge quota. + +This avoids proxying file bytes through Node.js and keeps server memory usage predictable. + +## Required Bucket CORS For S3/R2 + +If using AWS S3 or Cloudflare R2 with presigned browser uploads, bucket CORS must allow the dashboard origin. + +Required methods: + +- `PUT` +- `OPTIONS` +- `GET` +- `HEAD` + +Required headers should include at least: + +- `content-type` +- `content-length` + +Browser `PUT` requests to `signedUrl` should send the file body and include the correct `Content-Type`, and `Content-Length` when your client/runtime allows explicitly setting it. + +If preflight CORS is missing or restrictive, browser uploads will fail even when the signed URL is valid. diff --git a/mintlify/docs/guides/storage.mdx b/mintlify/docs/guides/storage.mdx index 2c9f4f37..09532911 100644 --- a/mintlify/docs/guides/storage.mdx +++ b/mintlify/docs/guides/storage.mdx @@ -51,7 +51,7 @@ Uploads use a presigned URL three-step flow so the binary is sent directly to st ### Step 2 — Upload the binary to storage -Send a `PUT` request to `signedUrl` with the raw file contents and the correct `Content-Type` header. +Send a browser `PUT` request to `signedUrl` with the raw file contents and the correct `Content-Type` header. ```javascript const uploadRes = await fetch(signedUrl, { @@ -181,6 +181,24 @@ console.log(url, provider, path, warning); SDK users (`urbackend-sdk`) do not need to change anything — `client.storage.upload()` uses this flow internally. +## Required bucket CORS (S3/R2) + +If you use AWS S3 or Cloudflare R2 with presigned browser uploads, configure bucket CORS so browser uploads to `signedUrl` can pass preflight. + +Required methods: + +- `PUT` +- `OPTIONS` +- `GET` +- `HEAD` + +Required `AllowedHeaders` (not `ExposeHeaders`) should include at least: + +- `content-type` — required for presigned `PUT` requests +- `content-length` — typically safelisted, but some providers still require it to be allow-listed + +Without these CORS rules, browser uploads can fail even when `POST /api/storage/upload-request` and `POST /api/storage/upload-confirm` are correct. + ## Delete a file To delete a file, pass the `path` returned from the upload response. @@ -225,6 +243,6 @@ If the path is invalid or the file has already been removed, the API returns `40 - If you use an external S3/R2 bucket, configure its CORS policy to allow `PUT` requests from your client origin. The presigned URL flow requires this bucket-level setting. + If you use an external S3/R2 bucket, configure bucket CORS to allow `PUT`, `OPTIONS`, `GET`, and `HEAD`, and allow `content-type` and `content-length` headers from your client origin. diff --git a/packages/common/src/index.js b/packages/common/src/index.js index 3c420201..640e34fb 100644 --- a/packages/common/src/index.js +++ b/packages/common/src/index.js @@ -66,6 +66,8 @@ const { createWebhookSchema, updateWebhookSchema, sendMailSchema, + sanitizeObjectId, + sanitizeNonEmptyString, } = require("./utils/input.validation"); const { garbageCollect, storageGarbageCollect } = require("./utils/GC"); const { generateApiKey, hashApiKey } = require("./utils/api"); @@ -130,6 +132,8 @@ module.exports = { createWebhookSchema, updateWebhookSchema, sendMailSchema, + sanitizeObjectId, + sanitizeNonEmptyString, garbageCollect, storageGarbageCollect, generateApiKey, diff --git a/packages/common/src/utils/input.validation.js b/packages/common/src/utils/input.validation.js index a61b307e..f25ee716 100644 --- a/packages/common/src/utils/input.validation.js +++ b/packages/common/src/utils/input.validation.js @@ -1,4 +1,5 @@ const z = require("zod"); +const mongoose = require("mongoose"); const { MAX_FIELD_DEPTH, UNIQUE_SUPPORTED_TYPES, @@ -294,6 +295,26 @@ module.exports.sanitize = (obj) => { return clean; }; +module.exports.sanitizeObjectId = (value) => { + if (typeof value !== "string") return null; + const normalized = value.trim(); + if (!normalized) return null; + return mongoose.Types.ObjectId.isValid(normalized) ? normalized : null; +}; + +module.exports.sanitizeNonEmptyString = (value, options = {}) => { + if (typeof value !== "string") return null; + + const { maxLength = 1024, allowNullByte = false } = options; + const normalized = value.trim(); + + if (!normalized) return null; + if (normalized.length > maxLength) return null; + if (!allowNullByte && normalized.includes("\0")) return null; + + return normalized; +}; + const emptyToUndefined = z.preprocess( (val) => (val === "" || val === null ? undefined : val), z.string().optional(),