STEMMechanics/Website
1
0
Fork 0
Code Issues 4 Pull Requests 1 Actions Packages Projects Releases 3 Wiki Activity

fixes to uploading in chunks

Browse Source
This commit is contained in:
James Collins
2023-09-10 19:59:26 +10:00
parent 3e6a9da0d1
commit 3f915d6964
8 changed files with 182 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
app/Conductors/Conductor.php
View File

@@ -593,7 +593,7 @@ class Conductor
$requestIncludes = [];
$modelFields = $conductor->fields(new $conductor->class());
// Limit fields
$limitFields = $modelFields;
if ($fields instanceof Request) {

1
app/Conductors/MediaJobConductor.php
View File

@@ -2,6 +2,7 @@
namespace App\Conductors;
use App\Models\MediaJob;
use App\Models\User;
use Illuminate\Database\Eloquent\Builder;
use Illuminate\Database\Eloquent\Model;

2
app/Providers/RouteServiceProvider.php
View File

@@ -42,7 +42,7 @@ class RouteServiceProvider extends ServiceProvider
if ($rateLimitEnabled === true) {
RateLimiter::for('api', function (Request $request) {
return Limit::perMinute(180)->by($request->user()?->id ?: $request->ip());
return Limit::perMinute(800)->by($request->user()?->id ?: $request->ip());
});
} else {
RateLimiter::for('api', function () {

28
database/migrations/2023_09_10_085850_add_progress_max_to_media_jobs_table.php Normal file
View File

@@ -0,0 +1,28 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
/**
* Run the migrations.
*/
public function up(): void
{
Schema::table('media_jobs', function (Blueprint $table) {
$table->integer('progress_max')->default(0);
});
}
/**
* Reverse the migrations.
*/
public function down(): void
{
Schema::table('media_jobs', function (Blueprint $table) {
$table->dropColumn('progress_max');
});
}
};

157
resources/js/components/dialogs/SMDialogMedia.vue
View File

@@ -151,8 +151,8 @@
<SMLoading
v-if="getMediaStatus(item).busy"
small
class="bg-white bg-op-90 w-full h-full"
>{{
class="bg-white bg-op-90 w-full h-full">
{{
getMediaStatusText(item)
}}</SMLoading
>
@@ -197,6 +197,27 @@
width: `${computedUploadProgress}%`,
}"></div>
</div>
<p class="m-0">
{{ computedUploadMediaStatus }}
</p>
</div>
<div
v-if="getProcessingMediaItems().length > 0"
class="flex flex-col text-xs border-b border-gray-3 pb-4 mb-4">
<h3 class="text-xs mb-2">
{{ computedProcessingMediaTitle }}
</h3>
<div
class="w-full bg-gray-3 h-3 mb-2 rounded-2">
<div
class="bg-sky-600 h-3 rounded-2"
:style="{
width: `${computedProcessingProgress}%`,
}"></div>
</div>
<p class="m-0">
{{ computedProcessingMediaStatus }}
</p>
</div>
<div v-if="lastSelected != null">
<div
@@ -818,6 +839,7 @@ const handleChangeSelectFile = async () => {
const handleFilesUpload = (files: FileList) => {
const fileList = [];
let count = 0;
let maxCount = 15;
let warnedUser = false;
fileList.push(...Array.from(files));
@@ -826,7 +848,7 @@ const handleFilesUpload = (files: FileList) => {
if (mimeMatches(props.mime, file.type) == true) {
count = getUploadingMediaItems().length;
if (count < 5) {
if (count <= maxCount) {
const uploadId = generateRandomId("upload_", 8, (s) => {
return getMediaItemById(s) != null;
});
@@ -834,18 +856,19 @@ const handleFilesUpload = (files: FileList) => {
mediaItems.value.unshift(
createMediaItem({
id: uploadId,
name: convertFileNameToTitle(file.name),
}),
);
window.setTimeout(() => {
uploadFileById(uploadId, file);
}, 50);
} else if (count >= 5 && warnedUser == false) {
} else if (count > maxCount && warnedUser == false) {
warnedUser = true;
useToastStore().addToast({
title: "Maximum Files",
type: "warning",
content: "You cannot upload more than 5 files at a time",
content: `You cannot upload more than ${maxCount} files at a time`,
});
return;
@@ -861,18 +884,33 @@ const handleFilesUpload = (files: FileList) => {
};
const getUploadingMediaItems = (): Media[] => {
return mediaItems.value.filter((item) => item.id.startsWith("upload_"));
return mediaItems.value.filter((item) => {
return (
item.id.startsWith("upload_") &&
item.jobs.length > 0 &&
item.jobs[0].status === "uploading"
);
});
};
const computedUploadMediaTitle = computed(() => {
const items = getUploadingMediaItems();
let prefix = "Uploading";
return `Uploading ${items.length} File${items.length == 1 ? "" : "s"}`;
});
if (computedUploadProgress.value >= 100) {
prefix = "Processing";
}
const computedUploadMediaStatus = computed(() => {
const items = getUploadingMediaItems();
let bytes = 0;
let maxBytes = 0;
return `${prefix} ${items.length} File${items.length == 1 ? "" : "s"}`;
items.forEach((item) => {
if (item.jobs.length > 0) {
bytes += item.jobs[0].progress;
maxBytes += item.jobs[0].progress_max;
}
});
return `${bytesReadable(bytes)} of ${bytesReadable(maxBytes)}`;
});
const computedUploadProgress = computed(() => {
@@ -883,7 +921,10 @@ const computedUploadProgress = computed(() => {
const totalProgress = items.reduce((accumulator, item) => {
if (item.jobs.length > 0) {
accumulator += item.jobs[0].progress || 100;
accumulator +=
Math.floor(
(item.jobs[0].progress / item.jobs[0].progress_max) * 100,
) || 100;
}
return accumulator;
}, 0);
@@ -891,6 +932,65 @@ const computedUploadProgress = computed(() => {
return Math.floor(totalProgress / items.length);
});
const getProcessingMediaItems = (): Media[] => {
return mediaItems.value.filter((item) => {
return (
item.id.startsWith("upload_") &&
item.jobs.length > 0 &&
item.jobs[0].status !== "uploading"
);
});
};
const computedProcessingMediaTitle = computed(() => {
const items = getProcessingMediaItems();
return `Processing ${items.length} File${items.length == 1 ? "" : "s"}`;
});
const computedProcessingProgress = computed(() => {
const items = getProcessingMediaItems();
if (items.length === 0) {
return 100;
}
const totalProgress = items.reduce((accumulator, item) => {
if (item.jobs.length > 0) {
if (item.jobs[0].progress_max != 0) {
accumulator +=
Math.floor(
(item.jobs[0].progress / item.jobs[0].progress_max) *
100,
) || 100;
}
}
return accumulator;
}, 0);
return Math.floor(totalProgress / items.length);
});
const computedProcessingMediaStatus = computed(() => {
const items = getProcessingMediaItems();
let status = "";
items.every((item) => {
let itemStatus = getMediaStatusText(item);
if (status == "" || (itemStatus != "" && itemStatus != "Queued")) {
const endLoop = !(status == "");
status = `${item.name}: ${itemStatus}`;
if (endLoop == true) {
return false;
}
}
return true;
});
return status;
// return `${bytesReadable(bytes)} of ${bytesReadable(maxBytes)}`;
});
/**
* Upload a File to the server.
* @param {string} uploadId The ID of the new media item.
@@ -909,14 +1009,14 @@ const uploadFileById = (uploadId: string, file: File): void => {
headers: {
"Content-Type": "multipart/form-data",
},
chunk: "file",
progress: (progressEvent) => {
const mediaItem = getMediaItemById(uploadId);
if (mediaItem != null) {
mediaItem.jobs[0] = createMediaJobItem({
status: "uploading",
progress: Math.floor(
(progressEvent.loaded / progressEvent.total) * 100,
),
progress: progressEvent.loaded,
progress_max: progressEvent.total,
});
}
},
@@ -934,10 +1034,20 @@ const uploadFileById = (uploadId: string, file: File): void => {
}
})
.catch((error) => {
// let errorString = "A server error occurred";
// if (error.status == 413) {
// errorString = `The file is larger than ${max_upload_size.value}`;
// }
if (error.status == 413) {
useToastStore().addToast({
title: "File too large",
type: "danger",
content: `Cannot upload the file ${file.name} as it larger than ${max_upload_size.value}.`,
});
} else {
useToastStore().addToast({
title: "File upload error",
type: "danger",
content: `Cannot upload the file ${file.name} as a server error occurred.`,
});
}
console.log(error);
});
};
@@ -949,7 +1059,7 @@ const uploadFileById = (uploadId: string, file: File): void => {
*/
const updateMediaItem = (id: string): void => {
let media = getMediaItemById(id);
let timeout = 50;
let timeout = 200;
if (media != null && media.jobs.length > 0) {
if (id.startsWith("upload_")) {
@@ -964,10 +1074,13 @@ const updateMediaItem = (id: string): void => {
if (data.media_job.media_id != null) {
media.id = data.media_job.media_id;
}
if (data.media_job.id == media.jobs[0].id) {
media.jobs[0] = data.media_job;
}
})
.catch((error) => {
if (error.status == 429) {
timeout = 500;
timeout = 1000;
}
/* error */
@@ -1398,7 +1511,7 @@ const postUpdate = (data: MediaUpdate): void => {
description: data.description,
},
}).catch((error) => {
console.log(error);
console.log("postupdate: " + error);
});
};

10
resources/js/helpers/api.ts
View File

@@ -134,8 +134,6 @@ export const api = {
for (const header in options.headers) {
xhr.setRequestHeader(header, options.headers[header]);
}
xhr.send(options.body as XMLHttpRequestBodyInit);
xhr.onload = function () {
const result = {
status: xhr.status,
@@ -188,6 +186,12 @@ export const api = {
return;
}
};
try {
xhr.send(options.body as XMLHttpRequestBodyInit);
} catch (e) {
console.log(e);
}
} else {
const fetchOptions: RequestInit = {
method: options.method.toUpperCase() || "GET",
@@ -379,7 +383,7 @@ export const api = {
const file = apiOptions.body.get(apiOptions.chunk);
if (file instanceof File) {
const chunkSize = 50 * 1024 * 1024;
const chunkSize = 2 * 1024 * 1024;
let chunk = 0;
let chunkCount = 1;
let job_id = -1;

1
resources/js/helpers/api.types.ts
View File

@@ -95,6 +95,7 @@ export interface MediaJob {
status: string;
status_text: string;
progress: number;
progress_max: number;
}
export interface MediaJobResponse {

14
resources/js/helpers/media.ts
View File

@@ -193,15 +193,15 @@ export const getMediaStatusText = (media: Media): string => {
) {
if (media.jobs[0].status_text != "") {
status = toTitleCase(media.jobs[0].status_text);
if (
media.jobs[0].status == "processing" &&
media.jobs[0].progress > -1
) {
status += ` ${media.jobs[0].progress}%`;
}
} else {
status = toTitleCase(media.jobs[0].status);
}
if (media.jobs[0].progress_max != 0) {
status += ` ${Math.floor(
(media.jobs[0].progress / media.jobs[0].progress_max) * 100,
)}%`;
}
}
}
@@ -234,6 +234,7 @@ export interface MediaJobParams {
status?: string;
status_text?: string;
progress?: number;
progress_max?: number;
}
export const createMediaItem = (params?: MediaParams): Media => {
@@ -267,6 +268,7 @@ export const createMediaJobItem = (params?: MediaJobParams): MediaJob => {
status: params.status || "",
status_text: params.status_text || "",
progress: params.progress || 0,
progress_max: params.progress_max || 0,
};
return job;