起初只是理论上了解大文件分片上传和断点续传的原理,但是没有实际项目中使用,所以一直没有去实现。上周在微信上看到一篇文章 实现大文件上传全流程详解(补偿版本),文中是使用vue+express实现的,正好最近了解到Hono框架就参考了思路实现了一个react+hono版本的例子。
原文中的并发部分的中断处理应该是有问题的,所以自己又实现了一下请求并发的部分。
基本思路:
- 文件使用hash分片,hash使用截取部分生成的方式优化计算时间。
- 后端使用hash路径存储,根据hash路径判断文件是否存在。
- 前端上传之前先通过接口检查文件是否存在,存在部分路径只要得到未上传部分的hash上传即可。
原理是比较简单的,就是处理文件存储部分需要使用buffer和arrayBuffer比较不常用,这次实践一次下次自己处理文件就有经验了。
前端代码
spark-md5是个计算文件hash的库,使用起来比较简单。
import { useState } from "react";
import sparkMD5 from "spark-md5";
/** 分片大小 */
const CHUNK_SIZE = 1024 * 1024;
/** 最大并发请求 */
const MAX_CONCURRENT = 6;
let abortFlag = false;
let abortControllers: AbortController[] = [];
/** 服务地址 */
const base_url = 'http://localhost:3003';
/**
* 文件chunks计算Hash,中间取样加速计算时间
* @param chunks 文件生成chunks
* @returns 计算的MD5 Hash
*/
const calculateHash = (chunks: Blob[]): Promise<string> => {
return new Promise((resolve, reject) => {
try {
const spark = new sparkMD5.ArrayBuffer();
const fileReader = new FileReader();
const targets: Blob[] = [];
chunks.forEach((chunk, index) => {
if (index === 0 || index === chunks.length - 1) {
targets.push(chunk);
} else {
targets.push(chunk.slice(0, 2));
targets.push(chunk.slice(CHUNK_SIZE / 2, CHUNK_SIZE / 2 + 2));
targets.push(chunk.slice(-2));
}
});
fileReader.readAsArrayBuffer(new Blob(targets));
fileReader.onload = (e) => {
spark.append(e.target?.result as ArrayBuffer);
resolve(spark.end());
};
} catch (error) {
reject(error);
}
});
};
/**
* file创建chunks
* @param file 上传的文件
* @returns file分割的chunks
*/
const createChunks = (file: File) => {
let cur = 0;
const chunks: Blob[] = [];
while (cur < file.size) {
const blob= file.slice(cur, cur + CHUNK_SIZE);
chunks.push(blob);
cur = CHUNK_SIZE;
}
return chunks;
};
/**
* 验证文件完整性
* @param fileHash 文件的Hash
* @param fileName 文件名称
* @returns 文件验证信息
*/
const verify= async (
fileHash: string,
fileName: string
): Promise<{
data: {
shouldUpload: boolean;
existChunks: string[];
};
}> => {
const res = await fetch("http://localhost:3003/api/verify", {
method: "POST",
headers: {
"Content-Type": "application/json",
},
body: JSON.stringify({
fileHash: fileHash,
fileName: fileName,
}),
});
return res.json();
};
/**
* 请求合并文件
* @param fileHash 文件的Hash
* @param fileName 文件名称
*/
const mergeRequest = async (fileHash: string, fileName: string) => {
await fetch(`${base_url}/api/merge`, {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({
fileHash,
fileName,
size: CHUNK_SIZE,
}),
});
};
interface UploadChunkResponse {
status: number;
message: string;
}
/**
* 并发控制上传
* TODO: 写的不对,不能中断
* @param formDataList 文件chunks数据
*/
const uploadWithConcurrencyControl = async (formDataList: FormData[]) => {
const tasks = formDataList.map((i) => () => {
const controller = new AbortController();
abortControllers.push(controller);
return fetch(`${base_url}/api/upload`, {
method: "POST",
body: i,
signal: controller.signal,
})
.then((res) => res.json())
.catch((err) => {
if (err.name !== "AbortError") {
console.error("分片上传失败:", err);
throw new Error("xxx");
}
});
});
let index = 0;
const result: Promise<UploadChunkResponse>[] = [];
const executing: Promise<any>[] = [];
const execute = () => {
if (index >= tasks.length) {
return Promise.resolve();
}
if (abortFlag) {
return Promise.resolve();
}
const p = tasks[index++];
const promise = Promise.resolve().then(() => p());
result.push(promise);
const remove = () => {
const i = executing.indexOf(promise);
if (i !== -1) {
executing.splice(i, 1);
}
};
const handler = (): Promise<any> => {
remove();
return execute();
};
const promiseWrapper = promise.then(handler, handler);
executing.push(promiseWrapper);
return promiseWrapper;
};
const allPromise = [];
while (allPromise.length < MAX_CONCURRENT && index < tasks.length) {
allPromise.push(execute());
}
return Promise.all(allPromise).then(()=> Promise.all(result));
};
interface UploadChunksProps {
/** 文件chunks */
chunks: Blob[];
/** 已上传的chunks */
existChunks: string[];
/** 文件 MD5 Hash */
fileHash: string;
/** 文件名称 */
fileName: string;
}
/**
* 分片上传
* @param param0
* @returns
*/
const uploadChunks= async ({
chunks,
existChunks,
fileHash,
fileName,
}: UploadChunksProps)=> {
const chunkInfoList= chunks.map((chunk, index)=> ({
fileHash: fileHash,
chunkHash: `${fileHash}-${index}`,
chunk,
}));
const formDataList= chunkInfoList
.filter((item)=> !existChunks.includes(item.chunkHash))
.map((item)=> {
const formData= new FormData();
formData.append("fileHash", item.fileHash);
formData.append("chunkHash", item.chunkHash);
formData.append("chunk", item.chunk);
return formData;
});
if (formDataList.length = 0) {
await uploadWithConcurrencyControl(formDataList);
}
if (abortFlag) {
abortFlag= false;
return Promise.reject("abort");
}
return mergeRequest(fileHash, fileName);
};
const FileUploader= ()=> {
/** 上传loading */
const [loading, setLoading]= useState(false);
/** 点击上传文件 */
const handleChange= async (e: React.ChangeEvent<HTMLInputElement>) => {
const file = e.target.files?.[0];
if (!file) {
console.error("file is null");
return;
}
const fileName = file.name;
const chunks = createChunks(file);
const fileHash = await calculateHash(chunks);
const verifyRes = await verify(fileHash, fileName);
if (!verifyRes.data.shouldUpload) {
alert("秒传成功,文件已存在");
return;
}
setLoading(true);
abortControllers = [];
await uploadChunks({
chunks,
existChunks: verifyRes.data.existChunks,
fileHash,
fileName,
})
.then(() => {
alert("文件上传完成!");
})
.catch((e) => {
if (e === "abort") {
return;
}
throw e;
})
.finally(() => {
setLoading(false);
});
};
/** 用户中断请求 */
const abortUpload = () => {
if (!loading) return;
abortFlag = true;
for (const abort of abortControllers) {
abort.abort();
}
setLoading(false);
alert("上传已中断,下次可继续上传");
};
return (
<div>
<input onChange={handleChange} type="file" />
<button onClick={abortUpload} type="button">
中断请求
</button>
</div>
);
};
export default FileUploader;
后端代码
后端使用hono框架,使用感受是真的很方便,尤其是内置了很多中间件不用安装而且对ts非常友好,写起来也比较快。总体感受就是比koa方便比express轻量。
import { serve } from "@hono/node-server";
import { Hono } from "hono";
import { cors } from "hono/cors";
import path from "node:path";
import { fileURLToPath } from "node:url";
import fse from "fs-extra";
const app = new Hono();
app.use("/api/*", cors());
app.get("/", (c) => {
return c.text("Hello Hono!");
});
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
const UPLOAD_DIR = path.resolve(__dirname, "uploads");
fse.ensureDir(UPLOAD_DIR);
const extractExt = (fileName: string) => {
return fileName.slice(fileName.lastIndexOf("."));
};
interface RequestParams {
fileName: string;
fileHash: string;
}
interface ResponseData {
status?: number;
data: {
shouldUpload?: boolean;
existChunks?: string[];
};
}
/**
* 验证文件数据
*/
app.post("/api/verify", async (c) => {
const { fileName, fileHash } = await c.req.json<RequestParams>();
const completeFilePath = path.resolve(
UPLOAD_DIR,
`${fileHash}${extractExt(fileName)}`
);
if (fse.existsSync(completeFilePath)) {
return c.json<ResponseData>({
status: 200,
data: {
shouldUpload: false,
},
});
}
const chunkDir = path.resolve(UPLOAD_DIR, fileHash);
const existChunks = fse.existsSync(chunkDir)
? await fse.readdir(chunkDir)
: [];
return c.json<ResponseData>({
data: {
existChunks,
shouldUpload: true,
},
});
});
/**
* 上传分片数据
*/
app.post("/api/upload", async (c) => {
const form = await c.req.formData();
const fileHash = form.get("fileHash") as string;
const chunkHash = form.get("chunkHash") as string;
const chunkFile = form.get("chunk") as File;
const chunkDir = path.resolve(UPLOAD_DIR, fileHash);
await fse.ensureDir(chunkDir);
const targetChunkPath = path.resolve(chunkDir, chunkHash);
await new Promise((res) => setTimeout(res, 2000));
const arrayBuffer = await chunkFile.arrayBuffer();
const buffer = Buffer.from(arrayBuffer);
await fse.writeFile(targetChunkPath, buffer);
return c.json({
status: 200,
message: "分片上传成功",
});
});
/** 合并文件 */
app.post("/api/merge", async (c) => {
const { fileHash, fileName, size: CHUNK_SIZE } = await c.req.json();
const completeFilePath = path.resolve(
UPLOAD_DIR,
`${fileHash}${extractExt(fileName)}`
);
const chunkDir = path.resolve(UPLOAD_DIR, fileHash);
if (!fse.existsSync(chunkDir)) {
c.status(400);
return c.json({
status: 400,
message: "分片目录不存在",
});
}
const chunkPaths = await fse.readdir(chunkDir);
chunkPaths.sort(
(a, b) => parseInt(a.split("-")[1]) - parseInt(b.split("-")[1])
);
const mergePromises = chunkPaths.map((chunkName, index) => {
return new Promise((resolve, reject) => {
const chunkPath = path.resolve(chunkDir, chunkName);
const readStream = fse.createReadStream(chunkPath);
const writeStream = fse.createWriteStream(completeFilePath, {
start: index * CHUNK_SIZE,
});
readStream.on("end", async () => {
await fse.unlink(chunkPath);
resolve(true);
});
readStream.pipe(writeStream);
});
});
await Promise.all(mergePromises);
await fse.remove(chunkDir);
return c.json({
status: 200,
message: "文件合并成功",
});
});
serve(
{
fetch: app.fetch,
port: 3003,
},
(info) => {
console.log(`Server is running on http://localhost:${info.port}`);
}
);