94、springboot+minio实现分片上传(超大文件快速上传)
2024-10-20 21:11:04
设计由来
在实际的项目开发中常遇到超大附件上传的情况，有时候客户会上传GB大小的文件，如果按照普通的
MultipartFile方式来接收上传的文件，那么无疑会把服务器给干崩溃，更别说并发操作了。
于是笔者决定要写一个超大附件上传的方法，于是有此。
功能实现图
功能介绍
- 上传请求异步操作，前端使用Worker线程处理，避免主线程阻塞
- 使用vue+springboot+minio实现方式
- 前端对大文件进行分片+后台进行合并
- 由于md5计算耗时太大故隐去改功能，md5可以实现妙传功能（校验文件是否存在）
- 支持多文件上传+文件夹上传（递归文件夹中的所有文件）
核心代码
<template>
<div class="container">
<h2>Minio 上传示例</h2>
<el-upload
class="upload-demo"
ref="upload"
action="https://jsonplaceholder.typicode.com/posts/"
:on-remove="handleRemove"
:on-change="handleFileChange"
:file-list="uploadFileList"
:show-file-list="false"
:auto-upload="false">
<el-button slot="trigger" type="primary" plain>选择文件</el-button>
<el-button style="margin-left: 10px;" type="success" @click="handleUpload" plain>上传</el-button>
<el-button type="danger" @click="clearFileHandler" plain>清空</el-button>
</el-upload>
</div>
</div>
</template> <script>
import SparkMD5 from 'spark-md5'
import axios from 'axios'
const FILE_UPLOAD_ID_KEY = 'file_upload_id'
const chunkSize = 10 * 1024 * 1024
let currentFileIndex = 0
const FileStatus = {
wait: '等待上传',
getMd5: '校验MD5',
uploading: '正在上传',
success: '上传成功',
error: '上传错误'
}
export default {
data () {
return {
changeDisabled: false,
uploadDisabled: false,
// 上传并发数
simultaneousUploads: 3,
partCount:0,
uploadIdInfo: null,
uploadFileList: [],
retryList: []
}
},
methods: {
handleUpload() {
const self = this
const files = this.uploadFileList
if (files.length === 0) {
this.$message.error('请先选择文件')
return
}
// 当前操作文件
const currentFile = files[currentFileIndex]
currentFile.status = FileStatus.getMd5
// 1. 计算MD5
this.getFileMd5(currentFile.raw, async (md5) => {
// 2. 检查是否已上传
// const checkResult = await self.checkFileUploadedByMd5(md5)
// // 已上传
// if (checkResult.data.status === 1) {
// self.$message.success(`上传成功，文件地址：${checkResult.data.url}`)
// console.log('文件访问地址：' + checkResult.data.url)
// currentFile.status = FileStatus.success
// currentFile.uploadProgress = 100
// return
// } else if (checkResult.data.status === 2) { // "上传中" 状态
// // 获取已上传分片列表
// let chunkUploadedList = checkResult.data.chunkUploadedList
// currentFile.chunkUploadedList = chunkUploadedList
// } else { // 未上传
// console.log('未上传')
// } console.log('文件MD5：' + md5)
// 3. 正在创建分片
let fileChunks = self.createFileChunk(currentFile.raw, chunkSize) let param = {
fileName: currentFile.name,
fileSize: currentFile.size,
chunkSize: chunkSize,
fileMd5: md5,
contentType: 'application/octet-stream',
partCount:this.partCount
}
// 4. 获取上传url
let uploadIdInfoResult = await self.getFileUploadUrls(param)
self.uploadIdInfo = uploadIdInfoResult.data.uploadId
self.saveFileUploadId(uploadIdInfoResult.data.uploadId)
let uploadUrls = uploadIdInfoResult.data.uploadUrls
if (fileChunks.length !== uploadUrls.length) {
self.$message.error('文件分片上传地址获取错误')
return
}
self.$set(currentFile, 'chunkList', [])
fileChunks.map((chunkItem, index) => {
currentFile.chunkList.push({
chunkNumber: index + 1,
chunk: chunkItem,
uploadUrl: uploadUrls[index],
progress: 0,
status: '—'
})
})
let tempFileChunks = []
currentFile.chunkList.forEach((item) => {
tempFileChunks.push(item)
})
currentFile.status = FileStatus.uploading
// 处理分片列表，删除已上传的分片
tempFileChunks = self.processUploadChunkList(tempFileChunks)
// 5. 上传
await self.uploadChunkBase(tempFileChunks)
console.log('上传完成')
debugger
// 6. 合并文件
const mergeResult = await self.mergeFile({
uploadId: self.uploadIdInfo,
fileName: currentFile.name,
md5: md5
})
if (!mergeResult.success) {
currentFile.status = FileStatus.error
self.$message.error(mergeResult.error)
} else {
currentFile.status = FileStatus.success
console.log('文件访问地址：' + mergeResult.data.url)
self.$message.success(`上传成功，文件地址：${mergeResult.data.url}`)
}
})
},
clearFileHandler() {
this.uploadFileList = []
this.uploadIdInfo = null
},
handleFileChange(file, fileList) {
this.uploadFileList = fileList
this.uploadFileList.forEach((item) => {
// 初始化自定义属性
this.initFileProperties(item)
})
},
initFileProperties(file) {
file.chunkList = []
file.status = FileStatus.wait
file.progressStatus = 'warning'
file.uploadProgress = 0
},
handleRemove(file, fileList) {
this.uploadFileList = fileList
},
/**
* 分片读取文件 MD5
*/
getFileMd5(file, callback) {
const blobSlice = File.prototype.slice || File.prototype.mozSlice || File.prototype.webkitSlice
const fileReader = new FileReader()
// 计算分片数
const totalChunks = Math.ceil(file.size / chunkSize)
console.log('总分片数：' + totalChunks)
this.partCount=totalChunks
let currentChunk = 0
const spark = new SparkMD5.ArrayBuffer()
loadNext()
fileReader.onload = function (e) {
try {
spark.append(e.target.result)
} catch (error) {
console.log('获取Md5错误：' + currentChunk)
}
if (currentChunk < totalChunks) {
currentChunk++
loadNext()
} else {
callback(spark.end())
}
}
fileReader.onerror = function () {
console.warn('读取Md5失败，文件读取错误')
}
function loadNext () {
const start = currentChunk * chunkSize
const end = ((start + chunkSize) >= file.size) ? file.size : start + chunkSize
// 注意这里的 fileRaw
fileReader.readAsArrayBuffer(blobSlice.call(file, start, end))
}
},
/**
* 文件分片
*/
createFileChunk(file, size = chunkSize) {
const fileChunkList = []
let count = 0
while(count < file.size) {
fileChunkList.push({
file: file.slice(count, count + size),
})
count += size
}
return fileChunkList
},
/**
* 处理即将上传的分片列表，判断是否有已上传的分片，有则从列表中删除
*/
processChunkList(chunkList) {
const currentFile = this.uploadFileList[currentFileIndex]
let chunkUploadedList = currentFile.chunkUploadedList
if (chunkUploadedList === undefined || chunkUploadedList === null || chunkUploadedList.length === 0) {
return chunkList
}
//
for (let i = chunkList.length - 1; i >= 0; i--) {
const chunkItem = chunkList[i]
for (let j = 0; j < chunkUploadedList.length; j++) {
if (chunkItem.chunkNumber === chunkUploadedList[j]) {
chunkList.splice(i, 1)
break
}
}
}
return chunkList
},
uploadBase(chunkList) {
const self = this
let successCount = 0
let totalChunks = chunkList.length
return new Promise((resolve, reject) => {
const handler = () => {
if (chunkList.length) {
const chunkItem = chunkList.shift()
// 直接上传二进制，不需要构造 FormData，否则上传后文件损坏
axios.put(chunkItem.uploadUrl, chunkItem.chunk.file, {
// 上传进度处理
onUploadProgress: self.checkChunkUploadProgress(chunkItem),
headers: {
'Content-Type': 'application/octet-stream'
}
}).then(response => {
if (response.status === 200) {
console.log('分片：' + chunkItem.chunkNumber + ' 上传成功')
successCount++
// 继续上传下一个分片
handler()
} else {
console.log('上传失败：' + response.status + '，' + response.statusText)
}
}).catch(error => {
// 更新状态
console.log('分片：' + chunkItem.chunkNumber + ' 上传失败，' + error)
// 重新添加到队列中
chunkList.push(chunkItem)
handler()
})
}
if (successCount >= totalChunks) {
resolve()
}
}
// 并发
for (let i = 0; i < this.simultaneousUploads; i++) {
handler()
}
})
},
getFileUploadUrls(fileParam) {
debugger
let url = `http://127.0.0.1:8006/multipart/init`
return axios.post(url, fileParam)
},
saveFileUploadId(data) {
localStorage.setItem(FILE_UPLOAD_ID_KEY, data)
},
checkFileUploadedByMd5(md5) {
console.log(md5);
// let url = `http://127.0.0.1:8006/upload/check?md5=${md5}`
// return new Promise((resolve, reject) => {
// axios.get(url).then((response) => {
// resolve(response.data)
// }).catch(error => {
// reject(error)
// })
// })
},
/**
* 合并文件
*/
mergeFile(file) {
const self = this
let url = `http://127.0.0.1:8006/multipart/complete`
return new Promise((resolve, reject) => {
axios.post(url,{
"uploadId":file.uploadId,
"fileName":file.fileName,
"md5":file.md5
}).then(response => {
let data = response.data
if (!data.success) {
resolve(data)
} else {
file.status = FileStatus.success
resolve(data)
}
}).catch(error => {
self.$message.error('合并文件失败：' + error)
file.status = FileStatus.error
reject()
})
})
},
/**
* 检查分片上传进度
*/
checkChunkUploadProgress(item) {
return p => {
item.progress = parseInt(String((p.loaded / p.total) * 100))
this.updateChunkUploadStatus(item)
}
},
updateChunkUploadStatus(item) {
let status = FileStatus.uploading
let progressStatus = 'normal'
if (item.progress >= 100) {
status = FileStatus.success
progressStatus = 'success'
}
let chunkIndex = item.chunkNumber - 1
let currentChunk = this.uploadFileList[currentFileIndex].chunkList[chunkIndex]
// 修改状态
currentChunk.status = status
currentChunk.progressStatus = progressStatus
// 更新状态
this.$set(this.uploadFileList[currentFileIndex].chunkList, chunkIndex, currentChunk)
// 获取文件上传进度
this.getCurrentFileProgress()
},
getCurrentFileProgress() {
const currentFile = this.uploadFileList[currentFileIndex]
if (!currentFile || !currentFile.chunkList) {
return
}
const chunkList = currentFile.chunkList
const uploadedSize = chunkList.map((item) => item.chunk.file.size * item.progress).reduce((acc, cur) => acc + cur)
// 计算方式：已上传大小 / 文件总大小
let progress = parseInt((uploadedSize / currentFile.size).toFixed(2))
currentFile.uploadProgress = progress
this.$set(this.uploadFileList, currentFileIndex, currentFile)
}
},
filters: {
transformByte(size) {
if (!size) {
return '0B'
}
const unitSize = 1024
if (size < unitSize) {
return size + ' B'
}
// KB
if (size < Math.pow(unitSize, 2)) {
return (size / unitSize).toFixed(2) + ' K';
}
// MB
if (size < Math.pow(unitSize, 3)) {
return (size / Math.pow(unitSize, 2)).toFixed(2) + ' MB'
}
// GB
if (size < Math.pow(unitSize, 4)) {
return (size / Math.pow(unitSize, 3)).toFixed(2) + ' GB';
}
// TB
return (size / Math.pow(unitSize, 4)).toFixed(2) + ' TB';
}
}
}
</script>
说明：由于篇幅有限仅提供核心内容部分，如果疑问请联系QQ:3313749159 一同探讨学习。
最新文章
- https简介/原理/部署【转】
- linux命令巧用，随手记
- ArcGIS中的三种查询
- Apache Spark源码走读之9 -- Spark源码编译
- C Primer Plus(第五版)1
- controller.pp 各组件的安装顺序
- Python 学习之urllib模块---用于发送网络请求，获取数据（2）
- i++ ++i的原子性
- HttpWebResponse请求状态代码
- Cloudera Development Kit(CDK) 简介
- 20155304 实验一《Java开发环境的熟悉》实验报告
- Sqoop1.99.7将MySQL数据导入到HDFS中
- tmux的详细讲解
- 【基础篇】elasticsearch之索引模板Template[转]
- 每天一个Linux命令 10
- 面向对象【day08】：静态方法、类方法、属性方法（九）
- Fiddler抓包配置具体步骤
- 02-01:springboot整合servlet开发
- zoj 3827(2014牡丹江现场赛 I题 )
- IE6-IE9兼容性问题列表及解决办法:锁表头的JQuery方案和非JQuery方案(不支持IE6,7,8)