接上篇文章 java 超大文件分片上传 在其基础上继续实现 断点续传和文件秒传功能
在上篇中,我们可以使用 file. slice 方法对文件进行分片,可以从后台读到当前文件已经上传的大小,就可以知道从哪里开始切片,断点续传的原理就是基于这个的。
前端计算文件的 md5 ,后台数据库查询一遍(前提是把 md5 存储了,计算文件 md5 也是需要消耗时间的)即可知道是否有相同文件,这是实现文件秒传的方法。
可能存在的问题:
- 有两个人同时在上传同一个文件,但前一个人还没有上传完成,此时第二个文件认为是新文件不能秒传
- 此时获取文件原数据时需要将文件信息保存起来,重点是要保存 md5 ,保证一个文件的 md5 保计算一次
- 获取断点文件时,真实的文件上传位置应该是从文件系统中读出来的
根据需求说明,后台应该存在四个接口,获取文件信息(包含是否可以秒传),获取断点文件列表,分片上传接口,文件完整性验证
全部源码位置 : https://gitee.com/sanri/example/tree/master/test-mvc
/** * 加载断点文件列表 * @return */ @GetMapping("/breakPointFiles") public List<FileInfoPo> breakPointFiles(){ List<FileInfoPo> fileInfoPos = fileMetaDataRepository.breakPointFiles(); return fileInfoPos; } /** * 获取文件元数据,判断文件是否可以秒传 * @param originFileName * @param fileSize * @param md5 * @return * @throws URISyntaxException */ @GetMapping("/fileMetaData") public FileMetaData fileMetaData(String originFileName, Long fileSize, String md5) throws URISyntaxException, MalformedURLException { FileMetaData similarFile = bigFileStorage.checkSimilarFile(originFileName,fileSize, md5); if(similarFile != null){ similarFile.setSecUpload(true); // 如果文件名不一致,则创建链接文件 if(!similarFile.getOriginFileName() .equals(originFileName)) { bigFileStorage.createSimilarLink(similarFile); } return similarFile; } //获取文件相关信息 String baseName = FilenameUtils.getBaseName(originFileName); String extension = FilenameUtils.getExtension(originFileName); String finalFileName = bigFileStorage.rename(baseName, fileSize); if(StringUtils.isNotEmpty(extension)){ finalFileName += ("."+extension); } URI relativePath = bigFileStorage.relativePath(finalFileName); //如果没有相似文件,则要创建记录到数据库中,为后面断点续传做准备 FileInfoPo fileInfoPo = new FileInfoPo(); fileInfoPo.setName(originFileName); fileInfoPo.setType(extension); fileInfoPo.setUploaded(0); fileInfoPo.setSize(fileSize); fileInfoPo.setRelativePath(relativePath.toString()); fileInfoPo.setMd5(md5); fileMetaDataRepository.insert(fileInfoPo); URI absoluteURI = bigFileStorage.absolutePath(relativePath); FileMetaData fileMetaData = new FileMetaData(originFileName, finalFileName, fileSize, relativePath.toString(), absoluteURI.toString()); fileMetaData.setMd5(md5); fileMetaData.setFileType(extension); return fileMetaData; } /** * 获取当前文件已经上传的大小,用于断点续传 * @return */ @GetMapping("/filePosition") public long filePosition(String relativePath) throws IOException, URISyntaxException { return bigFileStorage.filePosition(relativePath); } /** * 上传分段 * @param multipartFile * @return */ @PostMapping("/uploadPart") public long uploadPart(@RequestParam("file") MultipartFile multipartFile, String relativePath) throws IOException, URISyntaxException { bigFileStorage.uploadPart(multipartFile,relativePath); return bigFileStorage.filePosition(relativePath); } /** * 检查文件是否完整 * @param relativePath * @param fileSize * @param md5 * @return */ @GetMapping("/checkIntegrity") public void checkIntegrity(String relativePath,Long fileSize,String fileName) throws IOException, URISyntaxException { long filePosition = bigFileStorage.filePosition(relativePath); Assert.isTrue(filePosition == fileSize ,"大文件上传失败,文件大小不完整 "+filePosition+" != "+fileSize); String targetMd5 = bigFileStorage.md5(relativePath); FileInfoPo fileInfoPo = fileMetaDataRepository.selectByPrimaryKey(fileName); String md5 = fileInfoPo.getMd5(); Assert.isTrue(targetMd5.equals(md5),"大文件上传失败,文件损坏 "+targetMd5+" != "+md5); //如果文件上传成功,更新文件上传大小 fileMetaDataRepository.updateFilePosition(fileName,filePosition); }
重要的处理部分其实还是前端,下面看前端的代码,需要使用到一个计算 md5 值的库 spark-md5.js
<!DOCTYPE html> <html lang="en"> <head> <meta charset="UTF-8"> <title>大文件批量上传,支持断点续传,文件秒传</title> <style> .upload-item{ padding: 15px 10px; list-style-type: none; display: flex; flex-direction: row; margin-bottom: 10px; border: 1px dotted lightgray; width: 1000px; position: relative; } .upload-item:before{ content: ' '; background-color: lightblue; width: 0px; position: absolute; left: 0; top: 0; bottom: 0; z-index: -1; } .upload-item span{ display: block; margin-left: 20px; } .upload-item>.file-name{ width: 200px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } .upload-item>.upload-process{ width: 50px; text-align: left; } .upload-item>.upload-status{ width: 100px; text-align: center; } table{ width: 100%; border-collapse: collapse; position: fixed; bottom: 200px; border: 1px solid whitesmoke; } </style> </head> <body> <div class="file-uploads"> <input type="file" multiple id="file" /> <button id="startUpload">开始上传</button> <ul id="uploadfiles"> </ul> <table class="" style="" id="table" > <thead> <tr> <td>文件名</td> <td>文件大小</td> <td>已上传大小</td> <td>相对路径</td> <td>md5</td> </tr> </thead> <tbody></tbody> </table> </div> <!-- <script src="jquery-1.8.3.min.js"></script>--> <script src="jquery1.11.1.min.js"></script> <script src="spark-md5.min.js"></script> <script> const root = ''; const breakPointFiles = root + '/breakPointFiles'; // 获取断点文件列表 const fileMetaData = root + '/fileMetaData'; // 新上传文件元数据,secUpload 属性用于判断是否可以秒传 const uploadPart = root +'/uploadPart'; // 分片上传,每片的上传接口 const checkIntegrity = root + '/checkIntegrity'; // 检查文件完整性 const fileInfoPos = root + '/fileInfoPos'; // 获取系统中所有已经上传的文件(调试) const shardSize = 1024 * 1024 * 2; // 分片上传,每片大小 2M const chunkSize = 1024 * 1024 * 4; // md5 计算每段大小 4M const statusInfoMap = {'0':'待上传','1':'正在计算','2':'正在上传','3':'上传成功','4':'上传失败','5':'暂停上传','6':'文件检查'}; let uploadFiles = {}; //用于存储当前需要上传的文件列表 fileName=>fileInfo $(function () { // 用于调试 begin 加载系统中已经上传过的文件列表 $.ajax({ type:'get', url:fileInfoPos, dataType:'json', success:function (res) { let htmlCodes = []; for(let i=0;i<res.length;i++){ htmlCodes.push('<tr>'); htmlCodes.push('<td>'+res[i].name+'</td>'); htmlCodes.push('<td>'+res[i].size+'</td>'); htmlCodes.push('<td>'+res[i].uploaded+'</td>'); htmlCodes.push('<td>'+res[i].relativePath+'</td>'); htmlCodes.push('<td>'+res[i].md5+'</td>'); htmlCodes.push('</tr>') } $('table').append(htmlCodes.join('')) } }) // 用于调试 end // 事件绑定 $('#file').change(changeFiles); // 选择文件列表事件 $('#startUpload').click(beginUpload); // 开始上传 $('#uploadfiles').on('change','input[type=file]',breakPointFileChange); // 断点文件选择事件 // 初始化时加载断点文件 (function () { $.ajax({ type:'get', url:breakPointFiles, dataType:'json', success:function (files) { if(files && files.length > 0){ for (let i=0;i<files.length;i++){ let fileId = id(); let process = parseFloat((files[i].uploaded / files[i].size ) * 100).toFixed(2); $('#uploadfiles').append(templateUploadItem(fileId,files[i],process,5,'断点续传',i+1)); uploadFiles[fileId] = {fileInfo:files[i],status:5}; } } } }) })(window); /** * 文件重新选择事件 * @param e */ function changeFiles(e) { // 检测文件列表是否符合要求,默认都符合 if(this.files.length == 0){return ;} // 先把文件信息追加上去,不做检查也不上传 for (let i = 0; i < this.files.length; i++) { let file = this.files[i]; let fileId = id(); $('#uploadfiles').append(templateUploadItem(fileId,file,0,0,'')); uploadFiles[fileId] = {file:file,status:0}; } } /** * 断点文件选择文件事件 */ function breakPointFileChange(e) { let fileId = $(e.target).closest('li').attr('fileId'); if(this.files.length > 0){ uploadFiles[fileId].file = this.files[0]; } } /** * 开始上传 */ function beginUpload() { // 先对每一个文件进行检查,除断点文件不需要检查外 // console.log(uploadFiles); for(let fileId in uploadFiles){ // 如果断点文件没有 file 信息,直接失败 if(uploadFiles[fileId].status == 5 && !uploadFiles[fileId].file){ //断点文件一定有 fileInfo let fileInfo = uploadFiles[fileId].fileInfo; let $li = $('#uploadfiles').find('li[fileId='+fileId+']'); $li.children('.upload-status').text('上传失败');fileInfo.status = 4; $li.children('.tips').text('无文件信息'); continue; } if(uploadFiles[fileId].status == 5){ //如果断点文件有 file 信息,则可以直接断点续传了 let $li = $('#uploadfiles').find('li[fileId='+fileId+']'); $li.children('.upload-status').text('正在上传');uploadFiles[fileId].status = 2; startUpload(uploadFiles[fileId],$li); continue; } //其它待上传的文件,先后台检查文件信息,再上传 if(uploadFiles[fileId].status == 0){ let $li = $('#uploadfiles').find('li[fileId='+fileId+']'); uploadFiles[fileId].status = 1; $li.children('.upload-status').text('正在计算') //正在计算 checkFileItem(uploadFiles[fileId].file,function (res) { if(res.message && res.message == 'fail'){ $li.children('.upload-status').text(res.returnCode || '上传出错');uploadFiles[fileId].status = 4; }else{ uploadFiles[fileId].fileInfo = res; if(res.secUpload){ $li.children('.upload-status').text('文件秒传');uploadFiles[fileId].status = 3; $li.children('.upload-process').text('100 %'); }else{ $li.children('.upload-status').text('正在上传');uploadFiles[fileId].status = 2; startUpload(uploadFiles[fileId],$li); } } }); } } /** * 计算 md5 值,请求后台查看是否可秒传 */ function checkFileItem(file,callback) { md5Hex(file,function (md5) { $.ajax({ type:'get', async:false, url:fileMetaData, data:{originFileName:file.name,fileSize:file.size,md5:md5}, dataType:'json', success:callback }); }); } /** * 开始正式上传单个文件 * */ function startUpload(uploadFile,$li) { let file = uploadFile.file; let offset = uploadFile.fileInfo.uploaded || 0; let shardCount =Math.ceil((file.size - offset )/shardSize); for(var i=0;i<shardCount;i++){ var start = i * shardSize + offset; var end = Math.min(file.size,start + shardSize );//在file.size和start+shardSize中取最小值,避免切片越界 var filePart = file.slice(start,end); var formData = new FormData(); formData.append("file",filePart,uploadFile.fileInfo.name || uploadFile.fileInfo.originFileName); formData.append('relativePath',uploadFile.fileInfo.relativePath); $.ajax({ async:false, url: uploadPart, cache: false, type: "POST", data: formData, dateType: 'json', processData: false, contentType: false, success:function (uploaded) { //进度计算 let process = parseFloat((uploaded / file.size) * 100).toFixed(2); console.log(file.name+'|'+process); $li.find('.upload-process').text(process + '%'); // 视觉进度 // $('.upload-item').append("<style>.upload-item::before{ width:"+(process * 1000)+ "% }</style>"); if(uploaded == file.size){ // 上传完成后,检查文件完整性 $li.children('.upload-status').text('文件检查'); $.ajax({ type:'get', async:false, url:checkIntegrity, data:{fileName:uploadFile.fileInfo.name || uploadFile.fileInfo.originFileName,fileSize:uploaded,relativePath:uploadFile.fileInfo.relativePath}, success:function (res) { if(res.message != 'fail'){ $li.children('.upload-status').text('上传成功'); }else{ $li.children('.upload-status').text('上传失败'); $li.children('.tips').text(res.returnCode); } } }) } } }); } } } /** * 创建模板 html 上传文件项 * @param fileName * @param process * @param status * @param tips * @returns {string} */ function templateUploadItem(fileId,fileInfo,process,status,tips,breakPoint) { let htmlCodes = []; htmlCodes.push('<li class="upload-item" fileId="'+fileId+'">'); htmlCodes.push('<span class="file-name">'+(fileInfo.name || fileInfo.originFileName)+'</span>'); htmlCodes.push('<span class="file-size">'+(fileInfo.size)+'</span>'); htmlCodes.push('<span class="upload-process">'+process+' %</span>'); htmlCodes.push('<span class="upload-status" >'+statusInfoMap[status+'']+'</span>'); htmlCodes.push('<span class="tips">'+tips+'</span>'); if(breakPoint){ htmlCodes.push('<input type="file" name="file" style="margin-left: 10px;"/>'); } htmlCodes.push('</li>'); return htmlCodes.join(''); } /** * 计算 md5 值(同步计算) * @param file */ function md5Hex(file,callback) { let blobSlice = File.prototype.slice || File.prototype.mozSlice || File.prototype.webkitSlice, chunks = Math.ceil(file.size / chunkSize), currentChunk = 0, spark = new SparkMD5.ArrayBuffer(), fileReader = new FileReader(); fileReader.onload = function (e) { spark.append(e.target.result); // Append array buffer currentChunk++; if (currentChunk < chunks) { loadNext(); } else { let hash = spark.end(); callback(hash); } } fileReader.onerror = function () { console.warn('md5 计算时出错'); }; function loadNext(){ var start = currentChunk * chunkSize, end = ((start + chunkSize) >= file.size) ? file.size : start + chunkSize; fileReader.readAsArrayBuffer(blobSlice.call(file, start, end)); } loadNext(); } function id() { return Math.floor(Math.random() * 1000); } }); </script> </body> </html>
源码位置: https://gitee.com/sanri/example/tree/master/test-mvc
一点小推广
创作不易,希望可以支持下我的开源软件,及我的小工具,欢迎来 gitee 点星,fork ,提 bug 。
http://www.72177.com/htm/201912/05/4509035.htm
http://www.72177.com/htm/201912/05/4509034.htm
http://www.72177.com/htm/201912/05/4509033.htm
http://www.72177.com/htm/201912/05/4509032.htm
http://www.72177.com/htm/201912/05/4509031.htm
http://www.72177.com/htm/201912/05/4509030.htm
http://www.72177.com/htm/201912/05/4509029.htm
http://www.72177.com/htm/201912/05/4509028.htm
http://www.72177.com/htm/201912/05/4509027.htm
http://www.72177.com/htm/201912/05/4509026.htm
http://www.72177.com/htm/201912/05/4509025.htm
http://www.72177.com/htm/201912/05/4509024.htm
http://www.72177.com/htm/201912/05/4509023.htm
http://www.72177.com/htm/201912/05/4509022.htm
http://www.72177.com/htm/201912/05/4509021.htm
http://www.72177.com/htm/201912/05/4509020.htm
http://www.72177.com/htm/201912/05/4509019.htm
http://www.72177.com/htm/201912/05/4509018.htm
http://www.72177.com/htm/201912/05/4509017.htm
http://www.72177.com/htm/201912/05/4509016.htm
http://www.72177.com/htm/201912/05/4509015.htm
http://www.72177.com/htm/201912/05/4509014.htm
http://www.72177.com/htm/201912/05/4509013.htm
http://www.72177.com/htm/201912/05/4509012.htm
http://www.72177.com/htm/201912/05/4509011.htm
http://www.72177.com/htm/201912/05/4509010.htm
http://www.72177.com/htm/201912/05/4509009.htm
http://www.wu0553.com/news/63350.html
http://www.wu0553.com/news/63343.html
http://www.wu0553.com/news/63342.html
http://www.wu0553.com/news/63340.html
http://www.wu0553.com/news/63339.html
http://www.wu0553.com/news/63337.html
http://www.wu0553.com/news/63336.html
http://www.wu0553.com/news/63334.html
http://www.wu0553.com/news/63333.html
http://www.wu0553.com/news/63332.html
http://www.wu0553.com/news/63330.html
http://www.wu0553.com/news/63329.html
http://www.wu0553.com/news/63327.html
http://www.wu0553.com/m/view.php?aid=63350
http://www.wu0553.com/m/view.php?aid=63343
http://www.wu0553.com/m/view.php?aid=63342
http://www.wu0553.com/m/view.php?aid=63340
http://www.wu0553.com/m/view.php?aid=63339
http://www.wu0553.com/m/view.php?aid=63337
http://www.wu0553.com/m/view.php?aid=63336
http://www.wu0553.com/m/view.php?aid=63334
http://www.wu0553.com/m/view.php?aid=63333
http://www.wu0553.com/m/view.php?aid=63332
http://www.wu0553.com/m/view.php?aid=63330
http://www.wu0553.com/m/view.php?aid=63329
http://www.wu0553.com/m/view.php?aid=63327
http://market.szonline.net/amaz/27235.html
http://market.szonline.net/amaz/27234.html
http://market.szonline.net/amaz/27233.html
http://market.szonline.net/amaz/27232.html
http://market.szonline.net/amaz/27231.html
http://market.szonline.net/amaz/27230.html
http://market.szonline.net/amaz/27229.html
http://market.szonline.net/amaz/27228.html
http://market.szonline.net/amaz/27227.html
http://market.szonline.net/amaz/27226.html
http://market.szonline.net/amaz/27225.html
http://market.szonline.net/amaz/27224.html
http://market.szonline.net/amaz/27223.html
http://market.szonline.net/amaz/27222.html
http://market.szonline.net/amaz/27218.html
http://market.szonline.net/amaz/27215.html
http://market.szonline.net/amaz/27212.html
http://market.szonline.net/amaz/27208.html
http://market.szonline.net/amaz/27205.html
http://market.szonline.net/amaz/27202.html
http://market.szonline.net/amaz/27198.html
http://market.szonline.net/amaz/27195.html
http://market.szonline.net/amaz/27192.html
http://market.szonline.net/amaz/27188.html
http://market.szonline.net/amaz/27185.html
http://market.szonline.net/amaz/27182.html