Merge pull request #317 from mnaamani/fix/large-file-upload

Works well, nice!

packages/joy-media/src/IterableFile.ts

@@ -0,0 +1,61 @@
+// Based on
+// https://gist.github.com/grishgrigoryan/bf6222d16d72cb28620399d27e83eb22
+
+interface IConfig{
+    chunkSize:number
+}
+
+const DEFAULT_CHUNK_SIZE : number = 64 * 1024; // 64K
+
+export class IterableFile implements AsyncIterable<Buffer>{
+    private reader: FileReader;
+    private file: File
+    private config: IConfig = { chunkSize : DEFAULT_CHUNK_SIZE }
+
+    constructor(file: File, config :Partial<IConfig> = {}) {
+        this.file = file
+        this.reader = new FileReader();
+        Object.assign(this.config, config)
+    }
+
+    [Symbol.asyncIterator]() {
+        return this.readFile();
+    }
+
+    get chunkSize() {
+        return this.config.chunkSize;
+    }
+
+    get fileSize() {
+        return this.file.size;
+    }
+
+    readBlobAsBuffer(blob: Blob) : Promise<Buffer>{
+        return new Promise((resolve,reject)=>{
+            this.reader.onload = (e:any)=>{
+                e.target.result && resolve(Buffer.from(e.target.result));
+                e.target.error && reject(e.target.error);
+            };
+            this.reader.readAsArrayBuffer(blob);
+        })
+    }
+
+    async* readFile() {
+        let offset = 0;
+        let blob;
+        let result;
+
+        while (offset < this.fileSize) {
+            blob = this.file.slice(offset, this.chunkSize + offset);
+            result = await this.readBlobAsBuffer(blob);
+            offset += result.length;
+            yield result;
+        }
+    }
+}
+
+// Usage:
+//  let iterableFile = new IterableFile(file)
+//  for await (const chunk: Buffer of iterableFile) {
+//      doSomethingWithBuffer(chunk)
+//  }

packages/joy-media/src/Upload.tsx

@@ -4,7 +4,7 @@ import axios, { CancelTokenSource } from 'axios';
 import { History } from 'history';
 import { Progress, Message } from 'semantic-ui-react';
 
-import { InputFile } from '@polkadot/react-components/index';
+import { InputFileAsync } from '@polkadot/react-components/index';
 import { ApiProps } from '@polkadot/react-api/types';
 import { I18nProps } from '@polkadot/react-components/types';
 import { SubmittableResult } from '@polkadot/api';
@@ -22,6 +22,7 @@ import TxButton from '@polkadot/joy-utils/TxButton';
 import IpfsHash from 'ipfs-only-hash';
 import { ChannelId } from '@joystream/types/content-working-group';
 import { EditVideoView } from './upload/EditVideo.view';
+import { IterableFile } from './IterableFile';
 
 const MAX_FILE_SIZE_MB = 500;
 const MAX_FILE_SIZE_BYTES = MAX_FILE_SIZE_MB * 1024 * 1024;
@@ -38,8 +39,8 @@ type Props = ApiProps & I18nProps & MyAccountProps & DiscoveryProviderProps & {
 
 type State = {
   error?: any,
-  file_name?: string,
-  file_data?: Uint8Array,
+  file?: File,
+  computingHash: boolean,
   ipfs_cid?: string,
   newContentId: ContentId,
   discovering: boolean,
@@ -50,8 +51,9 @@ type State = {
 
 const defaultState = (): State => ({
   error: undefined,
-  file_name: undefined,
-  file_data: undefined,
+  file: undefined,
+  computingHash: false,
+  ipfs_cid: undefined,
   newContentId: ContentId.generate(),
   discovering: false,
   uploading: false,
@@ -82,11 +84,12 @@ class Component extends React.PureComponent<Props, State> {
   }
 
   private renderContent () {
-    const { error, uploading, discovering } = this.state;
+    const { error, uploading, discovering, computingHash } = this.state;
 
     if (error) return this.renderError();
     else if (discovering) return this.renderDiscovering();
     else if (uploading) return this.renderUploading();
+    else if (computingHash) return this.renderComputingHash();
     else return this.renderFileInput();
   }
 
@@ -102,15 +105,16 @@ class Component extends React.PureComponent<Props, State> {
   }
 
   private resetForm = () => {
-    let newDefaultState = defaultState();
     const { cancelSource } = this.state;
-    newDefaultState.cancelSource = cancelSource;
-    this.setState(newDefaultState);
+    this.setState({
+      cancelSource,
+      ...defaultState()
+    });
   }
 
   private renderUploading () {
-    const { file_name, newContentId, progress, error } = this.state;
-    if (!file_name) return <em>Loading...</em>;
+    const { file, newContentId, progress, error } = this.state;
+    if (!file || !file.name) return <em>Loading...</em>;
 
     const success = !error && progress >= 100;
     const { history, match: { params: { channelId } } } = this.props
@@ -121,7 +125,7 @@ class Component extends React.PureComponent<Props, State> {
         <EditVideoView
           channelId={new ChannelId(channelId)}
           contentId={newContentId}
-          fileName={fileNameWoExt(file_name)}
+          fileName={fileNameWoExt(file.name)}
           history={history}
         />
       }
@@ -155,11 +159,12 @@ class Component extends React.PureComponent<Props, State> {
   }
 
   private renderFileInput () {
-    const { file_name, file_data } = this.state;
-    const file_size = file_data ? file_data.length : 0;
+    const { file } = this.state;
+    const file_size = file ? file.size : 0;
+    const file_name = file ? file.name : '';
 
     return <div className='UploadSelectForm'>
-      <InputFile
+      <InputFileAsync
         label=""
         withLabel={false}
         className={`UploadInputFile ${file_name ? 'FileSelected' : ''}`}
@@ -190,29 +195,60 @@ class Component extends React.PureComponent<Props, State> {
     </div>;
   }
 
-  private onFileSelected = async (data: Uint8Array, file_name: string) => {
-    if (!data || data.length === 0) {
+  private onFileSelected = async (file: File) => {
+    if (!file.size) {
       this.setState({ error: `You cannot upload an empty file.` });
-    } else if (data.length > MAX_FILE_SIZE_BYTES) {
+    } else if (file.size > MAX_FILE_SIZE_BYTES) {
       this.setState({ error:
-        `You cannot upload a file that is more than ${MAX_FILE_SIZE_MB} MB.`
+        `You can't upload files larger than ${MAX_FILE_SIZE_MB} MBytes in size.`
       });
     } else {
-      const ipfs_cid = await IpfsHash.of(Buffer.from(data));
-      console.log('computed IPFS hash:', ipfs_cid)
-      // File size is valid and can be uploaded:
-      this.setState({ file_name, file_data: data, ipfs_cid });
+      this.setState({ file, computingHash: true })
+      this.startComputingHash();
+    }
+  }
+
+  private async startComputingHash() {
+    const { file } = this.state;
+
+    if (!file) {
+      return this.hashComputationComplete(undefined, 'No file passed to hasher');
+    }
+
+    try {
+      const iterableFile = new IterableFile(file, { chunkSize: 65535 });
+      const ipfs_cid = await IpfsHash.of(iterableFile);
+
+      this.hashComputationComplete(ipfs_cid)
+    } catch (err) {
+      return this.hashComputationComplete(undefined, err);
+    }
+  }
+
+  private hashComputationComplete(ipfs_cid: string | undefined, error?: string) {
+    if (!error) {
+      console.log('Computed IPFS hash:', ipfs_cid)
     }
+
+    this.setState({
+      computingHash: false,
+      ipfs_cid,
+      error
+    })
+  }
+
+  private renderComputingHash() {
+    return <em>Processing your file. Please wait...</em>
   }
 
   private buildTxParams = () => {
-    const { file_name, file_data, newContentId, ipfs_cid } = this.state;
-    if (!file_name || !file_data || !ipfs_cid) return [];
+    const { file, newContentId, ipfs_cid } = this.state;
+    if (!file || !ipfs_cid) return [];
 
     // TODO get corresponding data type id based on file content
     const dataObjectTypeId = new BN(1);
 
-    return [ newContentId, dataObjectTypeId, new BN(file_data.length), ipfs_cid];
+    return [ newContentId, dataObjectTypeId, new BN(file.size), ipfs_cid];
   }
 
   private onDataObjectCreated = async (_txResult: SubmittableResult) => {
@@ -248,8 +284,8 @@ class Component extends React.PureComponent<Props, State> {
   }
 
   private uploadFileTo = async (storageProvider: AccountId) => {
-    const { file_data, newContentId, cancelSource } = this.state;
-    if (!file_data || !file_data.length) {
+    const { file, newContentId, cancelSource } = this.state;
+    if (!file || !file.size) {
       this.setState({
         error: new Error('No file to upload!'),
         discovering: false,
@@ -296,7 +332,7 @@ class Component extends React.PureComponent<Props, State> {
     this.setState({ discovering: false, uploading: true, progress: 0 });
 
     try {
-      await axios.put<{ message: string }>(url, file_data, config);
+      await axios.put<{ message: string }>(url, file, config);
     } catch(err) {
       this.setState({ progress: 0, error: err, uploading: false });
       if (axios.isCancel(err)) {

packages/react-components/src/InputFileAsync.tsx

@@ -0,0 +1,117 @@
+// Copyright 2017-2019 @polkadot/react-components authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import { WithTranslation } from 'react-i18next';
+import { BareProps } from './types';
+
+import React, { useState, createRef } from 'react';
+import Dropzone, { DropzoneRef } from 'react-dropzone';
+import styled from 'styled-components';
+import { formatNumber } from '@polkadot/util';
+
+import { classes } from './util';
+import Labelled from './Labelled';
+import translate from './translate';
+
+interface Props extends BareProps, WithTranslation {
+  // Reference Example Usage: https://github.com/react-dropzone/react-dropzone/tree/master/examples/Accept
+  // i.e. MIME types: 'application/json, text/plain', or '.json, .txt'
+  accept?: string;
+  clearContent?: boolean;
+  help?: React.ReactNode;
+  isDisabled?: boolean;
+  isError?: boolean;
+  label: React.ReactNode;
+  onChange?: (blob: File) => void;
+  placeholder?: React.ReactNode | null;
+  withEllipsis?: boolean;
+  withLabel?: boolean;
+}
+
+interface FileState {
+  name: string;
+  size: number;
+  type: string;
+}
+
+function InputFileAsync ({ accept, className, clearContent, help, isDisabled, isError = false, label, onChange, placeholder, t, withEllipsis, withLabel }: Props): React.ReactElement<Props> {
+  const dropRef = createRef<DropzoneRef>();
+  const [file, setFile] = useState<FileState | undefined>();
+
+  const _onDrop = (files: File[]): void => {
+    if (!files.length) return;
+    const blob = files[0];
+    onChange && onChange(blob);
+    dropRef && setFile({
+      ...blob
+    });
+  };
+
+  const dropZone = (
+    <Dropzone
+      accept={accept}
+      disabled={isDisabled}
+      multiple={false}
+      ref={dropRef}
+      onDrop={_onDrop}
+    >
+      {({ getRootProps, getInputProps }): JSX.Element => (
+        <div {...getRootProps({ className: classes('ui--InputFile', isError ? 'error' : '', className) })} >
+          <input {...getInputProps()} />
+          <em className='label' >
+            {
+              !file || clearContent
+                ? placeholder || t('click to select or drag and drop the file here')
+                : placeholder || t('{{name}} ({{size}} bytes)', {
+                  replace: {
+                    name: file.name,
+                    size: formatNumber(file.size)
+                  }
+                })
+            }
+          </em>
+        </div>
+      )}
+    </Dropzone>
+  );
+
+  return label
+    ? (
+      <Labelled
+        help={help}
+        label={label}
+        withEllipsis={withEllipsis}
+        withLabel={withLabel}
+      >
+        {dropZone}
+      </Labelled>
+    )
+    : dropZone;
+}
+
+export default translate(
+  styled(InputFileAsync)`
+    background: #fff;
+    border: 1px solid rgba(34, 36, 38, 0.15);
+    border-radius: 0.28571429rem;
+    font-size: 1rem;
+    margin: 0.25rem 0;
+    padding: 1rem;
+    width: 100% !important;
+
+    &.error {
+      background: #fff6f6;
+      border-color: #e0b4b4;
+    }
+
+    &:hover {
+      background: #fefefe;
+      cursor: pointer;
+    }
+
+    .label {
+      color: rgba(0, 0, 0, .6);
+    }
+  `
+);

packages/react-components/src/index.tsx

@@ -46,6 +46,7 @@ export { default as InputConsts } from './InputConsts';
 export { default as InputError } from './InputError';
 export { default as InputExtrinsic } from './InputExtrinsic';
 export { default as InputFile } from './InputFile';
+export { default as InputFileAsync } from './InputFileAsync';
 export { default as InputNumber } from './InputNumber';
 export { default as InputRpc } from './InputRpc';
 export { default as InputStorage } from './InputStorage';