Compare commits

...
3 Commits
Author SHA1 Message Date
dependabot[bot]anddependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> 2acec9c0dc Bump @types/node from 17.0.45 to 18.0.2 (#140)
Bumps [@types/node](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/HEAD/types/node) from 17.0.45 to 18.0.2.
- [Release notes](https://github.com/DefinitelyTyped/DefinitelyTyped/releases)
- [Commits](https://github.com/DefinitelyTyped/DefinitelyTyped/commits/HEAD/types/node)

---
updated-dependencies:
- dependency-name: "@types/node"
  dependency-type: direct:development
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2022-07-06 19:04:46 +02:00
Jimmy Wärting 20ee08f33e feat: new way of creating temporary files backed up by fs (#137)
* feat: new way of creating temporary files backed up by fs

* add readme
2022-07-06 19:01:17 +02:00
Jimmy Wärting f077471a23 fix: avoid reading files when the size is empty (#134)
* fix: avoid reading files when the size is empty

* also use stat.size for checking if it is modified

* bump minor version number

* Don't push empty chunks into the array (creates faster reads)

* reverted back to avoid making a if check
2022-03-10 15:11:10 +01:00
5 changed files with 154 additions and 11 deletions
+30
View File
@@ -87,6 +87,34 @@ console.log(blob.size) // ~4 GiB
`blobFrom|blobFromSync|fileFrom|fileFromSync(path, [mimetype])`
### Creating a temporary file on the disk
(requires [FinalizationRegistry] - node v14.6)
When using both `createTemporaryBlob` and `createTemporaryFile`
then you will write data to the temporary folder in their respective OS.
The arguments can be anything that [fsPromises.writeFile] supports. NodeJS
v14.17.0+ also supports writing (async)Iterable streams and passing in a
AbortSignal, so both NodeJS stream and whatwg streams are supported. When the
file have been written it will return a Blob/File handle with a references to
this temporary location on the disk. When you no longer have a references to
this Blob/File anymore and it have been GC then it will automatically be deleted.
This files are also unlinked upon exiting the process.
```js
import { createTemporaryBlob, createTemporaryFile } from 'fetch-blob/from.js'
const req = new Request('https://httpbin.org/image/png')
const res = await fetch(req)
const type = res.headers.get('content-type')
const signal = req.signal
let blob = await createTemporaryBlob(res.body, { type, signal })
// const file = createTemporaryBlob(res.body, 'img.png', { type, signal })
blob = undefined // loosing references will delete the file from disk
```
`createTemporaryBlob(data, { type, signal })`
`createTemporaryFile(data, FileName, { type, signal, lastModified })`
### Creating Blobs backed up by other async sources
Our Blob & File class are more generic then any other polyfills in the way that it can accept any blob look-a-like item
An example of this is that our blob implementation can be constructed with parts coming from [BlobDataItem](https://github.com/node-fetch/fetch-blob/blob/8ef89adad40d255a3bbd55cf38b88597c1cd5480/from.js#L32) (aka a filepath) or from [buffer.Blob](https://nodejs.org/api/buffer.html#buffer_new_buffer_blob_sources_options), It dose not have to implement all the methods - just enough that it can be read/understood by our Blob implementation. The minium requirements is that it has `Symbol.toStringTag`, `size`, `slice()` and either a `stream()` or a `arrayBuffer()` method. If you then wrap it in our Blob or File `new Blob([blobDataItem])` then you get all of the other methods that should be implemented in a blob or file
@@ -104,3 +132,5 @@ See the [MDN documentation](https://developer.mozilla.org/en-US/docs/Web/API/Blo
[install-size-image]: https://flat.badgen.net/packagephobia/install/fetch-blob
[install-size-url]: https://packagephobia.now.sh/result?p=fetch-blob
[fs-blobs]: https://github.com/nodejs/node/issues/37340
[fsPromises.writeFile]: https://nodejs.org/dist/latest-v18.x/docs/api/fs.html#fspromiseswritefilefile-data-options
[FinalizationRegistry]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/FinalizationRegistry
+70 -6
View File
@@ -1,11 +1,20 @@
import { statSync, createReadStream, promises as fs } from 'node:fs'
import { basename } from 'node:path'
import {
realpathSync,
statSync,
rmdirSync,
createReadStream,
promises as fs
} from 'node:fs'
import { basename, sep, join } from 'node:path'
import { tmpdir } from 'node:os'
import process from 'node:process'
import DOMException from 'node-domexception'
import File from './file.js'
import Blob from './index.js'
const { stat } = fs
const { stat, mkdtemp } = fs
let i = 0, tempDir, registry
/**
* @param {string} path filepath on the disk
@@ -49,6 +58,42 @@ const fromFile = (stat, path, type = '') => new File([new BlobDataItem({
start: 0
})], basename(path), { type, lastModified: stat.mtimeMs })
/**
* Creates a temporary blob backed by the filesystem.
* NOTE: requires node.js v14 or higher to use FinalizationRegistry
*
* @param {*} data Same as fs.writeFile data
* @param {BlobPropertyBag & {signal?: AbortSignal}} options
* @param {AbortSignal} [signal] in case you wish to cancel the write operation
* @returns {Promise<Blob>}
*/
const createTemporaryBlob = async (data, {signal, type} = {}) => {
registry = registry || new FinalizationRegistry(fs.unlink)
tempDir = tempDir || await mkdtemp(realpathSync(tmpdir()) + sep)
const id = `${i++}`
const destination = join(tempDir, id)
if (data instanceof ArrayBuffer) data = new Uint8Array(data)
await fs.writeFile(destination, data, { signal })
const blob = await blobFrom(destination, type)
registry.register(blob, destination)
return blob
}
/**
* Creates a temporary File backed by the filesystem.
* Pretty much the same as constructing a new File(data, name, options)
*
* NOTE: requires node.js v14 or higher to use FinalizationRegistry
* @param {*} data
* @param {string} name
* @param {FilePropertyBag & {signal?: AbortSignal}} opts
* @returns {Promise<File>}
*/
const createTemporaryFile = async (data, name, opts) => {
const blob = await createTemporaryBlob(data)
return new File([blob], name, opts)
}
/**
* This is a blob backed up by a file on the disk
* with minium requirement. Its wrapped around a Blob as a blobPart
@@ -65,6 +110,9 @@ class BlobDataItem {
this.#start = options.start
this.size = options.size
this.lastModified = options.lastModified
this.originalSize = options.originalSize === undefined
? options.size
: options.originalSize
}
/**
@@ -75,16 +123,19 @@ class BlobDataItem {
return new BlobDataItem({
path: this.#path,
lastModified: this.lastModified,
originalSize: this.originalSize,
size: end - start,
start: this.#start + start
})
}
async * stream () {
const { mtimeMs } = await stat(this.#path)
if (mtimeMs > this.lastModified) {
const { mtimeMs, size } = await stat(this.#path)
if (mtimeMs > this.lastModified || this.originalSize !== size) {
throw new DOMException('The requested file could not be read, typically due to permission problems that have occurred after a reference to a file was acquired.', 'NotReadableError')
}
yield * createReadStream(this.#path, {
start: this.#start,
end: this.#start + this.size - 1
@@ -96,5 +147,18 @@ class BlobDataItem {
}
}
process.once('exit', () => {
tempDir && rmdirSync(tempDir, { recursive: true })
})
export default blobFromSync
export { File, Blob, blobFrom, blobFromSync, fileFrom, fileFromSync }
export {
Blob,
blobFrom,
blobFromSync,
createTemporaryBlob,
File,
fileFrom,
fileFromSync,
createTemporaryFile
}
+6 -2
View File
@@ -83,8 +83,12 @@ const _Blob = class Blob {
part = encoder.encode(`${element}`)
}
this.#size += ArrayBuffer.isView(part) ? part.byteLength : part.size
this.#parts.push(part)
const size = ArrayBuffer.isView(part) ? part.byteLength : part.size
// Avoid pushing empty parts into the array to better GC them
if (size) {
this.#size += size
this.#parts.push(part)
}
}
this.#endings = `${options.endings === undefined ? 'transparent' : options.endings}`
+2 -2
View File
@@ -1,6 +1,6 @@
{
"name": "fetch-blob",
"version": "3.1.4",
"version": "3.1.5",
"description": "Blob & File implementation in Node.js, originally from node-fetch.",
"main": "index.js",
"type": "module",
@@ -35,7 +35,7 @@
},
"homepage": "https://github.com/node-fetch/fetch-blob#readme",
"devDependencies": {
"@types/node": "^17.0.9",
"@types/node": "^18.0.2",
"c8": "^7.11.0",
"typescript": "^4.5.4"
},
+46 -1
View File
@@ -3,7 +3,14 @@
import fs from 'node:fs'
import buffer from 'node:buffer'
import syncBlob, { blobFromSync, blobFrom, fileFromSync, fileFrom } from '../from.js'
import syncBlob, {
blobFromSync,
blobFrom,
fileFromSync,
fileFrom,
createTemporaryBlob,
createTemporaryFile
} from '../from.js'
const license = fs.readFileSync('./LICENSE')
@@ -189,6 +196,44 @@ promise_test(async () => {
assert_equals(await (await fileFrom('./LICENSE')).text(), license.toString())
}, 'blob part backed up by filesystem slice correctly')
promise_test(async () => {
let blob
// Can construct a temporary blob from a string
blob = await createTemporaryBlob(license.toString())
assert_equals(await blob.text(), license.toString())
// Can construct a temporary blob from a async iterator
blob = await createTemporaryBlob(blob.stream())
assert_equals(await blob.text(), license.toString())
// Can construct a temporary file from a arrayBuffer
blob = await createTemporaryBlob(await blob.arrayBuffer())
assert_equals(await blob.text(), license.toString())
// Can construct a temporary file from a arrayBufferView
blob = await createTemporaryBlob(await blob.arrayBuffer().then(ab => new Uint8Array(ab)))
assert_equals(await blob.text(), license.toString())
// Can specify a mime type
blob = await createTemporaryBlob('abc', { type: 'text/plain' })
assert_equals(blob.type, 'text/plain')
// Can create files too
let file = await createTemporaryFile('abc', 'abc.txt', {
type: 'text/plain',
lastModified: 123
})
assert_equals(file.name, 'abc.txt')
assert_equals(file.size, 3)
assert_equals(file.lastModified, 123)
}, 'creating temporary blob/file backed up by filesystem')
promise_test(async () => {
fs.writeFileSync('temp', '')
await blobFromSync('./temp').text()
fs.unlinkSync('./temp')
}, 'can read empty files')
test(async () => {
const blob = blobFromSync('./LICENSE')
await new Promise(resolve => setTimeout(resolve, 2000))