Skip to content

Commit

Permalink
feat(datastore-fs): fast-write-atomic -> steno
Browse files Browse the repository at this point in the history
  • Loading branch information
SgtPooki committed Dec 14, 2023
1 parent 4e39c60 commit 42f2943
Show file tree
Hide file tree
Showing 2 changed files with 17 additions and 11 deletions.
2 changes: 1 addition & 1 deletion packages/datastore-fs/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -136,8 +136,8 @@
"dep-check": "aegir dep-check"
},
"dependencies": {
"@sgtpooki/steno-patched": "^3.1.1",
"datastore-core": "^9.0.0",
"fast-write-atomic": "^0.2.0",
"interface-datastore": "^8.0.0",
"interface-store": "^5.0.0",
"it-glob": "^2.0.1",
Expand Down
26 changes: 16 additions & 10 deletions packages/datastore-fs/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,10 @@

import fs from 'node:fs/promises'
import path from 'node:path'
import { promisify } from 'util'
import { Writer as StenoWriter } from '@sgtpooki/steno-patched'
import {
BaseDatastore, Errors
} from 'datastore-core'
// @ts-expect-error no types
import fwa from 'fast-write-atomic'
import {
Key, type KeyQuery, type Pair, type Query
} from 'interface-datastore'
Expand All @@ -28,21 +26,20 @@ import map from 'it-map'
import parallel from 'it-parallel-batch'
import type { AwaitIterable } from 'interface-store'

const writeAtomic = promisify(fwa)

/**
* Write a file atomically
*/
async function writeFile (path: string, contents: Uint8Array): Promise<void> {
async function writeFile (writer: StenoWriter, file: string, contents: Uint8Array): Promise<void> {
try {
await writeAtomic(path, contents)
await writer.write(contents)
} catch (err: any) {
if (err.code === 'EPERM' && err.syscall === 'rename') {
// fast-write-atomic writes a file to a temp location before renaming it.
// steno writes a file to a temp location before renaming it.
// On Windows, if the final file already exists this error is thrown.
// No such error is thrown on Linux/Mac
// Make sure we can read & write to this file
await fs.access(path, fs.constants.F_OK | fs.constants.W_OK)
// 2023-12-14: Is this still needed with steno?
await fs.access(file, fs.constants.F_OK | fs.constants.W_OK)

// The file was created by another context - this means there were
// attempts to write the same block by two different function calls
Expand Down Expand Up @@ -76,6 +73,7 @@ export class FsDatastore extends BaseDatastore {
private readonly deleteManyConcurrency: number
private readonly getManyConcurrency: number
private readonly putManyConcurrency: number
private readonly writers = new Map<string, StenoWriter>()

constructor (location: string, init: FsDatastoreInit = {}) {
super()
Expand Down Expand Up @@ -156,7 +154,15 @@ export class FsDatastore extends BaseDatastore {
await fs.mkdir(parts.dir, {
recursive: true
})
await writeFile(parts.file, val)
const filePath = parts.file
let writer = this.writers.get(filePath)
if (writer == null) {
writer = new StenoWriter(filePath)
this.writers.set(filePath, writer)
}

await writeFile(writer, filePath, val)
this.writers.delete(filePath)

return key
} catch (err: any) {
Expand Down

0 comments on commit 42f2943

Please sign in to comment.