diff --git a/package.json b/package.json
index 61e7c0245..f4225950c 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
{
"name": "bldrs",
- "version": "1.0.1103",
+ "version": "1.0.1101",
"main": "src/index.jsx",
"license": "AGPL-3.0",
"homepage": "https://github.com/bldrs-ai/Share",
@@ -12,7 +12,7 @@
"build-conway": "yarn clean && yarn build-share-conway && yarn build-cosmos",
"build-webifc": "yarn clean && yarn build-share-webifc && yarn build-cosmos",
"build-cosmos": "shx mkdir -p docs ; shx rm -rf docs/cosmos; cosmos-export --config .cosmos.config.json && shx mv cosmos-export docs/cosmos",
- "build-share": "yarn update-version && node tools/esbuild/build.js && shx mkdir -p docs/static/js && shx cp src/OPFS/OPFS.worker.js docs/",
+ "build-share": "yarn update-version && node tools/esbuild/build.js && shx mkdir -p docs/static/js && shx cp src/OPFS/OPFS.worker.js docs/ && shx cp src/net/github/Cache.js docs/",
"build-share-conway": "run-script-os",
"build-share-conway:win32": "set USE_WEBIFC_SHIM=true&& yarn build-share",
"build-share-conway:linux:darwin": "USE_WEBIFC_SHIM=true yarn build-share",
diff --git a/src/Components/Open/SaveModelControl.jsx b/src/Components/Open/SaveModelControl.jsx
index eca282a87..4c35cbc07 100644
--- a/src/Components/Open/SaveModelControl.jsx
+++ b/src/Components/Open/SaveModelControl.jsx
@@ -232,63 +232,67 @@ function SaveModelDialog({isDialogDisplayed, setIsDialogDisplayed, navigate, org
justifyContent='center'
alignItems='center'
>
- {!isAuthenticated ?
-
- :
-
-
- Projects
-
-
-
- {requestCreateFolder && (
-
- setCreateFolderName(e.target.value)}
- data-testid='CreateFolderId'
- sx={{flexGrow: 1}}
- onKeyDown={(e) => {
- // Stops the event from propagating up to parent elements
- e.stopPropagation()
- }}
- />
- setRequestCreateFolder(false)}
- size='small'
- >
-
-
-
- )}
- setSelectedFileName(e.target.value)}
- onKeyDown={(e) => e.stopPropagation()}
- sx={{marginBottom: '.5em'}}
- data-testid='CreateFileId'
- />
-
- }
+ {!isAuthenticated ? (
+
+ ) : (
+ file instanceof File && (
+
+ Projects
+
+
+
+ {requestCreateFolder && (
+
+ setCreateFolderName(e.target.value)}
+ data-testid='CreateFolderId'
+ sx={{flexGrow: 1}}
+ onKeyDown={(e) => {
+ // Stops the event from propagating up to parent elements
+ e.stopPropagation()
+ }}
+ />
+ setRequestCreateFolder(false)}
+ size='small'
+ >
+
+
+
+ )}
+ setSelectedFileName(e.target.value)}
+ onKeyDown={(e) => e.stopPropagation()}
+ sx={{marginBottom: '.5em'}}
+ data-testid='CreateFileId'
+ />
+
+ )
+ )}
)
diff --git a/src/Components/Open/SaveModelControl.test.jsx b/src/Components/Open/SaveModelControl.test.jsx
index af84a3bc9..b88d390f8 100644
--- a/src/Components/Open/SaveModelControl.test.jsx
+++ b/src/Components/Open/SaveModelControl.test.jsx
@@ -33,7 +33,7 @@ describe('SaveModelControl', () => {
expect(loginText).toBeInTheDocument()
})
- it('Renders file selector if the user is logged in', async () => {
+ it('Renders file selector if the user is logged in', async () => {
mockedUseAuth0.mockReturnValue(mockedUserLoggedIn)
const {getByTestId} = render()
const saveControlButton = getByTestId('control-button-save')
diff --git a/src/Components/Versions/VersionsPanel.fixture.js b/src/Components/Versions/VersionsPanel.fixture.js
index 02a4b76bf..d69666ac7 100644
--- a/src/Components/Versions/VersionsPanel.fixture.js
+++ b/src/Components/Versions/VersionsPanel.fixture.js
@@ -1,4 +1,4 @@
-const RAW_GIT_PROXY_URL = process.env.RAW_GIT_PROXY_URL
+const RAW_GIT_PROXY_URL_NEW = process.env.RAW_GIT_PROXY_URL_NEW
export const MOCK_MODEL_PATH_GIT = {
@@ -7,5 +7,5 @@ export const MOCK_MODEL_PATH_GIT = {
branch: 'main',
filepath: '/ZGRAGGEN.ifc',
eltPath: '',
- gitpath: `${RAW_GIT_PROXY_URL}/user2/Schneestock-Public/main/ZGRAGGEN.ifc`,
+ gitpath: `${RAW_GIT_PROXY_URL_NEW}/user2/Schneestock-Public/main/ZGRAGGEN.ifc`,
}
diff --git a/src/Containers/CadView.jsx b/src/Containers/CadView.jsx
index 827727545..1d0224f75 100644
--- a/src/Containers/CadView.jsx
+++ b/src/Containers/CadView.jsx
@@ -12,11 +12,10 @@ import ElementGroup from '../Components/ElementGroup'
import HelpControl from '../Components/Help/HelpControl'
import {useIsMobile} from '../Components/Hooks'
import LoadingBackdrop from '../Components/LoadingBackdrop'
-import {getModelFromOPFS, downloadToOPFS} from '../OPFS/utils'
+import {getModelFromOPFS, downloadToOPFS, downloadModel} from '../OPFS/utils'
import usePlaceMark from '../hooks/usePlaceMark'
import * as Analytics from '../privacy/analytics'
import useStore from '../store/useStore'
-import {getLatestCommitHash} from '../net/github/Commits'
// TODO(pablo): use ^^ instead of this
import {parseGitHubPath} from '../utils/location'
import {getParentPathIdsForElement, setupLookupAndParentLinks} from '../utils/TreeUtils'
@@ -32,7 +31,7 @@ import ControlsGroupAndDrawer from './ControlsGroupAndDrawer'
import OperationsGroupAndDrawer from './OperationsGroupAndDrawer'
import ViewerContainer from './ViewerContainer'
import {elementSelection} from './selection'
-import {getFinalUrl} from './urls'
+import {getFinalDownloadData} from './urls'
import {initViewer} from './viewer'
@@ -252,8 +251,15 @@ export default function CadView({
// NB: for LFS targets, this will now be media.githubusercontent.com, so
// don't use for further API endpoint construction.
- const ifcURL = (uploadedFile || filepath.indexOf('/') === 0) ?
- filepath : await getFinalUrl(filepath, accessToken)
+ let ifcURL; let shaHash
+
+ if (uploadedFile || filepath.indexOf('/') === 0) {
+ ifcURL = filepath
+ shaHash = ''
+ } else {
+ [ifcURL, shaHash] = await getFinalDownloadData(filepath, accessToken, isOpfsAvailable)
+ }
+
const isCamHashSet = onHash(location, viewer.IFC.context.ifcCamera.cameraControls)
@@ -332,29 +338,21 @@ export default function CadView({
} else {
// TODO(pablo): probably already available in this scope, or use
// parseGitHubRepositoryURL instead.
+ // eslint-disable-next-line no-unused-vars
const {isPublic, owner, repo, branch, filePath} = parseGitHubPath(new URL(gitpath).pathname)
- const commitHash = isPublic ?
- await getLatestCommitHash(owner, repo, filePath, '', branch) :
- await getLatestCommitHash(owner, repo, filePath, accessToken, branch)
-
- if (commitHash === null) {
- // downloadToOpfs below will error out as well.
- debug().error(
- `Error obtaining commit hash for: ` +
- `owner:${owner}, repo:${repo}, filePath:${filePath}, branch:${branch} ` +
- `accessToken (present?):${accessToken ? 'true' : 'false'}`)
- }
- const file = await downloadToOPFS(
+ const file = await downloadModel(
navigate,
appPrefix,
handleBeforeUnload,
ifcURL,
+ shaHash,
filePath,
- commitHash,
+ accessToken,
owner,
repo,
branch,
+ setOpfsFile,
(progressEvent) => {
if (Number.isFinite(progressEvent.receivedLength)) {
const loadedBytes = progressEvent.receivedLength
@@ -365,21 +363,17 @@ export default function CadView({
}
})
- if (file instanceof File) {
- setOpfsFile(file)
- } else {
- debug().error('Retrieved object is not of type File.')
+ if (file) {
+ loadedModel = await viewer.loadIfcFile(
+ file,
+ !isCamHashSet,
+ (error) => {
+ debug().log('CadView#loadIfc$onError: ', error)
+ // TODO(pablo): error modal.
+ setIsModelLoading(false)
+ setErrorPath(filePath)
+ }, customViewSettings)
}
-
- loadedModel = await viewer.loadIfcFile(
- file,
- !isCamHashSet,
- (error) => {
- debug().log('CadView#loadIfc$onError: ', error)
- // TODO(pablo): error modal.
- setIsModelLoading(false)
- setErrorPath(filePath)
- }, customViewSettings)
}
if (loadedModel) {
diff --git a/src/Containers/CadView.test.jsx b/src/Containers/CadView.test.jsx
index f8dd9311c..49af2c0ba 100644
--- a/src/Containers/CadView.test.jsx
+++ b/src/Containers/CadView.test.jsx
@@ -49,7 +49,19 @@ jest.mock('../OPFS/utils', () => {
return {
...actualUtils, // Preserve other exports from the module
downloadToOPFS: jest.fn().mockImplementation(() => {
- // Read the file content from disk (consider using async read in real use-cases)
+ // Read the file content from disk
+ const fileContent = fs.readFileSync(path.join(__dirname, './index.ifc'), 'utf8')
+
+ const uint8Array = new Uint8Array(fileContent)
+ const blob = new Blob([uint8Array])
+
+ // The lastModified property is optional, and can be omitted or set to Date.now() if needed
+ const file = new FileMock([blob], 'index.ifc', {type: 'text/plain', lastModified: Date.now()})
+ // Return the mocked File in a promise if it's an async function
+ return Promise.resolve(file)
+ }),
+ downloadModel: jest.fn().mockImplementation(() => {
+ // Read the file content from disk
const fileContent = fs.readFileSync(path.join(__dirname, './index.ifc'), 'utf8')
const uint8Array = new Uint8Array(fileContent)
diff --git a/src/Containers/urls.js b/src/Containers/urls.js
index 8072852aa..190010226 100644
--- a/src/Containers/urls.js
+++ b/src/Containers/urls.js
@@ -1,4 +1,4 @@
-import {getDownloadUrl} from '../net/github/Files'
+import {getDownloadUrl, getPathContents} from '../net/github/Files'
import {parseGitHubRepositoryUrl} from '../net/github/utils'
@@ -12,7 +12,7 @@ export async function getFinalUrl(url, accessToken) {
switch (u.host.toLowerCase()) {
case 'github.com':
if (!accessToken) {
- const proxyUrl = new URL(process.env.RAW_GIT_PROXY_URL)
+ const proxyUrl = new URL(process.env.RAW_GIT_PROXY_URL_NEW)
// Replace the protocol, host, and hostname in the target
u.protocol = proxyUrl.protocol
@@ -39,6 +39,42 @@ export async function getFinalUrl(url, accessToken) {
}
}
+/**
+ *
+ */
+export async function getFinalDownloadData(url, accessToken, isOpfsAvailable) {
+ const u = new URL(url)
+
+ switch (u.host.toLowerCase()) {
+ case 'github.com':
+ if (!accessToken) {
+ const proxyUrl = new URL(isOpfsAvailable ? process.env.RAW_GIT_PROXY_URL_NEW : process.env.RAW_GIT_PROXY_URL)
+
+ // Replace the protocol, host, and hostname in the target
+ u.protocol = proxyUrl.protocol
+ u.host = proxyUrl.host
+ u.hostname = proxyUrl.hostname
+
+ // If the port is specified, replace it in the target URL
+ if (proxyUrl.port) {
+ u.port = proxyUrl.port
+ }
+
+ // If there's a path, *and* it's not just the root, then prepend it to the target URL
+ if (proxyUrl.pathname && proxyUrl.pathname !== '/') {
+ u.pathname = proxyUrl.pathname + u.pathname
+ }
+
+ return [u.toString(), '']
+ }
+
+ return await getGitHubPathContents(url, accessToken)
+
+ default:
+ return [url, '']
+ }
+}
+
/**
* @param {string} url
@@ -58,3 +94,20 @@ async function getGitHubDownloadUrl(url, accessToken) {
)
return downloadUrl
}
+
+/**
+ *
+ */
+async function getGitHubPathContents(url, accessToken) {
+ const repo = parseGitHubRepositoryUrl(url)
+ const [downloadUrl, sha] = await getPathContents(
+ {
+ orgName: repo.owner,
+ name: repo.repository,
+ },
+ repo.path,
+ repo.ref,
+ accessToken,
+ )
+ return [downloadUrl, sha]
+}
diff --git a/src/Containers/urls.test.js b/src/Containers/urls.test.js
index 861ac4bb1..6e581b14f 100644
--- a/src/Containers/urls.test.js
+++ b/src/Containers/urls.test.js
@@ -18,9 +18,9 @@ describe.only('With environment variables', () => {
it.only('getFinalUrl', async () => {
- expect(await getFinalUrl('https://github.com/')).toStrictEqual(`${testEnvVars.RAW_GIT_PROXY_URL}/`)
+ expect(await getFinalUrl('https://github.com/')).toStrictEqual(`${testEnvVars.RAW_GIT_PROXY_URL_NEW}/`)
- process.env.RAW_GIT_PROXY_URL = 'https://rawgit.bldrs.dev'
- expect(await getFinalUrl('https://github.com/')).toStrictEqual('https://rawgit.bldrs.dev/')
+ process.env.RAW_GIT_PROXY_URL = 'https://rawgit.bldrs.dev.jest/model'
+ expect(await getFinalUrl('https://github.com/')).toStrictEqual('https://rawgit.bldrs.dev.jest/model/')
})
})
diff --git a/src/OPFS/OPFS.worker.js b/src/OPFS/OPFS.worker.js
index 5f35a8731..dab7ed128 100644
--- a/src/OPFS/OPFS.worker.js
+++ b/src/OPFS/OPFS.worker.js
@@ -1,8 +1,25 @@
// opfsWorker.js
+let GITHUB_BASE_URL_AUTHENTICATED = null
+let GITHUB_BASE_URL_UNAUTHENTICATED = null
+
+/**
+ * @global
+ * @typedef {object} CacheModule
+ * @property {function(string): Promise} checkCacheRaw Function to check the cache.
+ */
+
+/* global importScripts, CacheModule */
+importScripts('./Cache.js')
self.addEventListener('message', async (event) => {
try {
- if (event.data.command === 'writeObjectURLToFile') {
+ if (event.data.command === 'initializeWorker') {
+ const {GITHUB_BASE_URL_AUTHED, GITHUB_BASE_URL_UNAUTHED} =
+ assertValues(event.data, ['GITHUB_BASE_URL_AUTHED', 'GITHUB_BASE_URL_UNAUTHED'])
+
+ GITHUB_BASE_URL_AUTHENTICATED = GITHUB_BASE_URL_AUTHED
+ GITHUB_BASE_URL_UNAUTHENTICATED = GITHUB_BASE_URL_UNAUTHED
+ } else if (event.data.command === 'writeObjectURLToFile') {
const {objectUrl, fileName} =
assertValues(event.data, ['objectUrl', 'fileName'])
await writeFileToOPFS(objectUrl, fileName)
@@ -28,6 +45,11 @@ self.addEventListener('message', async (event) => {
assertValues(event.data,
['objectUrl', 'commitHash', 'owner', 'repo', 'branch', 'onProgress', 'originalFilePath'])
await downloadModelToOPFS(objectUrl, commitHash, originalFilePath, owner, repo, branch, onProgress)
+ } else if (event.data.command === 'downloadModel') {
+ const {objectUrl, shaHash, originalFilePath, owner, repo, branch, accessToken, onProgress} =
+ assertValues(event.data,
+ ['objectUrl', 'shaHash', 'originalFilePath', 'owner', 'repo', 'branch', 'accessToken', 'onProgress'])
+ await downloadModel(objectUrl, shaHash, originalFilePath, owner, repo, branch, accessToken, onProgress)
} else if (event.data.command === 'doesFileExist') {
const {commitHash, originalFilePath, owner, repo, branch} =
assertValues(event.data,
@@ -103,11 +125,585 @@ async function deleteAllEntries(dirHandle) {
}
}
+// Function to fetch the latest commit hash
+/**
+ *
+ */
+async function fetchLatestCommitHash(baseURL, owner, repo, filePath, accessToken, branch) {
+ const url = `${baseURL}/repos/${owner}/${repo}/commits?sha=${branch}&path=${filePath}`
+ const headers = accessToken ? {Authorization: `Bearer ${accessToken}`} : {}
+
+ const response = await fetch(url, {headers})
+
+ if (!response.ok) {
+ throw new Error(`Failed to fetch commits: ${response.statusText}`)
+ }
+
+ const data = await response.json()
+
+ if (data.length === 0) {
+ throw new Error('No commits found for the specified file.')
+ }
+
+ const latestCommitHash = data[0].sha
+ // eslint-disable-next-line no-console
+ console.log(`The latest commit hash for the file is: ${latestCommitHash}`)
+ return latestCommitHash
+}
+
+/**
+ *
+ */
+async function fetchRGHUC(modelUrl) {
+ try {
+ // fetch model
+ const modelResponse = await fetch(modelUrl)
+
+ if (!modelResponse.ok) {
+ throw new Error('Failed to make model request')
+ }
+
+ return modelResponse
+ } catch (error) {
+ // eslint-disable-next-line no-console
+ console.error('Error:', error)
+ }
+}
+
+/**
+ * Fetch the final URL and make a HEAD request
+ */
+async function fetchAndHeadRequest(jsonUrl, etag_ = null) {
+ try {
+ const STATUS_CACHED = 304
+ // Step 1: Fetch the JSON response with ETag header if provided
+ const fetchOptions = etag_ ? {headers: {ETag: etag_}} : {}
+ const proxyResponse = await fetch(jsonUrl, fetchOptions)
+
+ if (proxyResponse.status === STATUS_CACHED) {
+ // eslint-disable-next-line no-console
+ console.log('Cached version is valid')
+ return null
+ }
+
+ if (!proxyResponse.ok) {
+ throw new Error('Failed to fetch JSON response')
+ }
+
+ // clone response
+ const clonedResponse = proxyResponse.clone()
+
+ const json = await clonedResponse.json()
+
+ // eslint-disable-next-line no-unused-vars
+ const {_, etag, finalURL} = json
+
+ // Step 3: fetch model
+ const modelResponse = await fetch(finalURL)
+
+ if (!modelResponse.ok) {
+ throw new Error('Failed to make model request')
+ }
+
+ return {proxyResponse, modelResponse, etag}
+ } catch (error) {
+ // eslint-disable-next-line no-console
+ console.error('Error:', error)
+ }
+}
+
+/* eslint-disable jsdoc/no-undefined-types */
+/**
+ * Computes the Git blob SHA-1 hash for a given File.
+ *
+ *
+ * @param {FileSystemFileHandle} file - The File object to compute the SHA-1 hash for.
+ * @return {Promise} The computed SHA-1 hash in hexadecimal format.
+ */
+async function computeGitBlobSha1FromHandle(modelBlobFileHandle) {
+ // Create FileSystemSyncAccessHandle on the file
+ const blobAccessHandle = await modelBlobFileHandle.createSyncAccessHandle()
+
+ try {
+ // Get the size of the file
+ const fileSize = await blobAccessHandle.getSize()
+
+ // Read the entire file into an ArrayBuffer
+ const fileArrayBuffer = new ArrayBuffer(fileSize)
+ await blobAccessHandle.read(fileArrayBuffer, {at: 0})
+
+ // Create the Git blob header
+ const header = `blob ${fileSize}\u0000`
+ const headerBuffer = new TextEncoder().encode(header)
+
+ // Create a new ArrayBuffer to hold the header and the file data
+ const combinedBuffer = new Uint8Array(headerBuffer.byteLength + fileArrayBuffer.byteLength)
+
+ // Copy the header and file data into the combined buffer
+ combinedBuffer.set(headerBuffer, 0)
+ combinedBuffer.set(new Uint8Array(fileArrayBuffer), headerBuffer.byteLength)
+
+ // Compute the SHA-1 hash
+ const hashBuffer = await crypto.subtle.digest('SHA-1', combinedBuffer)
+
+ // Convert the hash to a hexadecimal string
+ const hashArray = Array.from(new Uint8Array(hashBuffer))
+ const HEX_IDENTIFIER = 16
+ const hashHex = hashArray.map((b) => b.toString(HEX_IDENTIFIER).padStart(2, '0')).join('')
+
+ return hashHex
+ } finally {
+ // Close the handle
+ await blobAccessHandle.close()
+ }
+}
+
+/* eslint-enable jsdoc/no-undefined-types */
+
+/**
+ * Computes the Git blob SHA-1 hash for a given File.
+ *
+ * @param {File} file - The File object to compute the SHA-1 hash for.
+ * @return {Promise} The computed SHA-1 hash in hexadecimal format.
+ */
+async function computeGitBlobSha1FromFile(file) {
+ // Get the size of the file
+ const fileSize = file.size
+
+ // Read the entire file into an ArrayBuffer
+ const fileArrayBuffer = await file.arrayBuffer()
+
+ // Create the Git blob header
+ const header = `blob ${fileSize}\u0000`
+ const headerBuffer = new TextEncoder().encode(header)
+
+ // Create a new ArrayBuffer to hold the header and the file data
+ const combinedBuffer = new Uint8Array(headerBuffer.byteLength + fileArrayBuffer.byteLength)
+
+ // Copy the header and file data into the combined buffer
+ combinedBuffer.set(headerBuffer, 0)
+ combinedBuffer.set(new Uint8Array(fileArrayBuffer), headerBuffer.byteLength)
+
+ // Compute the SHA-1 hash
+ const hashBuffer = await crypto.subtle.digest('SHA-1', combinedBuffer)
+
+ // Convert the hash to a hexadecimal string
+ const hashArray = Array.from(new Uint8Array(hashBuffer))
+ const HEX_IDENTIFIER = 16
+ const hashHex = hashArray.map((b) => b.toString(HEX_IDENTIFIER).padStart(2, '0')).join('')
+
+ return hashHex
+}
+
+
+// Function to write temporary file to OPFS (Origin Private File System)
+/**
+ *
+ */
+async function writeTemporaryFileToOPFS(response, originalFilePath, _etag, onProgress) {
+ const opfsRoot = await navigator.storage.getDirectory()
+ let modelDirectoryHandle = null
+ let modelBlobFileHandle = null
+
+ // lets see if our etag matches
+ // Get file handle for file blob
+ try {
+ [modelDirectoryHandle, modelBlobFileHandle] = await
+ retrieveFileWithPathNew(opfsRoot, originalFilePath, _etag, null, false)
+
+ if (modelBlobFileHandle !== undefined) {
+ const blobFile = await modelBlobFileHandle.getFile()
+
+ self.postMessage({completed: true, event: 'download', file: blobFile})
+ return [modelDirectoryHandle, modelBlobFileHandle]
+ }
+ } catch (error) {
+ // expected if file not found
+ }
+ let blobAccessHandle = null
+
+ try {
+ [modelDirectoryHandle, modelBlobFileHandle] = await writeFileToPath(opfsRoot, originalFilePath, _etag, null)
+ // Create FileSystemSyncAccessHandle on the file.
+ blobAccessHandle = await modelBlobFileHandle.createSyncAccessHandle()
+ } catch (error) {
+ const workerMessage = `Error getting file handle for ${originalFilePath}: ${error}`
+ self.postMessage({error: workerMessage})
+ return
+ }
+
+ if (!response.body) {
+ throw new Error('ReadableStream not supported in this browser.')
+ }
+
+ const reader = response.body.getReader()
+ const contentLength = response.headers.get('Content-Length')
+
+ let receivedLength = 0 // length of received bytes
+
+ let isDone = false
+
+ try {
+ while (!isDone) {
+ const {done, value} = await reader.read()
+
+ if (done) {
+ isDone = true
+ break
+ }
+
+ try {
+ if (value !== undefined) {
+ // Write buffer
+ // eslint-disable-next-line no-unused-vars
+ const blobWriteSize = await blobAccessHandle.write(value, {at: receivedLength})
+ }
+ } catch (error) {
+ const workerMessage = `Error writing to ${response.headers.etag}: ${error}.`
+ // Close the access handle when done
+ await blobAccessHandle.close()
+ self.postMessage({error: workerMessage})
+ return
+ }
+
+ receivedLength += value.length
+
+ if (onProgress) {
+ self.postMessage({
+ progressEvent: onProgress,
+ lengthComputable: contentLength !== 0,
+ contentLength: contentLength,
+ receivedLength: receivedLength,
+ })
+ }
+ }
+
+ if (isDone) {
+ // close blob handle
+ await blobAccessHandle.close()
+ // if done, the file should be written. Signal the worker has completed.
+ try {
+ const blobFile = await modelBlobFileHandle.getFile()
+
+ self.postMessage({completed: true, event: 'download', file: blobFile})
+
+ return [modelDirectoryHandle, modelBlobFileHandle]
+ } catch (error) {
+ const workerMessage = `Error Getting file handle: ${error}.`
+ self.postMessage({error: workerMessage})
+ return
+ }
+ }
+ } catch (error) {
+ reader.cancel()
+ self.postMessage({error: error})
+ }
+}
+
+
+/**
+ * Generates a mock HTTP Response object with a specified SHA hash header.
+ *
+ * @param {string} shaHash - The SHA hash value to include in the response headers.
+ * @return {Response} A mock Response object with JSON content and specified headers.
+ */
+function generateMockResponse(shaHash) {
+ // Mock response body data
+ const mockBody = JSON.stringify({
+ cached: false,
+ etag: '"mockEtag"',
+ finalURL: 'mockURL',
+ })
+
+ // Mock response headers
+ const mockHeaders = new Headers({
+ 'Content-Type': 'application/json',
+ 'ETag': '"mockEtag"',
+ 'shahash': shaHash,
+ })
+
+ // Create a mock Response object
+ const mockResponse = new Response(mockBody, {
+ status: 200,
+ statusText: 'OK',
+ headers: mockHeaders,
+ })
+
+ return mockResponse
+}
+
+/**
+ *
+ */
+async function downloadModel(objectUrl, shaHash, originalFilePath, owner, repo, branch, accessToken, onProgress) {
+ let _etag = null
+ let commitHash = null
+ let cleanEtag = null
+ let modelDirectoryHandle = null
+ let modelBlobFileHandle = null
+ const opfsRoot = await navigator.storage.getDirectory()
+ const cacheKey = `${owner}/${repo}/${branch}/${originalFilePath}`
+
+ const cached = await CacheModule.checkCacheRaw(cacheKey)
+
+ const cacheExist = cached && cached.headers
+
+
+ if (cacheExist) {
+ const clonedCached = cached.clone()
+ // eslint-disable-next-line no-unused-vars
+ const {_, etag, finalURL} = await clonedCached.json()
+ _etag = etag
+
+ // Remove any enclosing quotes from the ETag value
+ cleanEtag = _etag.replace(/"/g, '')
+
+ if (clonedCached.headers.get('commithash')) {
+ commitHash = clonedCached.headers.get('commithash')
+ }
+ }
+
+ if (shaHash) {
+ // This will be the authed case - in this case we don't use the proxy at all.
+ // For this we just see if the either GIT SHA or the commit hash exists in a file name in OPFS.
+ // If the file exists in cache it should have a commit hash already.
+ // TODO: There is a race condition where someone can load a file unauthed and log in and refresh
+ // the page before the file is renamed with the commit hash. This would cause a duplicate file
+ // to be stored in OPFS
+
+ try {
+ [modelDirectoryHandle, modelBlobFileHandle] = await
+ retrieveFileWithPathNew(opfsRoot, cacheKey, shaHash, commitHash, false)
+
+ if (modelBlobFileHandle === null) {
+ // couldn't find via shaHash or commitHash, see if we have an unauthed etag
+ if (cleanEtag) {
+ [modelDirectoryHandle, modelBlobFileHandle] = await
+ retrieveFileWithPathNew(opfsRoot, cacheKey, cleanEtag, null, false)
+ }
+ }
+
+ if (modelBlobFileHandle !== null ) {
+ // Display model
+ const blobFile = await modelBlobFileHandle.getFile()
+
+ self.postMessage({completed: true, event: (commitHash === null ) ? 'download' : 'exists', file: blobFile})
+
+ if (commitHash !== null) {
+ return
+ }
+ // get commit hash
+ const _commitHash = await fetchLatestCommitHash(GITHUB_BASE_URL_AUTHENTICATED, owner, repo, originalFilePath, accessToken, branch)
+
+ if (_commitHash !== null) {
+ const pathSegments = originalFilePath.split('/') // Split the path into segments
+ const lastSegment = pathSegments[pathSegments.length - 1]
+ const newFileName = `${lastSegment }.${shaHash}.${_commitHash}`
+ const newResult = await renameFileInOPFS(modelDirectoryHandle, modelBlobFileHandle, newFileName)
+
+ if (newResult !== null) {
+ const mockResponse = generateMockResponse(shaHash)
+ // Update cache with new data
+ await CacheModule.updateCacheRaw(cacheKey, mockResponse, _commitHash)
+ const updatedBlobFile = await newResult.getFile()
+
+ self.postMessage({completed: true, event: 'renamed', file: updatedBlobFile})
+ }
+ }
+ } else {
+ // we don't have it and need to fetch
+ const result = await fetchRGHUC(objectUrl)
+
+ if (result !== null) {
+ [modelDirectoryHandle, modelBlobFileHandle] = await writeTemporaryFileToOPFS(result, cacheKey, shaHash, onProgress)
+
+ const mockResponse = generateMockResponse(shaHash)
+
+ await CacheModule.updateCacheRaw(cacheKey, mockResponse, null)
+
+ // get commit hash
+ const _commitHash = await fetchLatestCommitHash(GITHUB_BASE_URL_AUTHENTICATED, owner, repo, originalFilePath, accessToken, branch)
+
+ if (_commitHash !== null) {
+ const pathSegments = originalFilePath.split('/') // Split the path into segments
+ const lastSegment = pathSegments[pathSegments.length - 1]
+ const newFileName = `${lastSegment }.${shaHash}.${_commitHash}`
+ const newResult = await renameFileInOPFS(modelDirectoryHandle, modelBlobFileHandle, newFileName)
+
+ if (newResult !== null) {
+ // Update cache with new data
+ const clonedResponse = generateMockResponse(shaHash)
+ await CacheModule.updateCacheRaw(cacheKey, clonedResponse, _commitHash)
+ const updatedBlobFile = await newResult.getFile()
+
+ self.postMessage({completed: true, event: 'renamed', file: updatedBlobFile})
+ return
+ }
+ }
+ }
+ }
+ } catch (error) {
+ return
+ }
+
+ return
+ }
+
+ // const etag = "\"d3796370c5691ef25bbc6e829194623e4a2521a78092fa3abec23c0e8fe34e1a\""
+ const result = await fetchAndHeadRequest(objectUrl, _etag)
+
+ if (result === null) {
+ // result SHOULD be cached, let's see.
+ try {
+ [modelDirectoryHandle, modelBlobFileHandle] = await
+ retrieveFileWithPathNew(opfsRoot, cacheKey, cleanEtag, commitHash === null ? 'temporary' : commitHash, false)
+
+ if (modelBlobFileHandle !== null ) {
+ // Display model
+ const blobFile = await modelBlobFileHandle.getFile()
+
+ self.postMessage({completed: true, event: (commitHash === null ) ? 'download' : 'exists', file: blobFile})
+
+ if (commitHash !== null) {
+ return
+ }
+ // TODO: get commit hash
+ const _commitHash = await fetchLatestCommitHash(GITHUB_BASE_URL_UNAUTHENTICATED, owner, repo, originalFilePath, accessToken, branch)
+
+ if (_commitHash !== null) {
+ const pathSegments = originalFilePath.split('/') // Split the path into segments
+ const lastSegment = pathSegments[pathSegments.length - 1]
+ const newFileName = `${lastSegment }.${cleanEtag}.${ _commitHash === null ? 'temporary' : _commitHash}`
+ const newResult = await renameFileInOPFS(modelDirectoryHandle, modelBlobFileHandle, newFileName)
+
+ if (newResult !== null) {
+ // Update cache with new data
+ await CacheModule.updateCacheRaw(cacheKey, proxyResponse, _commitHash)
+ const updatedBlobFile = await newResult.getFile()
+
+ self.postMessage({completed: true, event: 'renamed', file: updatedBlobFile})
+ }
+ }
+ }
+ } catch (error) {
+ // expected if file not found - lets see if we have a temporary file
+
+ if (commitHash !== null) {
+ try {
+ [modelDirectoryHandle, modelBlobFileHandle] = await
+ retrieveFileWithPathNew(opfsRoot, cacheKey, cleanEtag, 'temporary', false)
+
+ if (modelBlobFileHandle !== null ) {
+ // Display model and get commitHash
+ const blobFile = await modelBlobFileHandle.getFile()
+
+ self.postMessage({completed: true, event: 'download', file: blobFile})
+
+ // eslint-disable-next-line no-console
+ console.log('Getting commit hash...')
+ // TODO: get commit hash here
+ const _commitHash = await fetchLatestCommitHash(
+ GITHUB_BASE_URL_UNAUTHENTICATED,
+ owner,
+ repo,
+ originalFilePath,
+ accessToken,
+ branch)
+
+ if (_commitHash !== null) {
+ const pathSegments = originalFilePath.split('/') // Split the path into segments
+ const lastSegment = pathSegments[pathSegments.length - 1]
+ const newFileName = `${lastSegment }.${cleanEtag}.${ _commitHash === null ? 'temporary' : _commitHash}`
+ const newResult = await renameFileInOPFS(modelDirectoryHandle, modelBlobFileHandle, newFileName)
+
+ if (newResult !== null) {
+ // Update cache with new data
+ await CacheModule.updateCacheRaw(cacheKey, proxyResponse, _commitHash)
+ const updatedBlobFile = await newResult.getFile()
+
+ self.postMessage({completed: true, event: 'renamed', file: updatedBlobFile})
+ }
+ }
+ }
+ } catch (error_) {
+ // expected if file not found - invalidate cache and try again
+ // eslint-disable-next-line no-console
+ console.log('File not found in cache, invalidating cache and request again with no etag')
+ await CacheModule.deleteCache(cacheKey)
+ downloadModel(objectUrl, shaHash, originalFilePath, owner, repo, branch, accessToken, onProgress)
+ return
+ }
+ }
+
+ // eslint-disable-next-line no-console
+ console.log('File not found in cache, invalidating cache and request again with no etag')
+ await CacheModule.deleteCache(cacheKey)
+ downloadModel(objectUrl, shaHash, originalFilePath, owner, repo, branch, accessToken, onProgress)
+ return
+ }
+ }
+
+ // not cached, download model
+ const {proxyResponse, modelResponse, etag} = result
+
+ // Remove any enclosing quotes from the ETag value
+ cleanEtag = etag.replace(/"/g, '');
+
+ [modelDirectoryHandle, modelBlobFileHandle] = await writeTemporaryFileToOPFS(modelResponse, cacheKey, cleanEtag, onProgress)
+
+ // Compute file git sha1 hash
+ const computedShaHash = await computeGitBlobSha1FromHandle(modelBlobFileHandle)
+ // eslint-disable-next-line no-console
+ console.log('SHA-1 Hash:', computedShaHash)
+
+ try {
+ // eslint-disable-next-line no-unused-vars
+ const [modelDirectoryHandle_, modelBlobFileHandle_] = await
+ retrieveFileWithPathNew(opfsRoot, cacheKey, computedShaHash, null, false)
+
+ if (modelBlobFileHandle_ !== null) {
+ // eslint-disable-next-line no-console
+ console.log('SHA match found in OPFS')
+ // we already have this file, just delete the one we downloaded and update the cached response.
+ const newResponse = proxyResponse.clone()
+ await CacheModule.updateCacheRaw(cacheKey, newResponse, commitHash)
+ modelDirectoryHandle.removeEntry(modelBlobFileHandle.name)
+ return
+ }
+ } catch (error_) {
+ return
+ }
+
+
+ // Update cache with new data
+ const clonedResponse = proxyResponse.clone()
+ await CacheModule.updateCacheRaw(cacheKey, clonedResponse, null)
+
+ // TODO: get commit hash
+ const _commitHash = await fetchLatestCommitHash(GITHUB_BASE_URL_UNAUTHENTICATED, owner, repo, originalFilePath, accessToken, branch)
+
+ if (_commitHash !== null) {
+ const pathSegments = originalFilePath.split('/') // Split the path into segments
+ const lastSegment = pathSegments[pathSegments.length - 1]
+ const newFileName = `${lastSegment }.${cleanEtag}.${ _commitHash === null ? 'temporary' : _commitHash}`
+ const newResult = await renameFileInOPFS(modelDirectoryHandle, modelBlobFileHandle, newFileName)
+
+ if (newResult !== null) {
+ // Update cache with new data
+ await CacheModule.updateCacheRaw(cacheKey, proxyResponse, _commitHash)
+ const updatedBlobFile = await newResult.getFile()
+
+ self.postMessage({completed: true, event: 'renamed', file: updatedBlobFile})
+ }
+ }
+}
+
/**
*
*/
async function downloadModelToOPFS(objectUrl, commitHash, originalFilePath, owner, repo, branch, onProgress) {
const opfsRoot = await navigator.storage.getDirectory()
+
let ownerFolderHandle = null
let repoFolderHandle = null
let branchFolderHandle = null
@@ -276,6 +872,44 @@ async function downloadModelToOPFS(objectUrl, commitHash, originalFilePath, owne
}
}
+/**
+ * writeFileToPath
+ */
+async function writeFileToPath(rootHandle, filePath, etag, commitHash = null) {
+ const pathSegments = filePath.split('/') // Split the path into segments
+ let currentHandle = rootHandle
+
+ for (let i = 0; i < pathSegments.length; i++) {
+ const segment = pathSegments[i]
+ const isLastSegment = i === pathSegments.length - 1
+
+ if (!isLastSegment) {
+ // Try to get the directory handle; if it doesn't exist, create it
+ try {
+ currentHandle = await currentHandle.getDirectoryHandle(segment, {create: true})
+ } catch (error) {
+ const workerMessage = `Error getting/creating directory handle for segment: ${error}.`
+ self.postMessage({error: workerMessage})
+ return null
+ }
+ } else {
+ // Last segment, treat it as a file
+ try {
+ // Create or get the file handle
+ const fileHandle = await
+ currentHandle.getFileHandle(`${segment }.${etag}.${ commitHash === null ? 'temporary' : commitHash}`,
+ {create: true})
+ return [currentHandle, fileHandle] // Return the file handle for further processing
+ } catch (error) {
+ const workerMessage = `Error getting/creating file handle for file ${segment}: ${error}.`
+ self.postMessage({error: workerMessage})
+ return null
+ }
+ }
+ }
+}
+
+
/**
*
*/
@@ -314,6 +948,53 @@ async function retrieveFileWithPath(rootHandle, filePath, commitHash, shouldCrea
}
}
+/**
+ *
+ */
+async function retrieveFileWithPathNew(rootHandle, filePath, etag, commitHash, create = false) {
+ const pathSegments = filePath.split('/') // Split the path into segments
+ let currentHandle = rootHandle
+
+ for (let i = 0; i < pathSegments.length; i++) {
+ const segment = pathSegments[i]
+ const isLastSegment = i === pathSegments.length - 1
+
+ if (!isLastSegment) {
+ // Try to get the directory handle; if it doesn't exist, create it
+ try {
+ currentHandle = await currentHandle.getDirectoryHandle(segment, {create: true})
+ } catch (error) {
+ const workerMessage = `Error getting/creating directory handle for segment: ${error}.`
+ self.postMessage({error: workerMessage})
+ return [null, null]
+ }
+ } else {
+ // Last segment, treat it as a file
+ try {
+ if (create) {
+ // If no matching file is found, create a new file handle
+ const fileHandle = await currentHandle.getFileHandle(
+ `${segment}.${etag}.${commitHash === null ? 'temporary' : commitHash}`,
+ {create: create},
+ )
+ return [currentHandle, fileHandle] // Return the new file handle
+ }
+
+ // Search for any file in the directory that contains either the etag or commitHash
+ for await (const [name, handle] of currentHandle.entries()) {
+ if (handle.kind === 'file' && (name.includes(etag) || (commitHash !== null && name.includes(commitHash)))) {
+ return [currentHandle, handle] // Return the handle of the matching file
+ }
+ }
+
+ return [null, null]
+ } catch (error) {
+ return [null, null]
+ }
+ }
+ }
+}
+
/**
*
*/
@@ -344,72 +1025,30 @@ async function writeFileToHandle(blobAccessHandle, modelFile) {
*/
async function writeModelToOPFSFromFile(modelFile, objectKey, originalFilePath, owner, repo, branch) {
const opfsRoot = await navigator.storage.getDirectory()
- let ownerFolderHandle = null
- let repoFolderHandle = null
- let branchFolderHandle = null
- // See if owner folder handle exists
- try {
- ownerFolderHandle = await opfsRoot.getDirectoryHandle(owner, {create: false})
- } catch (error) {
- // Expected: folder does not exist
- }
- if (ownerFolderHandle === null) {
- try {
- ownerFolderHandle = await opfsRoot.getDirectoryHandle(owner, {create: true})
- } catch (error) {
- const workerMessage = `Error getting folder handle for ${owner}: ${error}`
- self.postMessage({error: workerMessage})
- return
- }
- }
-
- // See if repo folder handle exists
- try {
- repoFolderHandle = await ownerFolderHandle.getDirectoryHandle(repo, {create: false})
- } catch (error) {
- // Expected: folder does not exist
- }
-
- if (repoFolderHandle === null) {
- try {
- repoFolderHandle = await ownerFolderHandle.getDirectoryHandle(repo, {create: true})
- } catch (error) {
- const workerMessage = `Error getting folder handle for ${repo}: ${error}`
- self.postMessage({error: workerMessage})
- return
- }
- }
-
- // See if branch folder handle exists
- try {
- branchFolderHandle = await repoFolderHandle.getDirectoryHandle(branch, {create: false})
- } catch (error) {
- // Expected: folder does not exist
- }
-
- if (branchFolderHandle === null) {
- try {
- branchFolderHandle = await repoFolderHandle.getDirectoryHandle(branch, {create: true})
- } catch (error) {
- const workerMessage = `Error getting folder handle for ${branch}: ${error}`
- self.postMessage({error: workerMessage})
- return
- }
- }
+ // Compute file git sha1 hash
+ const computedShaHash = await computeGitBlobSha1FromFile(modelFile)
+ // eslint-disable-next-line no-console
+ console.log('SHA-1 Hash:', computedShaHash)
// Get a file handle in the folder for the model
- let modelBlobFileHandle = null
- let modelDirectoryHandle = null
let blobAccessHandle = null
+ const cacheKey = `${owner}/${repo}/${branch}/${originalFilePath}`
+ let modelDirectoryHandle = null
+ let modelBlobFileHandle = null
+
try {
// eslint-disable-next-line no-unused-vars
- [modelDirectoryHandle, modelBlobFileHandle] = await retrieveFileWithPath(branchFolderHandle, originalFilePath, objectKey, true)
+ [modelDirectoryHandle, modelBlobFileHandle] = await
+ retrieveFileWithPathNew(opfsRoot, cacheKey, computedShaHash, objectKey, true)
// Create FileSystemSyncAccessHandle on the file.
blobAccessHandle = await modelBlobFileHandle.createSyncAccessHandle()
if (await writeFileToHandle(blobAccessHandle, modelFile)) {
+ // Update cache with new data
+ const mockResponse = generateMockResponse(computedShaHash)
+ await CacheModule.updateCacheRaw(cacheKey, mockResponse, objectKey)
self.postMessage({completed: true, event: 'write'})
}
} catch (error) {
@@ -418,6 +1057,26 @@ async function writeModelToOPFSFromFile(modelFile, objectKey, originalFilePath,
}
}
+// Function to rename the file in OPFS
+/**
+ *
+ */
+async function renameFileInOPFS(parentDirectory, fileHandle, newFileName) {
+ const newFileHandle = await parentDirectory.getFileHandle(newFileName, {create: true})
+
+ // Copy the contents of the old file to the new file
+ const oldFile = await fileHandle.getFile()
+ const writable = await newFileHandle.createWritable()
+ await writable.write(await oldFile.arrayBuffer())
+ await writable.close()
+
+ // Remove the old file
+ await parentDirectory.removeEntry(fileHandle.name)
+
+ return newFileHandle
+}
+
+
/**
* This function navigates to a filepath in OPFS to see if it exists.
* If any parent folders or the file do not exist, it will return 'notexist'.
diff --git a/src/OPFS/OPFSService.js b/src/OPFS/OPFSService.js
index 147767f93..0e3c6bd1d 100644
--- a/src/OPFS/OPFSService.js
+++ b/src/OPFS/OPFSService.js
@@ -1,4 +1,5 @@
import debug from '../utils/debug'
+import {GITHUB_BASE_URL_AUTHED, GITHUB_BASE_URL_UNAUTHED} from '../net/github/OctokitExport'
// TODO(pablo): probably don't need global state, can
// pass worker refs as needed.
@@ -16,6 +17,12 @@ let workerRef = null
export function initializeWorker() {
if (workerRef === null) {
workerRef = new Worker('/OPFS.Worker.js')
+
+ workerRef.postMessage({
+ command: 'initializeWorker',
+ GITHUB_BASE_URL_AUTHED: GITHUB_BASE_URL_AUTHED,
+ GITHUB_BASE_URL_UNAUTHED: GITHUB_BASE_URL_UNAUTHED,
+ })
}
return workerRef
@@ -206,6 +213,41 @@ export function opfsDownloadToOPFS(objectUrl, commitHash, originalFilePath, owne
})
}
+/**
+ * Downloads a file to the OPFS repository from a specified URL.
+ *
+ * Initiates a download process for a file from the provided URL to
+ * store it within the OPFS repository under a specific commit hash,
+ * original file path, and within the specified owner's repository and branch.
+ * The function also supports progress tracking through a callback function.
+ *
+ * @param {string} objectUrl The URL from which the file is to be downloaded
+ * @param {string} shaHash The file hash for the object
+ * @param {string} originalFilePath The path where the file will be stored in the repository
+ * @param {string} owner The owner of the repository
+ * @param {string} repo The name of the repository
+ * @param {string} branch The branch name where the file will be stored
+ * @param {string} accessToken GitHub access token
+ * @param {Function} onProgress A callback function to track the progress of the download
+ */
+export function opfsDownloadModel(objectUrl, shaHash, originalFilePath, owner, repo, branch, accessToken, onProgress) {
+ if (!workerRef) {
+ debug().error('Worker not initialized')
+ return
+ }
+ workerRef.postMessage({
+ command: 'downloadModel',
+ objectUrl: objectUrl,
+ shaHash: shaHash,
+ originalFilePath: originalFilePath,
+ owner: owner,
+ repo: repo,
+ branch: branch,
+ accessToken: accessToken,
+ onProgress: onProgress,
+ })
+}
+
/**
* Reads a file from the OPFS storage.
*
diff --git a/src/OPFS/utils.js b/src/OPFS/utils.js
index 3d5e607d7..d8dd9cf85 100644
--- a/src/OPFS/utils.js
+++ b/src/OPFS/utils.js
@@ -1,6 +1,7 @@
import {
initializeWorker,
opfsDownloadToOPFS,
+ opfsDownloadModel,
opfsReadModel,
opfsWriteModel,
opfsWriteModelFileHandle,
@@ -150,6 +151,93 @@ export function downloadToOPFS(
})
}
+/**
+ * Downloads a model, handles progress updates, and updates the OPFS file handle.
+ *
+ * @param {Function} navigate Function to navigate to a different route.
+ * @param {string} appPrefix The application prefix for routing.
+ * @param {Function} handleBeforeUnload Function to handle the beforeunload event.
+ * @param {string} objectUrl The URL of the object to be downloaded.
+ * @param {string} originalFilePath The original file path of the model.
+ * @param {string} accessToken Access token for authentication.
+ * @param {string} owner The owner of the repository.
+ * @param {string} repo The repository name.
+ * @param {string} branch The branch name.
+ * @param {Function} setOpfsFile Function to set the OPFS file in the state.
+ * @param {Function} [onProgress] Optional function to handle progress events.
+ * @return {Promise} - A promise that resolves to the downloaded file.
+ */
+export function downloadModel(
+ navigate,
+ appPrefix,
+ handleBeforeUnload,
+ objectUrl,
+ shaHash,
+ originalFilePath,
+ accessToken,
+ owner,
+ repo,
+ branch,
+ setOpfsFile,
+ onProgress) {
+assertDefined(
+ navigate,
+ appPrefix,
+ handleBeforeUnload,
+ objectUrl,
+ shaHash,
+ originalFilePath,
+ accessToken,
+ owner,
+ repo,
+ branch)
+
+return new Promise((resolve, reject) => {
+ const workerRef = initializeWorker()
+ if (workerRef !== null) {
+ // Listener for messages from the worker
+ const listener = (event) => {
+ if (event.data.error) {
+ debug().error('Error from worker:', event.data.error)
+ workerRef.removeEventListener('message', listener) // Remove the event listener
+ reject(new Error(event.data.error))
+ } else if (event.data.progressEvent) {
+ if (onProgress) {
+ onProgress({
+ lengthComputable: event.data.contentLength !== 0,
+ contentLength: event.data.contentLength,
+ receivedLength: event.data.receivedLength,
+ }) // Custom progress event
+ }
+ } else if (event.data.completed) {
+ if (event.data.event === 'download') {
+ debug().warn('Worker finished downloading file')
+ } else if (event.data.event === 'exists') {
+ debug().warn('Commit exists in OPFS.')
+ }
+
+ const file = event.data.file
+ if (event.data.event === 'renamed' || event.data.event === 'exists') {
+ workerRef.removeEventListener('message', listener) // Remove the event listener
+ if (file instanceof File) {
+ setOpfsFile(file)
+ } else {
+ debug().error('Retrieved object is not of type File.')
+ }
+ }
+
+ resolve(file) // Resolve the promise with the file
+ }
+ }
+ workerRef.addEventListener('message', listener)
+ } else {
+ reject(new Error('Worker initialization failed'))
+ }
+
+ opfsDownloadModel(objectUrl, shaHash, originalFilePath, owner, repo, branch, accessToken, !!(onProgress))
+})
+}
+
/**
* Executes an asynchronous task using a Web Worker and returns a promise that resolves based on the task's outcome.
* This function initializes a worker, sets up a message listener for the worker's response, and performs cleanup
diff --git a/src/OPFS/utils.test.js b/src/OPFS/utils.test.js
index 39d62a049..eb93e6e1a 100644
--- a/src/OPFS/utils.test.js
+++ b/src/OPFS/utils.test.js
@@ -4,6 +4,7 @@ import {
writeSavedGithubModelOPFS,
getModelFromOPFS,
downloadToOPFS,
+ downloadModel,
doesFileExistInOPFS,
deleteFileFromOPFS,
checkOPFSAvailability,
@@ -153,6 +154,94 @@ describe('OPFS Test Suite', () => {
})
})
+ describe('downloadModel', () => {
+ it('should resolve with file when download completes', async () => {
+ const mockFile = new Blob(['dummy content'], {type: 'application/octet-stream'})
+ const mockWorker = {
+ addEventListener: jest.fn((_, handler) => {
+ process.nextTick(() => {
+ handler({data: {completed: true, event: 'exists', file: mockFile}})
+ })
+ }),
+ removeEventListener: jest.fn(),
+ }
+ OPFSService.initializeWorker.mockReturnValue(mockWorker)
+
+ const onProgressMock = jest.fn()
+ const setOPFSFile = jest.fn()
+ const result = await downloadModel(
+ // eslint-disable-next-line no-empty-function
+ () => {}, // navigate
+ 'appPrefix',
+ // eslint-disable-next-line no-empty-function
+ () => {}, // handleBeforeUnload
+ 'objectUrl',
+ 'shaHash',
+ 'originalFilePath',
+ 'accessToken',
+ 'owner',
+ 'repo',
+ 'branch',
+ setOPFSFile,
+ onProgressMock,
+ )
+
+ expect(result).toEqual(mockFile)
+ expect(OPFSService.initializeWorker).toHaveBeenCalled()
+ expect(OPFSService.opfsDownloadModel).toHaveBeenCalledWith(
+ 'objectUrl',
+ 'shaHash',
+ 'originalFilePath',
+ 'owner',
+ 'repo',
+ 'branch',
+ 'accessToken',
+ true, // Since onProgress is provided
+ )
+ expect(mockWorker.addEventListener).toHaveBeenCalled()
+ expect(mockWorker.removeEventListener).toHaveBeenCalledTimes(1) // Ensure it's called to clean up
+ })
+
+ it('should call onProgress with progress data', async () => {
+ const mockWorker = {
+ addEventListener: jest.fn((_, handler) => {
+ process.nextTick(() => {
+ handler({data: {progressEvent: true, contentLength: 100, receivedLength: 50}}) // Simulate a progress update
+ handler({data: {completed: true, event: 'download', file: new Blob(['content'])}}) // Then download
+ handler({data: {completed: true, event: 'renamed', file: new Blob(['content'])}}) // Then complete
+ })
+ }),
+ removeEventListener: jest.fn(),
+ }
+ OPFSService.initializeWorker.mockReturnValue(mockWorker)
+
+ const onProgressMock = jest.fn()
+ const setOPFSFile = jest.fn()
+ await downloadModel(
+ // eslint-disable-next-line no-empty-function
+ () => {}, // navigate
+ 'appPrefix',
+ // eslint-disable-next-line no-empty-function
+ () => {}, // handleBeforeUnload
+ 'objectUrl',
+ 'shaHash',
+ 'originalFilePath',
+ 'accessToken',
+ 'owner',
+ 'repo',
+ 'branch',
+ setOPFSFile,
+ onProgressMock,
+ )
+
+ expect(onProgressMock).toHaveBeenCalledWith({
+ lengthComputable: true,
+ contentLength: 100,
+ receivedLength: 50,
+ })
+ })
+ })
+
describe('doesFileExistInOPFS', () => {
it('should resolve true if the file exists', async () => {
const mockWorker = {
diff --git a/src/net/github/Cache.js b/src/net/github/Cache.js
index 53c67abb9..7a12063f5 100644
--- a/src/net/github/Cache.js
+++ b/src/net/github/Cache.js
@@ -1,105 +1,219 @@
-// http request etag cache
-let httpCache = null
-
-const httpCacheApiAvailable = ('caches' in window)
-
/**
- * Retrieves the HTTP cache, opening it if it doesn't already exist.
+ * This module implements an etag caching system for network requests
+ * using the browser Cache API.
+ * This module was rewritten to support ES6 + CommonJS for Web-Workers
+ * using the UMD (Universal Module Definition) pattern.
*
- * @return {Promise} The HTTP cache object.
+ * Usage:
+ * 1. ES6 Main Thread:
+ * import {checkCache} from './Cache'
+ * ...
+ * checkCache("test")
+ * 2. CommonJS (Web Worker):
+ * importScripts('./Cache.js');
+ * ...
+ * CacheModule.checkCache("test")
*/
-async function getCache() {
- if (!httpCache) {
+
+(function(root, factory) {
+ // eslint-disable-next-line no-undef
+ if (typeof define === 'function' && define.amd) {
+ // AMD
+ // eslint-disable-next-line no-undef
+ define([], factory)
+ } else if (typeof module === 'object' && module.exports) {
+ // CommonJS
+ module.exports = factory()
+ } else {
+ // Browser globals
+ root.CacheModule = factory()
+ }
+ // eslint-disable-next-line no-invalid-this
+}(typeof self !== 'undefined' ? self : this, function() {
+ // http request etag cache
+ let httpCache = null
+
+ const httpCacheApiAvailable = (typeof caches !== 'undefined')
+
+ /**
+ * Retrieves the HTTP cache, opening it if it doesn't already exist.
+ *
+ * @return {Promise} The HTTP cache object.
+ */
+ async function getCache() {
+ if (!httpCache) {
httpCache = await openCache()
+ }
+ return httpCache
}
- return httpCache
-}
-/**
- * Opens the HTTP cache if the Cache API is available.
- *
- * @return {Promise} A Cache object if the Cache API is available, otherwise an empty object.
- */
-async function openCache() {
- if (httpCacheApiAvailable) {
+ /**
+ * Opens the HTTP cache if the Cache API is available.
+ *
+ * @return {Promise} A Cache object if the Cache API is available, otherwise an empty object.
+ */
+ async function openCache() {
+ if (httpCacheApiAvailable) {
return await caches.open('bldrs-github-api-cache')
+ }
+ return {}
}
- // fallback to caching only on current page, won't survive page reloads
- return {}
-}
-/**
- * Converts a cached response to an Octokit response format.
- *
- * @param {Response|null} cachedResponse The cached response to convert.
- * @return {Promise