Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Working on download function with the file name #196

Merged
merged 23 commits into from
Jul 18, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 48 additions & 0 deletions .github/workflows/be_on_push.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
name: On Push Backend

on:
push:
branches:
- main
paths:
- packages/backend/**
pull_request:
branches:
- main

# Docs: https://learn.microsoft.com/en-us/azure/azure-functions/functions-how-to-github-actions?tabs=linux%2Cjavascript&pivots=method-manual

env:
AZURE_FUNCTIONAPP_NAME: 'gosqasbe' # set this to your function app name on Azure
AZURE_FUNCTIONAPP_PACKAGE_PATH: 'packages/backend' # set this to the path to your function app project, defaults to the repository root
NODE_VERSION: '18.x' # set this to the node version to use (e.g. '8.x', '10.x', '12.x')

jobs:
build-and-deploy:
runs-on: ubuntu-latest
steps:
- name: 'Checkout GitHub Action'
uses: actions/checkout@v4.1.7


- name: Setup Node ${{ env.NODE_VERSION }} Environment
uses: actions/setup-node@v4.0.3
with:
node-version: ${{ env.NODE_VERSION }}

- name: 'Resolve Project Dependencies Using Npm'
shell: bash
run: |
pushd './${{ env.AZURE_FUNCTIONAPP_PACKAGE_PATH }}'
npm install
npm run build --if-present
popd

- name: 'Run Azure Functions Action'
uses: Azure/functions-action@v1
if: github.ref == 'refs/heads/main' && github.event_name == 'push'
id: fa
with:
app-name: ${{ env.AZURE_FUNCTIONAPP_NAME }}
package: ${{ env.AZURE_FUNCTIONAPP_PACKAGE_PATH }}
publish-profile: ${{ secrets.AZURE_FUNCTIONAPP_PUBLISH_PROFILE }}
File renamed without changes.
3 changes: 1 addition & 2 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,5 @@
},
"workspaces": [
"packages/legacy"
],
"packageManager": "pnpm@9.1.1+sha512.14e915759c11f77eac07faba4d019c193ec8637229e62ec99eefb7cf3c3b75c64447882b7c485142451ee3a6b408059cdfb7b7fa0341b975f12d0f7629c71195"
]
}
5 changes: 5 additions & 0 deletions packages/backend/.vscode/extensions.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
{
"recommendations": [
"ms-azuretools.vscode-azurefunctions"
]
}
12 changes: 12 additions & 0 deletions packages/backend/.vscode/launch.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
{
"version": "0.2.0",
"configurations": [
{
"name": "Attach to Node Functions",
"type": "node",
"request": "attach",
"port": 9229,
"preLaunchTask": "func: host start"
}
]
}
9 changes: 9 additions & 0 deletions packages/backend/.vscode/settings.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
{
"azureFunctions.deploySubpath": ".",
"azureFunctions.postDeployTask": "npm install (functions)",
"azureFunctions.projectLanguage": "TypeScript",
"azureFunctions.projectRuntime": "~4",
"debug.internalConsoleOptions": "neverOpen",
"azureFunctions.projectLanguageModel": 4,
"azureFunctions.preDeployTask": "npm prune (functions)"
}
38 changes: 38 additions & 0 deletions packages/backend/.vscode/tasks.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
{
"version": "2.0.0",
"tasks": [
{
"type": "func",
"label": "func: host start",
"command": "host start",
"problemMatcher": "$func-node-watch",
"isBackground": true,
"dependsOn": "npm build (functions)"
},
{
"type": "shell",
"label": "npm build (functions)",
"command": "npm run build",
"dependsOn": "npm clean (functions)",
"problemMatcher": "$tsc"
},
{
"type": "shell",
"label": "npm install (functions)",
"command": "npm install"
},
{
"type": "shell",
"label": "npm prune (functions)",
"command": "npm prune --production",
"dependsOn": "npm build (functions)",
"problemMatcher": []
},
{
"type": "shell",
"label": "npm clean (functions)",
"command": "npm run clean",
"dependsOn": "npm install (functions)"
}
]
}
24 changes: 12 additions & 12 deletions packages/backend/local.settings.json
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@

{
"IsEncrypted": false,
"Values": {
"FUNCTIONS_WORKER_RUNTIME": "node",
"AzureWebJobsFeatureFlags": "EnableWorkerIndexing",
"AzureWebJobsStorage": "",
"AZURE_STORAGE_ACCOUNT_NAME": "",
"AZURE_STORAGE_ACCOUNT_KEY": ""
},
"Host": {
"CORS": "*"
}
}
"IsEncrypted": false,
"Values": {
"FUNCTIONS_WORKER_RUNTIME": "node",
"AzureWebJobsFeatureFlags": "EnableWorkerIndexing",
"AzureWebJobsStorage": "UseDevelopmentStorage=true",
"AZURE_STORAGE_ACCOUNT_NAME": "",
"AZURE_STORAGE_ACCOUNT_KEY": ""
},
"Host": {
"CORS": "*"
}
}
88 changes: 64 additions & 24 deletions packages/backend/src/functions/httpTrigger.ts
Original file line number Diff line number Diff line change
Expand Up @@ -49,9 +49,9 @@ async function calculateDeviceID(key: string | Uint8Array): Promise<string> {
return toHex(hash);
}

async function encrypt(key: Uint8Array, data: BufferSource): Promise<{ salt: Uint8Array; encryptedData: Uint8Array; }> {
async function encrypt(key: Uint8Array, data: BufferSource, salt?: Uint8Array): Promise<{ salt: Uint8Array; encryptedData: Uint8Array; }> {
const $key = await crypto.subtle.importKey("raw", key.buffer, "AES-CBC", false, ['encrypt']);
const salt = crypto.getRandomValues(new Uint8Array(16));
salt ??= crypto.getRandomValues(new Uint8Array(16));
const encryptedData = await crypto.subtle.encrypt({ name: "AES-CBC", iv: salt }, $key, data);
return { salt, encryptedData: new Uint8Array(encryptedData) };
}
Expand All @@ -62,19 +62,24 @@ async function decrypt(key: Uint8Array, salt: Uint8Array, encryptedData: Uint8Ar
return new Uint8Array(result);
}

async function upload(client: ContainerClient, deviceKey: Uint8Array, data: BufferSource, type: 'attach' | 'prov', contentType: string, timestamp: number): Promise<string> {
async function upload(client: ContainerClient, deviceKey: Uint8Array, data: BufferSource, type: 'attach' | 'prov', contentType: string, timestamp: number, fileName: string | undefined): Promise<string> {
const dataHash = toHex(await sha256(data));
const deviceID = await calculateDeviceID(deviceKey);
const { salt, encryptedData } = await encrypt(deviceKey, data);
const blobID = toHex(await sha256(encryptedData));
const blobName = `${client.containerName}/${deviceID}/${type}/${blobID}`;

const { encryptedData: encryptedName } = fileName
? await encrypt(deviceKey, new TextEncoder().encode(fileName), salt)
: { encryptedData: undefined };

await client.uploadBlockBlob(blobName, encryptedData.buffer, encryptedData.length, {
metadata: {
gdtcontenttype: contentType,
gdthash: dataHash,
gdtsalt: toHex(salt),
gdttimestamp: `${timestamp}`,
gdtname: encryptedName ? toHex(encryptedName) : ""
},
blobHTTPHeaders: {
blobContentType: "application/octet-stream"
Expand All @@ -83,23 +88,36 @@ async function upload(client: ContainerClient, deviceKey: Uint8Array, data: Buff
return blobID;
}

async function decryptBlob(client: BlockBlobClient, deviceKey: Uint8Array) {
interface DecryptedBlob {
data: Uint8Array;
contentType: string;
timestamp: number;
filename?: string;
}

async function decryptBlob(client: BlockBlobClient, deviceKey: Uint8Array): Promise<DecryptedBlob> {
const props = await client.getProperties();
const salt = props.metadata?.["gdtsalt"];
if (!salt) throw new Error(`Missing Salt ${client.name}`);
const timestamp = parseInt(props.metadata?.["gdttimestamp"]);
if (isNaN(timestamp) || !isFinite(timestamp)) throw new Error(`Invalid Timestamp ${client.name}`);

const buffer = await client.downloadToBuffer();
const data = await decrypt(deviceKey, fromHex(salt), buffer);
const saltBuffer = fromHex(salt);
const data = await decrypt(deviceKey, saltBuffer, buffer);
const hash = props.metadata?.["gdthash"];
if (hash) {
if (!areEqual(fromHex(hash), await sha256(data))) {
throw new Error(`Invalid Hash ${client.name}`);
}
}

const contentType = props.metadata?.["gdtcontenttype"];
return { data, contentType, timestamp };
const encryptedName = props.metadata?.["gdtname"] ?? "";
const encodedName = encryptedName.length > 0 ? await decrypt(deviceKey, saltBuffer, fromHex(encryptedName)) : undefined;
const filename = encodedName ? new TextDecoder().decode(encodedName) : undefined;

return { data, contentType, timestamp, filename };

function areEqual(first: Uint8Array, second: Uint8Array) {
return first.length === second.length
Expand Down Expand Up @@ -136,26 +154,44 @@ async function getProvenance(request: HttpRequest, context: InvocationContext):
return { jsonBody: records };
}

async function getAttachment(request: HttpRequest, context: InvocationContext): Promise<HttpResponseInit> {
async function getDecryptedBlob(request: HttpRequest, context: InvocationContext): Promise<DecryptedBlob | undefined> {
const deviceKey = decodeKey(request.params.deviceKey);
const deviceID = await calculateDeviceID(deviceKey);
const attachmentID = request.params.attachmentID;
context.log(`getAttachment`, { accountName, deviceKey: request.params.deviceKey, deviceID, attachmentID });
context.log(`getDecryptedBlob`, { accountName, deviceKey: request.params.deviceKey, deviceID, attachmentID });

const containerExists = await containerClient.exists();
if (!containerExists) { return { status: 404 }; }
if (!containerExists) { return undefined; }

const blobClient = containerClient.getBlockBlobClient(`gosqas/${deviceID}/attach/${attachmentID}`);
const exists = await blobClient.exists();
if (!exists) { return { status: 404 }; }
if (!exists) { return undefined; }

const { data, contentType } = await decryptBlob(blobClient, deviceKey);
return {
body: data,
headers: contentType
? { "Content-Type": contentType }
: undefined
};
return await decryptBlob(blobClient, deviceKey);
}

async function getAttachment(request: HttpRequest, context: InvocationContext): Promise<HttpResponseInit> {
const decryptedBlob = await getDecryptedBlob(request, context);
if (!decryptedBlob) { return { status: 404 } }

const { data, contentType, filename } = decryptedBlob;
const headers = new Headers();
headers.append("Access-Control-Allow-Headers", "Attachment-Name");
if (contentType) { headers.append("Content-Type", contentType); }
if (filename) {
headers.append("Content-Disposition", `attachment; filename="${filename}"`);
headers.append("Attachment-Name", filename);
}

return { body: data, headers };
};

async function getAttachmentName(request: HttpRequest, context: InvocationContext): Promise<HttpResponseInit> {
const decryptedBlob = await getDecryptedBlob(request, context);
if (!decryptedBlob) { return { status: 404 } }

const { filename } = decryptedBlob;
return { body: filename };
};

async function postProvenance(request: HttpRequest, context: InvocationContext): Promise<HttpResponseInit> {
Expand All @@ -172,23 +208,21 @@ async function postProvenance(request: HttpRequest, context: InvocationContext):

// https://stackoverflow.com/questions/9756120/how-do-i-get-a-utc-timestamp-in-javascript#comment73511758_9756120
const timestamp = new Date().getTime();

const attachments = new Array<string>();
{
for (const attach of formData.getAll("attachment")) {
for (const attach of formData.values()) {
if (typeof attach === 'string') continue;
const data = await attach.arrayBuffer()
const attachmentID = await upload(containerClient, deviceKey, data, "attach", attach.type, timestamp);
const attachmentID = await upload(containerClient, deviceKey, data, "attach", attach.type, timestamp, attach.name);
attachments.push(attachmentID);
}
}

{
const provRecord: ProvenanceRecord = { record, attachments };
const data = new TextEncoder().encode(JSON.stringify(provRecord));
const recordID = await upload(containerClient, deviceKey, data, "prov", "application/json", timestamp);
return {
jsonBody: { record: recordID, attachments } };
const recordID = await upload(containerClient, deviceKey, data, "prov", "application/json", timestamp, undefined);
return { jsonBody: { record: recordID, attachments } };
}
}
// blobNames look like: 'gosqas/63f4b781c0688d83d40908ff368fefa6a2fa4cd470216fd83b3d7d4c642578c0/prov/1a771caa4b15a45ae97b13d7a336e1e9c9ec1c91c70f1dc8f7749440c0af8114'
Expand Down Expand Up @@ -252,7 +286,13 @@ app.post("postProvenance", {
app.get("getAttachment", {
authLevel: 'anonymous',
route: 'attachment/{deviceKey}/{attachmentID}',
handler: getAttachment
handler: getAttachment,
})

app.get("getAttachmentName", {
authLevel: 'anonymous',
route: 'attachment/{deviceKey}/{attachmentID}/name',
handler: getAttachmentName,
})

app.get("getStatistics", {
Expand Down
26 changes: 19 additions & 7 deletions packages/backend/test/test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,17 +20,22 @@ async function getProvRecords(baseUrl: string, deviceKey: string) {
}

async function getAttachment(baseUrl: string, deviceKey: string, attachmentID: string) {
const response = await fetch(`${baseUrl}/attachment/${deviceKey}/${attachmentID}`, {
return await fetch(`${baseUrl}/attachment/${deviceKey}/${attachmentID}`, {
method: "GET",
});
return await response.blob();
}

async function putProvRecord(baseUrl: string, deviceKey: string, record: any, attachments: readonly Blob[]) {
async function getAttachmentName(baseUrl: string, deviceKey: string, attachmentID: string) {
return await fetch(`${baseUrl}/attachment/${deviceKey}/${attachmentID}/name`, {
method: "GET",
});
}

async function putProvRecord(baseUrl: string, deviceKey: string, record: any, attachments: readonly File[]) {
const formData = new FormData();
formData.append("provenanceRecord", JSON.stringify(record));
for (const blob of attachments) {
formData.append("attachment", blob);
formData.append(blob.name, blob as Blob);
}
const response = await fetch(`${baseUrl}/provenance/${deviceKey}`, {
method: "POST",
Expand Down Expand Up @@ -87,14 +92,21 @@ program
const attachment = json[0]?.attachments?.[0];
if (attachment) {
console.log(`Downloading ${attachment}`);
await getAttachment(baseUrl, testDeviceKey, attachment!);
const resp = await getAttachment(baseUrl, testDeviceKey, attachment);
console.log("Headers");
for (const [key, value] of resp.headers) {
console.log(` ${key}: ${value}`);
}
const resp2 = await getAttachmentName(baseUrl, testDeviceKey, attachment);
const name = await resp2.text();
console.log({name});
}
}
})

program.parse(process.argv);

async function getTestImages(): Promise<readonly Blob[]> {
async function getTestImages(): Promise<readonly File[]> {
const images = new Array<File>();
for (const fileName of await readdir(__dirname)) {
const ext = extname(fileName);
Expand All @@ -104,5 +116,5 @@ async function getTestImages(): Promise<readonly Blob[]> {
const file = new File([buffer], fileName, { type });
images.push(file)
}
return images as readonly Blob[];
return images;
}
Loading