diff --git a/deployment/frontend/src/pages/search.tsx b/deployment/frontend/src/pages/search.tsx
new file mode 100644
index 000000000..1de85bc6e
--- /dev/null
+++ b/deployment/frontend/src/pages/search.tsx
@@ -0,0 +1,20 @@
+import Search from '@/components/Search'
+import Header from '@/components/_shared/Header'
+import Highlights from '@/components/Highlights'
+import Recent from '@/components/Recent'
+import Footer from '@/components/_shared/Footer'
+
+
+
+export default function test() {
+ return (
+ <>
+
+
+
+
+
+
+
+ )
+}
diff --git a/deployment/frontend/src/pages/search_advanced.tsx b/deployment/frontend/src/pages/search_advanced.tsx
new file mode 100644
index 000000000..91f27a578
--- /dev/null
+++ b/deployment/frontend/src/pages/search_advanced.tsx
@@ -0,0 +1,28 @@
+import Search from "@/components/Search";
+import Footer from "@/components/_shared/Footer";
+import Header from "@/components/_shared/Header";
+import Pagination from "@/components/_shared/Pagination";
+import DatasetHorizontalCard from "@/components/search/DatasetHorizontalCard";
+import FilteredSearchLayout from "@/components/search/FilteredSearchLayout";
+import FiltersSelected from "@/components/search/FiltersSelected";
+import SortBy from "@/components/search/SortBy";
+
+export default function SearchPage() {
+ return (
+ <>
+
+
+
+
+
+
+ {[0, 1, 2, 3, 4, 5].map((number) => (
+
+ ))}
+
+
+
+
+
+ );
+}
diff --git a/deployment/frontend/src/pages/teams/[teamsName].tsx b/deployment/frontend/src/pages/teams/[teamsName].tsx
new file mode 100644
index 000000000..14f3b316e
--- /dev/null
+++ b/deployment/frontend/src/pages/teams/[teamsName].tsx
@@ -0,0 +1,24 @@
+import React from 'react'
+import Header from '@/components/_shared/Header'
+import TeamHeaderCard from '@/components/team/TeamHeaderCard'
+import TeamTab from '@/components/team/TeamTab'
+import Footer from '@/components/_shared/Footer'
+import { Breadcrumbs } from "@/components/_shared/Breadcrumbs";
+
+
+const links = [
+ { label: "Teams", url: "/teams", current: false },
+ { label: "Team 1", url: "/topics/team1", current: true },
+];
+
+export default function teams() {
+ return (
+ <>
+
+
+
+
+
+
+ )
+}
diff --git a/deployment/frontend/src/pages/teams/index.tsx b/deployment/frontend/src/pages/teams/index.tsx
new file mode 100644
index 000000000..d02944f2a
--- /dev/null
+++ b/deployment/frontend/src/pages/teams/index.tsx
@@ -0,0 +1,21 @@
+import Header from "@/components/_shared/Header";
+import Footer from "@/components/_shared/Footer";
+import TeamsSearch from "@/components/team/TeamsSearch";
+import TeamsSearchResults from "@/components/team/TeamsSearchResults";
+
+export default function TeamsPage() {
+
+ return (
+ <>
+
+
+
+
+
+ );
+}
diff --git a/deployment/frontend/src/pages/topics/[topicName].tsx b/deployment/frontend/src/pages/topics/[topicName].tsx
new file mode 100644
index 000000000..ff9b822b3
--- /dev/null
+++ b/deployment/frontend/src/pages/topics/[topicName].tsx
@@ -0,0 +1,33 @@
+import Header from "@/components/_shared/Header";
+import Footer from "@/components/_shared/Footer";
+import { Breadcrumbs } from "@/components/_shared/Breadcrumbs";
+import DatasetHorizontalCard from "@/components/search/DatasetHorizontalCard";
+import { Hero } from "@/components/topics/Hero";
+import Pagination from "@/components/_shared/Pagination";
+import Subtopics from "@/components/topics/Subtopics";
+
+const links = [
+ { label: "Topics", url: "/topics", current: false },
+ { label: "Topics 1", url: "/topics/test", current: true },
+];
+
+export default function TopicPage() {
+ return (
+ <>
+
+
+
+
+
+
+ Datasets associated with Topic 1 (784)
+
+ {[0, 1, 2, 3, 4, 5].map((number) => (
+
+ ))}
+
+
+
+
+ );
+}
diff --git a/deployment/frontend/src/pages/topics/index.tsx b/deployment/frontend/src/pages/topics/index.tsx
new file mode 100644
index 000000000..b59698804
--- /dev/null
+++ b/deployment/frontend/src/pages/topics/index.tsx
@@ -0,0 +1,23 @@
+import Search from "@/components/Search";
+import Header from "@/components/_shared/Header";
+import Highlights from "@/components/Highlights";
+import Recent from "@/components/Recent";
+import Footer from "@/components/_shared/Footer";
+import TopicsSearchResults from "@/components/topics/TopicsSearchResults";
+import TopicsSearch from "@/components/topics/TopicsSearch";
+
+export default function TopicsPage() {
+ return (
+ <>
+
+
+
+
+
+ );
+}
diff --git a/ckan-deployment/frontend/src/schema/ckan.schema.ts b/deployment/frontend/src/schema/ckan.schema.ts
similarity index 100%
rename from ckan-deployment/frontend/src/schema/ckan.schema.ts
rename to deployment/frontend/src/schema/ckan.schema.ts
diff --git a/ckan-deployment/frontend/src/server/api/root.ts b/deployment/frontend/src/server/api/root.ts
similarity index 100%
rename from ckan-deployment/frontend/src/server/api/root.ts
rename to deployment/frontend/src/server/api/root.ts
diff --git a/ckan-deployment/frontend/src/server/api/routers/example.ts b/deployment/frontend/src/server/api/routers/example.ts
similarity index 100%
rename from ckan-deployment/frontend/src/server/api/routers/example.ts
rename to deployment/frontend/src/server/api/routers/example.ts
diff --git a/ckan-deployment/frontend/src/server/api/trpc.ts b/deployment/frontend/src/server/api/trpc.ts
similarity index 100%
rename from ckan-deployment/frontend/src/server/api/trpc.ts
rename to deployment/frontend/src/server/api/trpc.ts
diff --git a/ckan-deployment/frontend/src/server/auth.ts b/deployment/frontend/src/server/auth.ts
similarity index 100%
rename from ckan-deployment/frontend/src/server/auth.ts
rename to deployment/frontend/src/server/auth.ts
diff --git a/deployment/frontend/src/styles/globals.css b/deployment/frontend/src/styles/globals.css
new file mode 100644
index 000000000..2532cecfc
--- /dev/null
+++ b/deployment/frontend/src/styles/globals.css
@@ -0,0 +1,23 @@
+@tailwind base;
+@tailwind components;
+@tailwind utilities;
+
+body, html {
+ letter-spacing: 0px;
+ accent-color: #32864B;
+}
+
+.mapboxgl-ctrl-bottom-left, .mapboxgl-ctrl-attrib{
+ display: none;
+}
+
+.mapboxgl-ctrl-bottom-right {
+ width: 100%;
+}
+
+.mapboxgl-ctrl-geocoder.mapboxgl-ctrl {
+ width: 100%;
+ max-width: 95%;
+ margin: 10px auto;
+ float: none;
+}
diff --git a/ckan-deployment/frontend/src/utils/api.ts b/deployment/frontend/src/utils/api.ts
similarity index 100%
rename from ckan-deployment/frontend/src/utils/api.ts
rename to deployment/frontend/src/utils/api.ts
diff --git a/ckan-deployment/frontend/src/utils/classnames.ts b/deployment/frontend/src/utils/classnames.ts
similarity index 100%
rename from ckan-deployment/frontend/src/utils/classnames.ts
rename to deployment/frontend/src/utils/classnames.ts
diff --git a/deployment/frontend/tailwind.config.ts b/deployment/frontend/tailwind.config.ts
new file mode 100644
index 000000000..0c9449144
--- /dev/null
+++ b/deployment/frontend/tailwind.config.ts
@@ -0,0 +1,41 @@
+import { type Config } from "tailwindcss";
+import defaultTheme from "tailwindcss/defaultTheme";
+
+export default {
+ content: ["./src/**/*.{js,ts,jsx,tsx}"],
+ theme: {
+ extend: {
+ fontFamily: {
+ sans: ["Inter", ...defaultTheme.fontFamily.sans],
+ acumin: ['var(--font-acumin)', ...defaultTheme.fontFamily.sans],
+ },
+ boxShadow: {
+ "wri": "0px 4px 4px 0px rgba(147, 147, 147, 0.25)",
+ },
+ colors: {
+ white: "#FFFFFF",
+ "wri-gold": "#F3B229",
+ "wri-black": "#1A1919",
+ "wri-green": "#32864B",
+ "wri-gray": "#eae8e4",
+ "wri-dark-green": "#2B7340",
+ "wri-light-green": "#BAE1BD",
+ "wri-light-blue": "#B5D6E8",
+ "wri-light-yellow": "#FBE8BE",
+ "wri-dark-gray": "#666666"
+ },
+ screens: {
+ "4xl": "2048px",
+ "3xl": "1920px",
+ "xxl": "1440px",
+ },
+ maxWidth: {
+ "8xl": "1350px",
+ "9xl": "1440px",
+ }
+ },
+ },
+ plugins: [
+ require('@tailwindcss/container-queries')
+ ],
+} satisfies Config;
diff --git a/ckan-deployment/frontend/tsconfig.json b/deployment/frontend/tsconfig.json
similarity index 100%
rename from ckan-deployment/frontend/tsconfig.json
rename to deployment/frontend/tsconfig.json
diff --git a/ckan-deployment/helm-templates/Chart.lock b/deployment/helm-templates/Chart.lock
similarity index 100%
rename from ckan-deployment/helm-templates/Chart.lock
rename to deployment/helm-templates/Chart.lock
diff --git a/ckan-deployment/helm-templates/Chart.yaml b/deployment/helm-templates/Chart.yaml
similarity index 100%
rename from ckan-deployment/helm-templates/Chart.yaml
rename to deployment/helm-templates/Chart.yaml
diff --git a/ckan-deployment/helm-templates/charts/ckan-v3.1.2-wri.tgz b/deployment/helm-templates/charts/ckan-v3.1.2-wri.tgz
similarity index 100%
rename from ckan-deployment/helm-templates/charts/ckan-v3.1.2-wri.tgz
rename to deployment/helm-templates/charts/ckan-v3.1.2-wri.tgz
diff --git a/ckan-deployment/helm-templates/charts/redis-16.12.1.tgz b/deployment/helm-templates/charts/redis-16.12.1.tgz
similarity index 100%
rename from ckan-deployment/helm-templates/charts/redis-16.12.1.tgz
rename to deployment/helm-templates/charts/redis-16.12.1.tgz
diff --git a/ckan-deployment/helm-templates/charts/solr-v1.1.0.tgz b/deployment/helm-templates/charts/solr-v1.1.0.tgz
similarity index 100%
rename from ckan-deployment/helm-templates/charts/solr-v1.1.0.tgz
rename to deployment/helm-templates/charts/solr-v1.1.0.tgz
diff --git a/ckan-deployment/helm-templates/values.yaml.staging.template b/deployment/helm-templates/values.yaml.staging.template
similarity index 85%
rename from ckan-deployment/helm-templates/values.yaml.staging.template
rename to deployment/helm-templates/values.yaml.staging.template
index a6993d4d1..8b9a41e97 100644
--- a/ckan-deployment/helm-templates/values.yaml.staging.template
+++ b/deployment/helm-templates/values.yaml.staging.template
@@ -29,11 +29,15 @@ ckan:
CKAN__SITE_TITLE: WRI Staging
CKAN__STORAGE_PATH: /tmp
CKAN__VIEWS__DEFAULT_VIEWS: recline_view text_view image_view pdf_view geojson_view
- CKAN__PLUGINS: image_view text_view webpage_view resource_proxy datatables_view datastore datapusher activity s3filestore envvars
+ CKAN__PLUGINS: image_view text_view webpage_view resource_proxy datatables_view datastore datapusher activity s3filestore scheming_datasets scheming_organizations scheming_groups wri auth envvars
CKAN__AUTH__CREATE_UNOWNED_DATASET: "True"
CKAN__AUTH__ALLOW_DATASET_COLLABORATORS: "True"
CKAN__AUTH__ALLOW_ADMIN_COLLABORATORS: "True"
CKAN__AUTH__ALLOW_COLLABORATORS_TO_CHANGE_OWNER_ORG: "True"
+ CKAN___SCHEMING__DATASET_SCHEMAS: ckanext.wri.schema:ckan_dataset.yaml
+ CKAN___SCHEMING__ORGANIZATION_SCHEMAS: ckanext.scheming:custom_org_with_address.json
+ CKAN___SCHEMING__GROUP_SCHEMAS: ckanext.scheming:custom_group_with_status.json
+ CKAN___SCHEMING__PRESETS: ckanext.wri.schema:presets.json
hpa:
enable: true
minReplicas: 2
@@ -48,7 +52,7 @@ ckan:
memory: 1G
image:
pullPolicy: Always
- repository: {{REGISTRY}}/{{REPOSITORY}}
+ repository: {{REGISTRY}}/{{CKAN_REPO}}
tag: {{GITHUB_SHA}}
runInitContainers: false
service:
@@ -71,10 +75,29 @@ ckan:
maxReplicas: 2
targetMemoryUtilizationPercentage: 80
image:
- tag: wri
+ tag: latest
pullPolicy: Always
- repository: viderum/datapusher
+ repository: {{REGISTRY}}/{{DATAPUSHER_REPO}}
replicaCount: 1
+ frontend:
+ enable: true
+ hpa:
+ enable: false
+ resources:
+ limits:
+ cpu: 400m
+ memory: 2.2G
+ requests:
+ cpu: 200m
+ memory: 1.8G
+ replicaCount: 1
+ enable: true
+ image:
+ repository: {{REGISTRY}}/{{FRONTEND_REPO}}
+ tag: {{GITHUB_SHA}}
+ env:
+ SITE_TITLE: WRI ODP
+ targetPort: 3000
general:
autoscaling:
enable: false
@@ -109,14 +132,21 @@ ckan:
- path: /(.*)
port: 80
service: wri-staging-ckan-svc
+ - host: wri.staging.frontend.datopian.com
+ isExternal: false
+ paths:
+ - path: /(.*)
+ port: 80
+ service: wri-staging-frontend-svc
ingressNamespace: nginx-ingress
ingressClass: nginx
tls:
- hosts:
- wri.staging.ckan.datopian.com
secretName: wri.staging.ckan.datopian.com
- frontend:
- enable: false
+ - hosts:
+ - wri.staging.frontend.datopian.com
+ secretName: wri.staging.frontend.datopian.com
giftless:
enable: false
auth:
@@ -140,6 +170,16 @@ ckan:
requests:
memory: 2.8G
cpu: 0.2
+ ckanmiddleware:
+ enable: false
+ dataSubscriptions:
+ enable: false
+ giftless:
+ enable: false
+ postgresql:
+ enable: false
+ xloader:
+ enable: false
solr:
nodeSelector: {}
image:
diff --git a/integration-tests/test.js b/integration-tests/test.js
index 7d05a511a..9a30dc24b 100644
--- a/integration-tests/test.js
+++ b/integration-tests/test.js
@@ -6,6 +6,15 @@ assets.addSpecs(".", ["ckan-classic-auth", "ckan-classic-api"]);
cypress
.run(assets.options)
- .then(console.log)
- .catch(console.error)
- .finally(() => assets.cleanUp());
+ .then((results) => {
+ if (results.totalFailed > 0) {
+ process.exit(1);
+ }
+ })
+ .catch((error) => {
+ console.error(error);
+ process.exit(1);
+ })
+ .finally(() => {
+ assets.cleanUp();
+ });
diff --git a/scripts/cost-splitting/data/storage_costs.csv b/scripts/cost-splitting/data/storage_costs.csv
index 984c97504..9b7a10560 100644
--- a/scripts/cost-splitting/data/storage_costs.csv
+++ b/scripts/cost-splitting/data/storage_costs.csv
@@ -1,3 +1,3 @@
date,org,storage,percentage
2023-10-05T14:03:33.047173,total,0,1
-2023-10-05T15:29:29.607489,total,0,1
+2023-10-05T15:29:29.607489,total,0,1
\ No newline at end of file
diff --git a/scripts/cost-splitting/docs/README.md b/scripts/cost-splitting/docs/README.md
index a92c66edd..e4c1d014c 100644
--- a/scripts/cost-splitting/docs/README.md
+++ b/scripts/cost-splitting/docs/README.md
@@ -30,12 +30,13 @@ Once this is done you will possibly have three things
The code can be seen in
-- https://github.com/wri/wri-odp/blob/main/scripts/cost-splitting/script.py
-- https://github.com/wri/wri-odp/blob/main/scripts/cost-splitting/lib.py
-- https://github.com/wri/wri-odp/blob/main/.github/workflows/cost-splitting-report.yml
+- https://github.com/wri/wri-odp/blob/staging/scripts/cost-splitting/script.py
+- https://github.com/wri/wri-odp/blob/staging/scripts/cost-splitting/lib.py
+- https://github.com/wri/wri-odp/blob/staging/.github/workflows/cost-splitting-report.yml
It will basically
+- Get the name of the bucket which is going to be `ckan-{NAME OF THE BRANCH}-storage`, thats because in our repo the 3 branches(staging, dev, prod) match the 3 environments that we have, so you can run the script by doing it against an specific branch
- Get a list of all top level folders in the S3 Bucket, assuming that the bucket has a structure similar to this
```
org_1/resources/{resourceId}/filename.csv
@@ -59,4 +60,4 @@ Besides this there are two more caveats
We are testing mostly the logic that decides the percentage costs, we use `moto` whic is a library that allows us to mock S3 Services, we then put two files with the "hello world" content, which takes exactly 11 bytes and then check if those results match what we expect
-The test files can be found in https://github.com/wri/wri-odp/blob/main/scripts/cost-splitting/test_script.py
+The test files can be found in https://github.com/wri/wri-odp/blob/staging/scripts/cost-splitting/test_script.py
diff --git a/scripts/cost-splitting/lib.py b/scripts/cost-splitting/lib.py
index 95affe369..be147860a 100644
--- a/scripts/cost-splitting/lib.py
+++ b/scripts/cost-splitting/lib.py
@@ -1,10 +1,22 @@
import boto3
import smtplib
+import requests
from email.mime.text import MIMEText
-
-def build_email(current_date):
- body = f"Please access
here to get a report with the aggregate of storage costs, you can also get a version with just the current values at
this link "
+def get_org_name(org_id, branch_name):
+ # make an http request to the get the org name using organization_show from ckan
+ # make a get request
+ try:
+ r = requests.get(f'https://wri.{branch_name}.ckan.datopian.com/api/action/organization_show?id={org_id}')
+ print(r.json())
+ org = r.json()['result']
+ return org['title']
+ except:
+ print("Coudlnt find org")
+ return org_id
+
+def build_email(current_date, branch_name):
+ body = f"Please access
here to get a report with the aggregate of storage costs, you can also get a version with just the current values at
this link "
return MIMEText(body, "html")
@@ -57,14 +69,14 @@ def get_total_costs(bucket_name, current_date):
return total
-def get_storage_for_every_org(bucket_name, total_cost, orgs, current_date):
+def get_storage_for_every_org(bucket_name, total_cost, orgs, current_date, branch_name):
orgs_storage = []
for org in orgs:
org_storage = get_storage_costs_from_org(bucket_name, org)
orgs_storage.append(
{
"date": current_date,
- "org": org if org != "resources" else "datasets without org",
+ "org": get_org_name(org, branch_name) if org != "resources" else "unwoned resources",
"storage": org_storage,
"percentage": org_storage / total_cost,
}
diff --git a/scripts/cost-splitting/poetry.lock b/scripts/cost-splitting/poetry.lock
index bc1269519..e5d8b3702 100644
--- a/scripts/cost-splitting/poetry.lock
+++ b/scripts/cost-splitting/poetry.lock
@@ -1176,4 +1176,4 @@ multidict = ">=4.0"
[metadata]
lock-version = "2.0"
python-versions = "^3.11"
-content-hash = "11155867c71c7ebe998fe19f6e19abf86659870df4329c5f2fdeff2cf3623918"
+content-hash = "5610001a471c2e499f3a130908b700de5e3370c2bd8242a36b940ad8a134ed55"
diff --git a/scripts/cost-splitting/poetry.toml b/scripts/cost-splitting/poetry.toml
new file mode 100644
index 000000000..53b35d370
--- /dev/null
+++ b/scripts/cost-splitting/poetry.toml
@@ -0,0 +1,3 @@
+[virtualenvs]
+create = true
+in-project = true
diff --git a/scripts/cost-splitting/pyproject.toml b/scripts/cost-splitting/pyproject.toml
index a1e0deae9..35063d0d1 100644
--- a/scripts/cost-splitting/pyproject.toml
+++ b/scripts/cost-splitting/pyproject.toml
@@ -10,6 +10,7 @@ python = "^3.11"
boto3 = "^1.28.57"
python-dotenv = "^1.0.0"
moto = "^4.2.5"
+requests = "^2.31.0"
[tool.poetry.group.test.dependencies]
pytest = "^7.4.2"
diff --git a/scripts/cost-splitting/script.py b/scripts/cost-splitting/script.py
index 45ed2df3a..d0c8093cf 100644
--- a/scripts/cost-splitting/script.py
+++ b/scripts/cost-splitting/script.py
@@ -15,13 +15,15 @@
## Env variables
load_dotenv()
-BUCKET_NAME = os.getenv("BUCKET_NAME")
EMAIL_RECIPIENTS = os.getenv("EMAIL_RECIPIENTS")
SMTP_SERVER = os.getenv("SMTP_SERVER")
SMTP_SENDER = os.getenv("SMTP_SENDER")
SMTP_USER = os.getenv("SMTP_USER")
SMTP_PORT = os.getenv("SMTP_PORT")
SMTP_PASSWORD = os.getenv("SMTP_PASSWORD")
+BRANCH_NAME = os.getenv("BRANCH_NAME")
+
+BUCKET_NAME = f"ckan-{BRANCH_NAME}-storage"
current_date = datetime.datetime.utcnow().isoformat()
@@ -29,7 +31,7 @@
orgs = get_all_orgs(BUCKET_NAME)
total = get_total_costs(BUCKET_NAME, current_date)
orgs_storage = get_storage_for_every_org(
- BUCKET_NAME, total["storage"], orgs, current_date
+ BUCKET_NAME, total["storage"], orgs, current_date, BRANCH_NAME
)
orgs_storage.append(total)
@@ -37,13 +39,14 @@
csv_string_to_append = get_csv_string(orgs_storage)
csv_string_to_create = get_csv_string(orgs_storage, "date,org,storage,percentage\n")
append_csv_to_file(csv_string_to_append, "data/storage_costs.csv")
-append_csv_to_file(csv_string_to_create, f"data/storage_costs_{current_date}.csv")
+date_path = current_date.replace(".", "_").replace(":", "_").replace("-", "_")
+append_csv_to_file(csv_string_to_create, f"data/storage_costs_{date_path}.csv")
# Send email with link
if EMAIL_RECIPIENTS and EMAIL_RECIPIENTS != "":
send_email(
build_subject(current_date),
- build_email(current_date),
+ build_email(date_path, BRANCH_NAME),
SMTP_SENDER,
EMAIL_RECIPIENTS.split(","),
SMTP_SERVER,
diff --git a/scripts/cost-splitting/test_script.py b/scripts/cost-splitting/test_script.py
index d22a9a609..486521b4f 100644
--- a/scripts/cost-splitting/test_script.py
+++ b/scripts/cost-splitting/test_script.py
@@ -44,5 +44,5 @@ def test_storage_report(s3_boto):
total_cost = get_total_costs(bucket, "")
all_orgs = get_all_orgs(bucket)
assert total_cost['storage'] == 22
- storage_report = get_storage_for_every_org(bucket, total_cost['storage'], all_orgs, "")
+ storage_report = get_storage_for_every_org(bucket, total_cost['storage'], all_orgs, "", "staging")
assert storage_report[0]['storage'] == 11