From 23815b540da598b2c3c2149703aa661411594205 Mon Sep 17 00:00:00 2001 From: abolfazl1381 Date: Tue, 3 Dec 2024 15:39:05 +0330 Subject: [PATCH 01/22] feat(compose): comelete compose prompt --- app/directory_generators/compose_generator.py | 1 - .../MyAnsible/roles/init_k8s/tasks/cni.yml | 20 +++ .../roles/init_k8s/tasks/initk8s.yml | 61 ++++++++ .../MyAnsible/roles/init_k8s/tasks/main.yml | 8 ++ .../MyAnsible/roles/k8s/handlers/main.yml | 12 ++ app/media/MyAnsible/roles/k8s/tasks/k8s.yml | 130 ++++++++++++++++++ app/media/MyAnsible/roles/k8s/tasks/main.yml | 3 + app/media/MyCompose/docker-compose.yaml | 4 + app/models/compose_models.py | 23 +++- app/template_generators/docker/compose.py | 39 +++++- 10 files changed, 298 insertions(+), 3 deletions(-) create mode 100644 app/media/MyAnsible/roles/init_k8s/tasks/cni.yml create mode 100644 app/media/MyAnsible/roles/init_k8s/tasks/initk8s.yml create mode 100644 app/media/MyAnsible/roles/init_k8s/tasks/main.yml create mode 100644 app/media/MyAnsible/roles/k8s/handlers/main.yml create mode 100644 app/media/MyAnsible/roles/k8s/tasks/k8s.yml create mode 100644 app/media/MyAnsible/roles/k8s/tasks/main.yml create mode 100644 app/media/MyCompose/docker-compose.yaml diff --git a/app/directory_generators/compose_generator.py b/app/directory_generators/compose_generator.py index acd5aaec..e69de29b 100644 --- a/app/directory_generators/compose_generator.py +++ b/app/directory_generators/compose_generator.py @@ -1 +0,0 @@ -Hello! It looks like you entered just the letter "M." How can I assist you today? \ No newline at end of file diff --git a/app/media/MyAnsible/roles/init_k8s/tasks/cni.yml b/app/media/MyAnsible/roles/init_k8s/tasks/cni.yml new file mode 100644 index 00000000..516dbff3 --- /dev/null +++ b/app/media/MyAnsible/roles/init_k8s/tasks/cni.yml @@ -0,0 +1,20 @@ +- block: + - name: Check if Calico CRDs exist + command: kubectl get crd felixconfigurations.crd.projectcalico.org + register: calico_crd_check + ignore_errors: true + delegate_to: "{{ groups['k8s_masters'][0] }}" + +- block: + - name: Apply CNI plugin (Calico) + command: kubectl create -f {{ calico_operator_url }} + retries: 3 + delay: 3 + + - name: Apply CNI plugin (Calico) + command: kubectl create -f {{ calico_crd_url }} + retries: 3 + delay: 3 + delegate_to: "{{ groups['k8s_masters'][0] }}" + when: calico_crd_check.rc != 0 + run_once: true diff --git a/app/media/MyAnsible/roles/init_k8s/tasks/initk8s.yml b/app/media/MyAnsible/roles/init_k8s/tasks/initk8s.yml new file mode 100644 index 00000000..ff134a27 --- /dev/null +++ b/app/media/MyAnsible/roles/init_k8s/tasks/initk8s.yml @@ -0,0 +1,61 @@ +- name: Init cluster | Check if kubeadm has already run + stat: + path: "/var/lib/kubelet/config.yaml" + register: kubeadm_already_run + when: inventory_hostname == groups['k8s_masters'][0] + delegate_to: "{{ groups['k8s_masters'][0] }}" + +- block: + - name: Init cluster | Copy kubeadmcnf.yaml + template: + src: kubeadmcnf.yml.j2 + dest: /root/kubeadmcnf.yaml + + - name: Init cluster | Initiate cluster on node groups['kube_master'][0] + shell: kubeadm init --config=/root/kubeadmcnf.yaml + register: kubeadm_init + until: kubeadm_init is succeeded or "field is immutable" in kubeadm_init.stderr + notify: Restart kubelet + + when: inventory_hostname == groups['k8s_masters'][0] and not kubeadm_already_run.stat.exists + delegate_to: "{{ groups['k8s_masters'][0] }}" + +- block: + - name: Create kubectl directory + file: + path: /root/.kube + state: directory + + - name: Configure kubectl + copy: + src: /etc/kubernetes/admin.conf + dest: /root/.kube/config + remote_src: yes + + - name: Fetch kubeconfig + fetch: + src: /etc/kubernetes/admin.conf + dest: kubeconfig/ + flat: yes + when: inventory_hostname == groups['k8s_masters'][0] + delegate_to: "{{ groups['k8s_masters'][0] }}" + +- name: Sleep for 300 seconds and reboot the Master1 server + wait_for: + timeout: 300 + delegate_to: localhost + +- name: Reboot the servers + command: reboot + async: 1 + poll: 0 + delegate_to: "{{ groups['k8s_masters'][0] }}" + +- name: Sleep for 300 seconds to Master1 up and running + wait_for: + timeout: 300 + delegate_to: localhost + +- name: Example Task After Reboot + debug: + msg: "Server back online and ready for tasks." diff --git a/app/media/MyAnsible/roles/init_k8s/tasks/main.yml b/app/media/MyAnsible/roles/init_k8s/tasks/main.yml new file mode 100644 index 00000000..bb40ddec --- /dev/null +++ b/app/media/MyAnsible/roles/init_k8s/tasks/main.yml @@ -0,0 +1,8 @@ +--- +# tasks file for init_k8s + +- name: Initialize kubernetes cluster + include_tasks: initk8s.yml + +- name: Initialize Calico CNI + include_tasks: cni.yml diff --git a/app/media/MyAnsible/roles/k8s/handlers/main.yml b/app/media/MyAnsible/roles/k8s/handlers/main.yml new file mode 100644 index 00000000..de036f51 --- /dev/null +++ b/app/media/MyAnsible/roles/k8s/handlers/main.yml @@ -0,0 +1,12 @@ +--- +# handlers file for k8s + +- name: Remove temporary GPG key file + file: + path: "/tmp/docker.list" + state: absent + +- name: Restart kubelet + service: + name: kubelet + state: restarted diff --git a/app/media/MyAnsible/roles/k8s/tasks/k8s.yml b/app/media/MyAnsible/roles/k8s/tasks/k8s.yml new file mode 100644 index 00000000..a346e99b --- /dev/null +++ b/app/media/MyAnsible/roles/k8s/tasks/k8s.yml @@ -0,0 +1,130 @@ +- name: Disable SWAP since kubernetes can't work with swap enabled + shell: | + swapoff -a + +- name: Disable SWAP in fstab since kubernetes can't work with swap enabled + replace: + path: /etc/fstab + regexp: '^([^#].*?\sswap\ssw\s+.*)$' + replace: '# \1' + +- name: Check if ufw is installed + package_facts: + manager: "auto" + +- name: Disable ufw # just in Ubuntu + ufw: + state: disabled + when: "'ufw' in ansible_facts.packages" + +- name: Ensure kernel modules for containerd are enabled + lineinfile: + path: /etc/modules-load.d/containerd.conf + line: "{{ item }}" + create: yes + state: present + loop: + - overlay + - br_netfilter + +- name: Load kernel modules + command: + cmd: "modprobe {{ item }}" + loop: + - overlay + - br_netfilter + +- name: Ensure sysctl settings for Kubernetes are present + blockinfile: + path: /etc/sysctl.d/kubernetes.conf + block: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + net.ipv4.ip_forward = 1 + create: yes + marker: "# {mark} ANSIBLE MANAGED BLOCK" + owner: root + group: root + mode: '0644' + +- name: Reload sysctl settings + command: + cmd: sysctl --system + +- name: Update apt cache + apt: + update_cache: yes + +- name: Install required packages + apt: + pkg: + - ca-certificates + - curl + - gnupg + - lsb-release + - gpg + state: present + update_cache: yes + +- name: Ensure the /etc/apt/keyrings directory exists + file: + path: /etc/apt/keyrings + state: directory + mode: '0755' # Adjust the permissions as necessary + owner: root # Set the owner, if required + group: root + +- name: Remove existing Docker GPG key if it exists + file: + path: '{{ docker_gpg_key_path }}' + state: absent + +- name: Download Docker GPG key + shell: | + curl -fsSL {{ docker_gpg_key_url }} | gpg --dearmor -o {{ docker_gpg_key_path }} + +- name: Determine the architecture + command: dpkg --print-architecture + register: architecture + +- name: Determine the distribution codename + command: lsb_release -cs + register: distribution_codename + +- name: Add Docker APT repository + lineinfile: + path: /etc/apt/sources.list.d/docker.list + create: yes + line: "deb [arch={{ architecture.stdout }} signed-by={{ docker_gpg_key_path }}] {{ docker_apt_repo }} {{ distribution_codename.stdout }} stable" + state: present + +- name: Update apt cache + apt: + update_cache: yes + +- name: Install required packages (containerd) + apt: + pkg: + - containerd.io + state: present + +- name: Generate default containerd configuration + shell: + cmd: containerd config default > /etc/containerd/config.toml + +- name: Replace SystemdCgroup from false to true in containerd config + replace: + path: /etc/containerd/config.toml + regexp: 'SystemdCgroup = false' + replace: 'SystemdCgroup = true' + +- name: Restart containerd service + systemd: + name: containerd + state: restarted + daemon_reload: yes + +- name: Enable containerd service + systemd: + name: containerd + enabled: yes diff --git a/app/media/MyAnsible/roles/k8s/tasks/main.yml b/app/media/MyAnsible/roles/k8s/tasks/main.yml new file mode 100644 index 00000000..a0ac6054 --- /dev/null +++ b/app/media/MyAnsible/roles/k8s/tasks/main.yml @@ -0,0 +1,3 @@ +--- +- name: Install kubernetes packages + include_tasks: k8s.yml diff --git a/app/media/MyCompose/docker-compose.yaml b/app/media/MyCompose/docker-compose.yaml new file mode 100644 index 00000000..e48d178b --- /dev/null +++ b/app/media/MyCompose/docker-compose.yaml @@ -0,0 +1,4 @@ +services: + web_server: + image: nginx:latest + build: diff --git a/app/models/compose_models.py b/app/models/compose_models.py index 163ee3cb..4294f751 100644 --- a/app/models/compose_models.py +++ b/app/models/compose_models.py @@ -1,5 +1,5 @@ from typing import List, Optional -from pydantic import BaseModel, validator, ValidationError +from pydantic import BaseModel, validator, ValidationError, computed_field class Port(BaseModel): machine_port:int = 80 @@ -12,9 +12,19 @@ class EnvironmentVariable(BaseModel): name:str = 'foo' value:str = "bar" + @computed_field + @property + def env_full(self) -> int: + return f"{self.name}:{self.value}" + class Volume(BaseModel): local_dir: str = './nginx/nginx.conf' container_dir:str = '/etc/nginx/nginx.conf' + + @computed_field + @property + def volume(self) -> int: + return f"{self.local_dir}:{self.container_dir}" class Build(BaseModel): context:str @@ -30,6 +40,17 @@ class Service(BaseModel): networks:List[Network] environments:List[EnvironmentVariable] + @computed_field + @property + def image_full(self) -> int: + return f"{self.image}:{self.version}" + + @computed_field + @property + def volumes_full(self) -> int: + return [i.volume for i in self.volumes] + + class DockerCompose(BaseModel): services: List[Service] diff --git a/app/template_generators/docker/compose.py b/app/template_generators/docker/compose.py index ae8e052d..79b739ff 100644 --- a/app/template_generators/docker/compose.py +++ b/app/template_generators/docker/compose.py @@ -1,3 +1,40 @@ def docker_compose_generator(input): - prompt = """M""" + compose_services = input.services + services = [i.container_name for i in compose_services] + images = [{i.container_name:i.image_full} for i in compose_services] + volumes = [{i.container_name:i.volumes_full} for i in compose_services] + depends_on = [{i.container_name:i.depends_on} for i in compose_services] + ports = [{i.container_name:i.ports} for i in compose_services] + env = [{i.container_name:i.environments} for i in compose_services] + networks = [{i.container_name:i.networks} for i in compose_services] + + + prompt = f""" + + generate a python code (with out any ```python entry or additionals) with generates a docker-compose.yaml file in the directory 'app/media/MyCompose' + + + + + finally just give me a python code without any note that can generate a project folder with the + given schema without ```python entry. and we dont need any base directory in the python code. + the final ansible template must work very well without any error! + + the python code you give me, must have structure like that: + + import os + project_name = "app/media/MyCompose" + foo_dir = os.path.join(project_name, "bar") + x_dir = os.path.join(modules_dir, "y") + + # Create project directories + os.makedirs(compose_dir, exist_ok=True) + + # Create main.tf + with open(os.path.join(project_name, "main.tf"), "w") as main_file: + # any thing you need + + + + """ return prompt \ No newline at end of file From 7c593ab1266599c8d7bca6634a84a637b6020045 Mon Sep 17 00:00:00 2001 From: abolfazl1381 Date: Tue, 3 Dec 2024 15:40:19 +0330 Subject: [PATCH 02/22] fix(kuber): remove lb --- app/models/ansible_models.py | 1 - app/routes/ansible.py | 5 +---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/app/models/ansible_models.py b/app/models/ansible_models.py index 748da389..7d80f39c 100644 --- a/app/models/ansible_models.py +++ b/app/models/ansible_models.py @@ -37,7 +37,6 @@ class AnsibleInstallKuber(AnsibleBase): os: str = 'ubuntu' k8s_worker_nodes: List[str] k8s_master_nodes: List[str] - lb_nodes: List[str] version:str = "1.31" diff --git a/app/routes/ansible.py b/app/routes/ansible.py index a3c61656..f24fedf9 100644 --- a/app/routes/ansible.py +++ b/app/routes/ansible.py @@ -48,9 +48,6 @@ async def ansible_install_generation_kuber(request:AnsibleInstallKuber) -> Outpu add_files_to_folder(files = ['app/media/kuber_configs/resolv.conf.j2'] , folder='app/media/MyAnsible/roles/preinstall/templates/') add_files_to_folder(files = ['app/media/kuber_configs/kubeadmcnf.yml.j2'] , folder='app/media/MyAnsible/roles/init_k8s/templates/') add_files_to_folder(files = ['app/media/kuber_configs/kubeadmcnf-join.yml.j2'] , folder='app/media/MyAnsible/roles/join_master/templates/') - add_files_to_folder(files = ['app/media/kuber_configs/check_apiserveer.sh.j2', - 'app/media/kuber_configs/haproxy.cfg.j2', - 'app/media/kuber_configs/keepalived.conf.j2' - ] , folder='app/media/MyAnsible/roles/lb/templates/') + return Output(output='output') \ No newline at end of file From 91a80bdf1dd14a44d35eb54c1c8cf13d4cf8d5e4 Mon Sep 17 00:00:00 2001 From: abolfazl1381 Date: Tue, 3 Dec 2024 17:33:38 +0330 Subject: [PATCH 03/22] feat(compose): compelete compose prompt --- app/directory_generators/compose_generator.py | 39 +++++++++++++++++++ app/media/MyCompose/docker-compose.yaml | 27 ++++++++++++- app/template_generators/docker/compose.py | 21 +++++++++- 3 files changed, 85 insertions(+), 2 deletions(-) diff --git a/app/directory_generators/compose_generator.py b/app/directory_generators/compose_generator.py index e69de29b..0627c345 100644 --- a/app/directory_generators/compose_generator.py +++ b/app/directory_generators/compose_generator.py @@ -0,0 +1,39 @@ +import os + +project_name = "app/media/MyCompose" +compose_file_path = os.path.join(project_name, "docker-compose.yaml") + +# Create project directories +os.makedirs(project_name, exist_ok=True) + +# Create docker-compose.yaml +with open(compose_file_path, "w") as compose_file: + compose_file.write("version: '3'\n") + compose_file.write("services:\n") + compose_file.write(" web_server:\n") + compose_file.write(" image: nginx:latest\n") + compose_file.write(" volumes:\n") + compose_file.write(" - ./nginx/nginx.conf:/etc/nginx/nginx.conf\n") + compose_file.write(" depends_on:\n") + compose_file.write(" - string\n") + compose_file.write(" ports:\n") + compose_file.write(" - '80:80'\n") + compose_file.write(" environment:\n") + compose_file.write(" - foo=bar\n") + compose_file.write(" networks:\n") + compose_file.write(" - app_network\n") + compose_file.write(" monitoring_server:\n") + compose_file.write(" image: grafana:latest\n") + compose_file.write(" volumes:\n") + compose_file.write(" - ./nginx/nginx.conf:/etc/nginx/nginx.conf\n") + compose_file.write(" depends_on:\n") + compose_file.write(" - string\n") + compose_file.write(" ports:\n") + compose_file.write(" - '82:80'\n") + compose_file.write(" environment:\n") + compose_file.write(" - foo=bar\n") + compose_file.write(" networks:\n") + compose_file.write(" - app_network\n") + compose_file.write("networks:\n") + compose_file.write(" app_network:\n") + compose_file.write(" driver: bridge\n") \ No newline at end of file diff --git a/app/media/MyCompose/docker-compose.yaml b/app/media/MyCompose/docker-compose.yaml index e48d178b..5a225be0 100644 --- a/app/media/MyCompose/docker-compose.yaml +++ b/app/media/MyCompose/docker-compose.yaml @@ -1,4 +1,29 @@ +version: '3' services: web_server: image: nginx:latest - build: + volumes: + - ./nginx/nginx.conf:/etc/nginx/nginx.conf + depends_on: + - string + ports: + - '80:80' + environment: + - foo=bar + networks: + - app_network + monitoring_server: + image: grafana:latest + volumes: + - ./nginx/nginx.conf:/etc/nginx/nginx.conf + depends_on: + - string + ports: + - '82:80' + environment: + - foo=bar + networks: + - app_network +networks: + app_network: + driver: bridge diff --git a/app/template_generators/docker/compose.py b/app/template_generators/docker/compose.py index 79b739ff..f2589073 100644 --- a/app/template_generators/docker/compose.py +++ b/app/template_generators/docker/compose.py @@ -1,4 +1,5 @@ def docker_compose_generator(input): + compose_network = input.network.name compose_services = input.services services = [i.container_name for i in compose_services] images = [{i.container_name:i.image_full} for i in compose_services] @@ -13,13 +14,31 @@ def docker_compose_generator(input): generate a python code (with out any ```python entry or additionals) with generates a docker-compose.yaml file in the directory 'app/media/MyCompose' - + the docker-compose.yaml, must following there instructions: + the version must be = 3 + set services following this list: {services} + set images to serivce following this dict : {images} + set volumes to service following this dict : {volumes} + set depends_on to service following this dict : {depends_on} + set ports to service following this dict : {ports} + set environment to service following this dict : {env} + set netwotks to service following this dict : {networks} + + + finally, at the end of docker-compose file, add following block: + ``` + networks: + {compose_network}: + driver: bridge + + ``` finally just give me a python code without any note that can generate a project folder with the given schema without ```python entry. and we dont need any base directory in the python code. the final ansible template must work very well without any error! + the python code you give me, must have structure like that: import os From 72814e8115d56c779f43a8a33539875af855b9ff Mon Sep 17 00:00:00 2001 From: abolfazl1381 Date: Tue, 3 Dec 2024 17:51:05 +0330 Subject: [PATCH 04/22] nothing --- app/media/MyCompose/docker-compose.yaml | 2 + .../ansible/install/kuber.py | 1677 ----------------- 2 files changed, 2 insertions(+), 1677 deletions(-) diff --git a/app/media/MyCompose/docker-compose.yaml b/app/media/MyCompose/docker-compose.yaml index 5a225be0..1bbac8cc 100644 --- a/app/media/MyCompose/docker-compose.yaml +++ b/app/media/MyCompose/docker-compose.yaml @@ -12,6 +12,7 @@ services: - foo=bar networks: - app_network + monitoring_server: image: grafana:latest volumes: @@ -24,6 +25,7 @@ services: - foo=bar networks: - app_network + networks: app_network: driver: bridge diff --git a/app/template_generators/ansible/install/kuber.py b/app/template_generators/ansible/install/kuber.py index 356d0a12..e69de29b 100644 --- a/app/template_generators/ansible/install/kuber.py +++ b/app/template_generators/ansible/install/kuber.py @@ -1,1677 +0,0 @@ -def ansible_kuber_install(input): - - kubernetes_ansible_port = input.ansible_port - kubernetes_ansible_user = input.ansible_user - k8s_master_nodes = input.k8s_master_nodes - k8s_worker_nodes = input.k8s_worker_nodes - k8s_version = input.version - sections = { - "[all]": [f"{name} private_ip=x.x.x.x" for name in k8s_master_nodes + k8s_worker_nodes], - "[k8s]": k8s_master_nodes + k8s_worker_nodes, - "[k8s_masters]": k8s_master_nodes, - "[k8s_workers]": k8s_worker_nodes, - } - kubernetes_inventory = "\n\n".join(f"{section}\n" + "\n".join(entries) for section, entries in sections.items()) - - inventory_hostname = "{{ inventory_hostname }}" - item_in_task = "{{ item }}" - ufw_in_task = "'ufw'" - docker_gpg_key_path_in_task = "{{ docker_gpg_key_path }}" - docker_gpg_key_url_in_task = "{{ docker_gpg_key_url }}" - architecture_stdout_in_task = "{{ architecture.stdout }}" - docker_apt_repo_in_task = "{{ docker_apt_repo }}" - distribution_codename_stdout_in_task = "{{ distribution_codename.stdout }}" - kubernetes_gpg_keyring_path_in_task = "{{ kubernetes_gpg_keyring_path }}" - kubernetes_gpg_key_url_in_task = "{{ kubernetes_gpg_key_url }}" - kubernetes_apt_repo_in_task = "{{ kubernetes_apt_repo }}" - private_ip_in_task = "{{ private_ip }}" - hostvars_private_ip_in_task = "{{ hostvars[item].private_ip }}" - domain_in_task = "{{ domain }}" - groups_all_in_task = "{{ groups['all'] }}" - hostvars_groups_k8s_masters_private_ip_in_task = "{{ hostvars[groups['k8s_masters'][0]].private_ip }}" - apiserver_url_in_task = "{{ apiserver_url }}" - groups_k8s_masters_in_task = "{{ groups['k8s_masters'][0] }}" - calico_operator_url_in_task = "{{ calico_operator_url }}" - calico_crd_url_in_task = "{{ calico_crd_url }}" - join_command_stdout_lines_in_task = "{{ join_command.stdout_lines[0] }}" - kubeadm_cert_key_stdout_lines_in_task = "{{ kubeadm_cert_key.stdout_lines[2] }}" - hostvars_k8s_masters_control_plane_certkey_in_task = "{{ hostvars[groups['k8s_masters'][0]].control_plane_certkey }}" - cri_socket_in_task = "{{ cri_socket }}" - - - - prompt = f""" - Generate a Python code to generate an Ansible project (project name is app/media/MyAnsible) - that dynamically provisions Ansible resources ensuring a modular, flexible structure. Only provide - Python code, no explanations or markdown formatting, without ```python entry. - The project should be organized as follows: - - The structure of this project must be as follows: - ``` - ├── ansible.cfg - ├── group_vars - │   |── all - │   - ├── hosts - ├── host_vars - ├── kubernetes_playbook.yml - └── roles - └── preinstall - ├── defaults - │   └── main.yml - ├── files - │   └── sample.sh - ├── handlers - │   └── main.yml - ├── tasks - │   └── basic.yml - │   └── main.yml - ├── templates - │   └── resolv.conf.j2 - └── vars - | └── main.yml - k8s - ├── defaults - │   └── main.yml - ├── files - │   └── sample.sh - ├── handlers - │   └── main.yml - ├── tasks - │   └── k8s.yml - │   └── main.yml - ├── templates - │   └── sample.j2 - └── vars - | └── main.yml - init_k8s - ├── defaults - │   └── main.yml - ├── files - │   └── sample.sh - ├── handlers - │   └── main.yml - ├── tasks - │   └── cni.yml - │   └── initk8s.yml - │   └── main.yml - ├── templates - │   └── kubeadmcnf.yml.j2 - └── vars - | └── main.yml - join_master - ├── defaults - │   └── main.yml - ├── files - │   └── sample.sh - ├── handlers - │   └── main.yml - ├── tasks - │   └── join_master.yml - │   └── main.yml - ├── templates - │   └── kubeadmcnf-join.yml.j2 - └── vars - | └── main.yml - join_worker - ├── defaults - │   └── main.yml - ├── files - │   └── sample.sh - ├── handlers - │   └── main.yml - ├── tasks - │   └── join_worker.yml - │   └── main.yml - ├── templates - │   └── sample.j2 - └── vars - └── main.yml - ``` - - The content of ansible.cfg must be as follows: - ``` - [defaults] - host_key_checking=false - ``` - - group_vars directory includes a single file called "all" and the content of this file must be as follows: - ``` - # General - install_ansible_modules: "true" - disable_transparent_huge_pages: "true" - - setup_interface: "false" - - # Network Calico see here for more details https://github.com/projectcalico/calico/releases - calico_operator_url: "https://raw.githubusercontent.com/projectcalico/calico/v3.29.0/manifests/tigera-operator.yaml" - calico_crd_url: "https://raw.githubusercontent.com/projectcalico/calico/v3.29.0/manifests/custom-resources.yaml" - pod_network_cidr: "192.168.0.0/16" - - # DNS - resolv_nameservers: [8.8.8.8, 4.2.2.4] # 403.online - - # Sanction shekan - use_iran: "true" # change it to "false" if you are outside of iran - - # Docker - docker_gpg_key_url: "https://download.docker.com/linux/ubuntu/gpg" - docker_gpg_key_path: "/etc/apt/keyrings/docker.gpg" - docker_apt_repo: "https://download.docker.com/linux/ubuntu" - - # Kubernetes - kubernetes_gpg_keyring_path: "/etc/apt/keyrings/kubernetes-apt-keyring.gpg" - kubernetes_gpg_key_url: "https://pkgs.k8s.io/core:/stable:/v{k8s_version}/deb/Release.key" - kubernetes_apt_repo: "https://pkgs.k8s.io/core:/stable:/v{k8s_version}/deb/" - k8s_version: "{k8s_version}.2" # see here https://kubernetes.io/releases/patch-releases/ and https://github.com/kubernetes/kubernetes/releases - - # CRI - cri_socket: unix:///var/run/containerd/containerd.sock - - # Ansible Connection - - ansible_user: {kubernetes_ansible_user} - ansible_port: {kubernetes_ansible_port} - ansible_python_interpreter: "/usr/bin/python3" - domain: "devopsgpt.com" - apiserver_url: "devopsgpt.com" - ``` - - there is file called "hosts" which its content must be as follows: - ``` - {kubernetes_inventory} - ``` - - There is an empty directory called "host_vars" with no files included - - There is a file called "kubernetes_playbook.yml" which its content must be as follows: - ``` - - hosts: all - roles: - - role: preinstall - gather_facts: yes - any_errors_fatal: true - tags: [preinstall] - - - hosts: k8s - roles: - - role: k8s - gather_facts: yes - any_errors_fatal: true - tags: [k8s] - - - hosts: k8s - roles: - - role: init_k8s - gather_facts: yes - any_errors_fatal: true - tags: [init_k8s] - - - hosts: k8s_masters - roles: - - role: preinstall - - role: k8s - - role: join_master - gather_facts: yes - any_errors_fatal: true - tags: [join_master] - - - hosts: k8s_workers - roles: - - role: preinstall - - role: k8s - - role: join_worker - gather_facts: yes - any_errors_fatal: true - tags: [join_worker] - ``` - - There is a directory called "roles" which a sub-directory called "preinstall" (roles/preinstall): - "preinstall" has multiple sub-directories, so let's dive deeper into each its sub-directories: - - (preinstall/tasks): This path has two files called "basic.yml" and "main.yml". - - 1. Create "preinstall/tasks/basic.yml" and it must be as follows:" - ``` - - name: Set timezone to UTC - timezone: - name: Etc/UTC - - - name: Set hostname - command: hostnamectl set-hostname {inventory_hostname} - - - name: Remove symlink resolve.conf - file: - path: "/etc/resolv.conf" - state: absent - ignore_errors: true - when: use_iran == "true" - - - name: Configure resolv.conf - template: - src: "resolv.conf.j2" - dest: "/etc/resolv.conf" - mode: "0644" - when: use_iran == "true" - - - name: Add hostname - lineinfile: - path: /etc/hosts - regexp: '^127\.0\.0\.1' - line: "127.0.0.1 {inventory_hostname} localhost" - owner: root - group: root - mode: 0644 - - - name: Install necessary tools - apt: - state: latest - update_cache: true - name: - - vim - - sudo - - wget - - curl - - telnet - - nload - - s3cmd - - cron - - ipset - - lvm2 - - python3 - - python3-setuptools - - python3-pip - - python3-apt - - intel-microcode - - htop - - tcpdump - - net-tools - - screen - - tmux - - byobu - - iftop - - bmon - - iperf - - sysstat - - ethtool - - plocate - - thin-provisioning-tools - - conntrack - - stress - - cpufrequtils - - rsync - - xz-utils - - build-essential - - apt-transport-https - - ca-certificates - - software-properties-common - - gnupg-agent - - iptables-persistent - - open-iscsi - - nfs-common - - tzdata - - tree - - - name: Fix broken packages - apt: - state: fixed - ``` - - 2. Create preinstall/tasks/main.yml and it must be as follows:" - ``` - --- - - name: basic setup - include_tasks: basic.yml - ``` - - There is a directory called "roles" which a sub-directory called "k8s" (roles/k8s): - "k8s" has multiple sub-directories, so let's dive deeper into each its sub-directories: - - (k8s/tasks): This path has two files called "k8s.yml" and "main.yml". - - 1. Create k8s/tasks/k8s.yml and it must be as follows:" - ``` - - name: Disable SWAP since kubernetes can't work with swap enabled - shell: | - swapoff -a - - - name: Disable SWAP in fstab since kubernetes can't work with swap enabled - replace: - path: /etc/fstab - regexp: '^([^#].*?\sswap\s+sw\s+.*)$' - replace: '# \\1' - - - name: Check if ufw is installed - package_facts: - manager: "auto" - - - name: Disable ufw # just in Ubuntu - ufw: - state: disabled - when: "{ufw_in_task} in ansible_facts.packages" - - - name: Ensure kernel modules for containerd are enabled - lineinfile: - path: /etc/modules-load.d/containerd.conf - line: "{item_in_task}" - create: yes - state: present - loop: - - overlay - - br_netfilter - - - name: Load kernel modules - command: - cmd: "modprobe {item_in_task}" - loop: - - overlay - - br_netfilter - - - name: Ensure sysctl settings for Kubernetes are present - blockinfile: - path: /etc/sysctl.d/kubernetes.conf - block: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - net.ipv4.ip_forward = 1 - create: yes - marker: "# {{mark}} ANSIBLE MANAGED BLOCK" - owner: root - group: root - mode: '0644' - - - name: Reload sysctl settings - command: - cmd: sysctl --system - - - name: Update apt cache - apt: - update_cache: yes - - - name: Install required packages - apt: - pkg: - - ca-certificates - - curl - - gnupg - - lsb-release - - gpg - - state: present - update_cache: yes - - - name: Ensure the /etc/apt/keyrings directory exists - file: - path: /etc/apt/keyrings - state: directory - mode: '0755' # Adjust the permissions as necessary - owner: root # Set the owner, if required - group: root - - - name: Remove existing Docker GPG key if it exists - file: - path: '{docker_gpg_key_path_in_task}' - state: absent - - - name: Download Docker GPG key - shell: | - curl -fsSL {docker_gpg_key_url_in_task} | gpg --dearmor -o {docker_gpg_key_path_in_task} - - - name: Determine the architecture - command: dpkg --print-architecture - register: architecture - - - name: Determine the distribution codename - command: lsb_release -cs - register: distribution_codename - - - name: Add Docker APT repository - lineinfile: - path: /etc/apt/sources.list.d/docker.list - create: yes - line: "deb [arch={architecture_stdout_in_task} signed-by={docker_gpg_key_path_in_task}] {docker_apt_repo_in_task} {distribution_codename_stdout_in_task} stable" - state: present - - - name: Update apt cache - apt: - update_cache: yes - - - name: Install required packages (containerd) - apt: - pkg: - - containerd.io - state: present - - - name: Generate default containerd configuration - shell: - cmd: containerd config default > /etc/containerd/config.toml - - - name: Replace SystemdCgroup from false to true in containerd config - replace: - path: /etc/containerd/config.toml - regexp: 'SystemdCgroup = false' - replace: 'SystemdCgroup = true' - - - name: Restart containerd service - systemd: - name: containerd - state: restarted - daemon_reload: yes - - - name: Enable containerd service - systemd: - name: containerd - enabled: yes - - name: Delete the existing Kubernetes APT keyring file if it exists - file: - path: '{kubernetes_gpg_keyring_path_in_task}' - state: absent - - - name: Download Kubernetes GPG key - shell: | - curl -fsSL '{kubernetes_gpg_key_url_in_task}' | gpg --dearmor -o '{kubernetes_gpg_keyring_path_in_task}' - - - name: Add Kubernetes repo - apt_repository: - repo: "deb [signed-by={kubernetes_gpg_keyring_path_in_task}] {kubernetes_apt_repo_in_task} /" - state: present - filename: kubernetes.list - - - name: Update apt cache - apt: - update_cache: yes - - - name: Install Kubernetes packages - apt: - name: "{item_in_task}" - state: present - loop: - - kubeadm={k8s_version}.2-1.1 - - kubelet={k8s_version}.2-1.1 - - kubectl={k8s_version}.2-1.1 - - - name: Hold Kubernetes packages - dpkg_selections: - name: "{item_in_task}" - selection: hold - loop: - - kubeadm - - kubelet - - kubectl - - containerd.io - - - name: Configure node ip - lineinfile: - path: /etc/default/kubelet - line: KUBELET_EXTRA_ARGS=--node-ip={private_ip_in_task} - create: yes - state: present - notify: Restart kubelet - - - name: Add hosts to /etc/hosts - lineinfile: - path: /etc/hosts - line: "{hostvars_private_ip_in_task} {item_in_task} {item_in_task}.{domain_in_task}" - state: present - create: no - loop: "{groups_all_in_task}" - when: hostvars[item].private_ip is defined - - - name: Add apiserver_url to point to the masters temporary" - lineinfile: - dest: /etc/hosts - line: "{hostvars_groups_k8s_masters_private_ip_in_task} {apiserver_url_in_task}" - state: present - - - name: Pull Kubernetes images | If you got error check your dns and sanction - command: - cmd: kubeadm config images pull - ``` - 2. Create k8s/tasks/main.yml and it must be as follows:" - ``` - --- - - name: Install kubernetes packages - include_tasks: k8s.yml - ``` - - (k8s/handlers): This path has a file called "main.yml". - - 3. Create k8s/handlers/main.yml and it must be as follows:" - ``` - --- - # handlers file for k8s - - - name: Remove temporary GPG key file - file: - path: "/tmp/docker.list" - state: absent - - - name: Restart kubelet - service: - name: kubelet - state: restarted - ``` - - There is a directory called "roles" which a sub-directory called "init_k8s" (roles/init_k8s): - "init_k8s" has multiple sub-directories, so let's dive deeper into each its sub-directories: - - (init_k8s/tasks): This path has three files called "cni.yml", "initk8s.yml" and "main.yml". - - 1. Create init_k8s/tasks/cni.yml and it must be as follows:" - ``` - - block: - - name: Check if Calico CRDs exist - command: kubectl get crd felixconfigurations.crd.projectcalico.org - register: calico_crd_check - ignore_errors: true - delegate_to: "{groups_k8s_masters_in_task}" - - - block: - - name: Apply CNI plugin (Calico) - command: kubectl create -f {calico_operator_url_in_task} - retries: 3 - delay: 3 - - - name: Apply CNI plugin (Calico) - command: kubectl create -f {calico_crd_url_in_task} - retries: 3 - delay: 3 - delegate_to: "{groups_k8s_masters_in_task}" - when: calico_crd_check.rc != 0 - run_once: true - ``` - 2. Create init_k8s/tasks/initk8s.yml and it must be as follows:" - ``` - - name: Init cluster | Check if kubeadm has already run - stat: - path: "/var/lib/kubelet/config.yaml" - register: kubeadm_already_run - when: inventory_hostname == groups['k8s_masters'][0] - delegate_to: "{groups_k8s_masters_in_task}" - - - block: - - name: Init cluster | Copy kubeadmcnf.yaml - template: - src: kubeadmcnf.yml.j2 - dest: /root/kubeadmcnf.yaml - - - name: Init cluster | Initiate cluster on node groups['kube_master'][0] - shell: kubeadm init --config=/root/kubeadmcnf.yaml - register: kubeadm_init - # Retry is because upload config sometimes fails - until: kubeadm_init is succeeded or "field is immutable" in kubeadm_init.stderr - notify: Restart kubelet - - when: inventory_hostname == groups['k8s_masters'][0] and not kubeadm_already_run.stat.exists - delegate_to: "{groups_k8s_masters_in_task}" - - - block: - - name: Create kubectl directory - file: - path: /root/.kube - state: directory - - - name: Configure kubectl - copy: - src: /etc/kubernetes/admin.conf - dest: /root/.kube/config - remote_src: yes - - - name: Fetch kubeconfig - fetch: - src: /etc/kubernetes/admin.conf - dest: kubeconfig/ - flat: yes - when: inventory_hostname == groups['k8s_masters'][0] - delegate_to: "{groups_k8s_masters_in_task}" - - - name: Sleep for 300 seconds and reboot the Master1 server - wait_for: - timeout: 300 - delegate_to: localhost - - - name: Reboot the servers - command: reboot - async: 1 - poll: 0 - # ignore_errors: yes - delegate_to: "{groups_k8s_masters_in_task}" - - - name: Sleep for 300 seconds to Master1 up and running - wait_for: - timeout: 300 - delegate_to: localhost - # when: use_iran == "true" - - - name: Example Task After Reboot - debug: - msg: "Server back online and ready for tasks." - ``` - 3. Create init_k8s/tasks/main.yml and it must be as follows:" - ``` - --- - # tasks file for init_k8s - - - name: Initialize kubernetes cluster - include_tasks: initk8s.yml - - - name: Initialize Calico CNI - include_tasks: cni.yml - ``` - - There is a directory called "roles" which a sub-directory called "join_master" (roles/join_master): - "join_master" has multiple sub-directories, so let's dive deeper into each its sub-directories: - - (join_master/tasks): This path has two files called "join_master.yml" and "main.yml". - - 1. Create "join_master/tasks/join_master.yml" and it must be as follows:" - ``` - - name: Init cluster | Check if kubeadm has already run - stat: - path: "/var/lib/kubelet/config.yaml" - register: kubeadm_already_run - - - block: - - name: Generate join command - command: kubeadm token create --print-join-command - register: join_command - - - name: Print join command - debug: - msg: "{join_command_stdout_lines_in_task}" - - - name: Copy join command to local file - become: false - local_action: copy content="{join_command_stdout_lines_in_task} $@" dest="roles/join_master/files/join-command" - - - name: copy kubeadmcnf.yaml - template: - src: kubeadmcnf-join.yml.j2 - dest: /root/kubeadm-config.yaml - - when: - - inventory_hostname == groups['k8s_masters'][0] - delegate_to: "{groups_k8s_masters_in_task}" - - - block: - - name: Copy the join command to server location - copy: - src: roles/join_master/files/join-command - dest: /root/join-command.sh - mode: "0777" - - when: - - inventory_hostname != groups['k8s_masters'][0] - - inventory_hostname in groups['k8s_masters'] - - not kubeadm_already_run.stat.exists - - - block: - - name: get certificate key - shell: kubeadm init phase upload-certs --upload-certs --config=/root/kubeadm-config.yaml - register: kubeadm_cert_key - - - name: Print certificate key - debug: - msg: "{kubeadm_cert_key_stdout_lines_in_task}" - - - name: register the cert key - set_fact: - control_plane_certkey: "{kubeadm_cert_key_stdout_lines_in_task}" - - when: - - inventory_hostname in groups['k8s_masters'][0] - delegate_to: "{groups_k8s_masters_in_task}" - run_once: false - delegate_facts: true - - - name: Join | Join control-plane to cluster - command: "sh /root/join-command.sh --control-plane --certificate-key={hostvars_k8s_masters_control_plane_certkey_in_task} --cri-socket={cri_socket_in_task}" - when: - - inventory_hostname != groups['k8s_masters'][0] - - inventory_hostname in groups['k8s_masters'] - - not kubeadm_already_run.stat.exists - - - block: - - name: Create kubectl directory - file: - path: /root/.kube - state: directory - - - name: Configure kubectl - copy: - src: /etc/kubernetes/admin.conf - dest: /root/.kube/config - remote_src: yes - - - name: Fetch kubeconfig - fetch: - src: /etc/kubernetes/admin.conf - dest: kubeconfig/ - flat: yes - when: - - inventory_hostname != groups['k8s_masters'][0] - - inventory_hostname in groups['k8s_masters'] - - not kubeadm_already_run.stat.exists - - - name: remove apiserver_url to point to the masters temporary - lineinfile: - dest: /etc/hosts - line: "{hostvars_groups_k8s_masters_private_ip_in_task} {apiserver_url_in_task}" - state: absent - - - - name: Add apiserver_url to point to the masters" - lineinfile: - dest: /etc/hosts - line: "{private_ip_in_task} {apiserver_url_in_task}" - state: present - when: - - inventory_hostname in groups['k8s_masters'] - ``` - 2. Create join_master/tasks/main.yml and it must be as follows:" - ``` - --- - # tasks file for join_master - - - name: Join master(s) node to cluster - include_tasks: join_master.yml - - ``` - - There is a directory called "roles" which a sub-directory called "join_worker" (roles/join_worker): - "join_worker" has multiple sub-directories, so let's dive deeper into each its sub-directories: - - (join_worker/tasks): This path has two files called "join_worker.yml" and "main.yml". - - 1. Create "join_worker/tasks/join_worker.yml" and it must be as follows:" - ``` - - name: Init cluster | Check if kubeadm has already run - stat: - path: "/var/lib/kubelet/config.yaml" - register: kubeadm_already_run - - - block: - - name: Generate join command - command: kubeadm token create --print-join-command - register: join_command - - - name: Print join command - debug: - msg: "{join_command_stdout_lines_in_task}" - - - name: Copy join command to local file - become: false - local_action: copy content="{join_command_stdout_lines_in_task} $@" dest="roles/join_worker/files/join-command" - - when: - - inventory_hostname not in groups['k8s_masters'][0] - delegate_to: "{groups_k8s_masters_in_task}" - - - block: - - name: Copy the join command to server location - copy: - src: roles/join_worker/files/join-command - dest: /root/join-command.sh - mode: "0777" - - when: - - inventory_hostname not in groups['k8s_masters'] - - not kubeadm_already_run.stat.exists - - - name: Join | Join worker nodes to the cluster - command: sh /root/join-command.sh - when: - - inventory_hostname not in groups['k8s_masters'] - - not kubeadm_already_run.stat.exists - ``` - 2. Create join_worker/tasks/main.yml and it must be as follows:" - ``` - --- - # tasks file for join_worker - - - name: Join worker(s) node to cluster - include_tasks: join_worker.yml - ``` - finally just give me a python code without any note that can generate a project folder with the - given schema without ```python entry. and we dont need any base directory in the python code. - the final ansible template must work very well without any error! - - the python code you give me, must have structure like that: - - import os - project_name = "app/media/MyAnsible" - foo_dir = os.path.join(project_name, "bar") - x_dir = os.path.join(modules_dir, "y") - - # Create project directories - os.makedirs(ansible_dir, exist_ok=True) - - # Create main.tf - with open(os.path.join(project_name, "main.tf"), "w") as main_file: - # any thing you need - """ - return prompt -cat: 7: No such file or directory -cat ../../template_generators/ansible/install/kuber.py -def ansible_kuber_install(input): - - - kubernetes_ansible_port = input.ansible_port - kubernetes_ansible_user = input.ansible_user - k8s_master_nodes = input.k8s_master_nodes - k8s_worker_nodes = input.k8s_worker_nodes - k8s_version = input.version - sections = { - "[all]": [f"{name} private_ip=x.x.x.x" for name in k8s_master_nodes + k8s_worker_nodes], - "[k8s]": k8s_master_nodes + k8s_worker_nodes, - "[k8s_masters]": k8s_master_nodes, - "[k8s_workers]": k8s_worker_nodes, - } - kubernetes_inventory = "\n\n".join(f"{section}\n" + "\n".join(entries) for section, entries in sections.items()) - - inventory_hostname = "{{ inventory_hostname }}" - item_in_task = "{{ item }}" - ufw_in_task = "'ufw'" - docker_gpg_key_path_in_task = "{{ docker_gpg_key_path }}" - docker_gpg_key_url_in_task = "{{ docker_gpg_key_url }}" - architecture_stdout_in_task = "{{ architecture.stdout }}" - docker_apt_repo_in_task = "{{ docker_apt_repo }}" - distribution_codename_stdout_in_task = "{{ distribution_codename.stdout }}" - kubernetes_gpg_keyring_path_in_task = "{{ kubernetes_gpg_keyring_path }}" - kubernetes_gpg_key_url_in_task = "{{ kubernetes_gpg_key_url }}" - kubernetes_apt_repo_in_task = "{{ kubernetes_apt_repo }}" - private_ip_in_task = "{{ private_ip }}" - hostvars_private_ip_in_task = "{{ hostvars[item].private_ip }}" - domain_in_task = "{{ domain }}" - groups_all_in_task = "{{ groups['all'] }}" - hostvars_groups_k8s_masters_private_ip_in_task = "{{ hostvars[groups['k8s_masters'][0]].private_ip }}" - apiserver_url_in_task = "{{ apiserver_url }}" - groups_k8s_masters_in_task = "{{ groups['k8s_masters'][0] }}" - calico_operator_url_in_task = "{{ calico_operator_url }}" - calico_crd_url_in_task = "{{ calico_crd_url }}" - join_command_stdout_lines_in_task = "{{ join_command.stdout_lines[0] }}" - kubeadm_cert_key_stdout_lines_in_task = "{{ kubeadm_cert_key.stdout_lines[2] }}" - hostvars_k8s_masters_control_plane_certkey_in_task = "{{ hostvars[groups['k8s_masters'][0]].control_plane_certkey }}" - cri_socket_in_task = "{{ cri_socket }}" - - - - prompt = f""" - Generate a Python code to generate an Ansible project (project name is app/media/MyAnsible) - that dynamically provisions Ansible resources ensuring a modular, flexible structure. Only provide - Python code, no explanations or markdown formatting, without ```python entry. - The project should be organized as follows: - - The structure of this project must be as follows: - ``` - ├── ansible.cfg - ├── group_vars - │   |── all - │   - ├── hosts - ├── host_vars - ├── kubernetes_playbook.yml - └── roles - └── preinstall - ├── defaults - │   └── main.yml - ├── files - │   └── sample.sh - ├── handlers - │   └── main.yml - ├── tasks - │   └── basic.yml - │   └── main.yml - ├── templates - │   └── resolv.conf.j2 - └── vars - | └── main.yml - k8s - ├── defaults - │   └── main.yml - ├── files - │   └── sample.sh - ├── handlers - │   └── main.yml - ├── tasks - │   └── k8s.yml - │   └── main.yml - ├── templates - │   └── sample.j2 - └── vars - | └── main.yml - init_k8s - ├── defaults - │   └── main.yml - ├── files - │   └── sample.sh - ├── handlers - │   └── main.yml - ├── tasks - │   └── cni.yml - │   └── initk8s.yml - │   └── main.yml - ├── templates - │   └── kubeadmcnf.yml.j2 - └── vars - | └── main.yml - join_master - ├── defaults - │   └── main.yml - ├── files - │   └── sample.sh - ├── handlers - │   └── main.yml - ├── tasks - │   └── join_master.yml - │   └── main.yml - ├── templates - │   └── kubeadmcnf-join.yml.j2 - └── vars - | └── main.yml - join_worker - ├── defaults - │   └── main.yml - ├── files - │   └── sample.sh - ├── handlers - │   └── main.yml - ├── tasks - │   └── join_worker.yml - │   └── main.yml - ├── templates - │   └── sample.j2 - └── vars - └── main.yml - ``` - - The content of ansible.cfg must be as follows: - ``` - [defaults] - host_key_checking=false - ``` - - group_vars directory includes a single file called "all" and the content of this file must be as follows: - ``` - # General - install_ansible_modules: "true" - disable_transparent_huge_pages: "true" - - setup_interface: "false" - - # Network Calico see here for more details https://github.com/projectcalico/calico/releases - calico_operator_url: "https://raw.githubusercontent.com/projectcalico/calico/v3.29.0/manifests/tigera-operator.yaml" - calico_crd_url: "https://raw.githubusercontent.com/projectcalico/calico/v3.29.0/manifests/custom-resources.yaml" - pod_network_cidr: "192.168.0.0/16" - - # DNS - resolv_nameservers: [8.8.8.8, 4.2.2.4] # 403.online - - # Sanction shekan - use_iran: "true" # change it to "false" if you are outside of iran - - # Docker - docker_gpg_key_url: "https://download.docker.com/linux/ubuntu/gpg" - docker_gpg_key_path: "/etc/apt/keyrings/docker.gpg" - docker_apt_repo: "https://download.docker.com/linux/ubuntu" - - # Kubernetes - kubernetes_gpg_keyring_path: "/etc/apt/keyrings/kubernetes-apt-keyring.gpg" - kubernetes_gpg_key_url: "https://pkgs.k8s.io/core:/stable:/v{k8s_version}/deb/Release.key" - kubernetes_apt_repo: "https://pkgs.k8s.io/core:/stable:/v{k8s_version}/deb/" - k8s_version: "{k8s_version}.2" # see here https://kubernetes.io/releases/patch-releases/ and https://github.com/kubernetes/kubernetes/releases - - # CRI - cri_socket: unix:///var/run/containerd/containerd.sock - - # Ansible Connection - - ansible_user: {kubernetes_ansible_user} - ansible_port: {kubernetes_ansible_port} - ansible_python_interpreter: "/usr/bin/python3" - domain: "devopsgpt.com" - apiserver_url: "devopsgpt.com" - ``` - - there is file called "hosts" which its content must be as follows: - ``` - {kubernetes_inventory} - ``` - - There is an empty directory called "host_vars" with no files included - - There is a file called "kubernetes_playbook.yml" which its content must be as follows: - ``` - - hosts: all - roles: - - role: preinstall - gather_facts: yes - any_errors_fatal: true - tags: [preinstall] - - - hosts: k8s - roles: - - role: k8s - gather_facts: yes - any_errors_fatal: true - tags: [k8s] - - - hosts: k8s - roles: - - role: init_k8s - gather_facts: yes - any_errors_fatal: true - tags: [init_k8s] - - - hosts: k8s_masters - roles: - - role: preinstall - - role: k8s - - role: join_master - gather_facts: yes - any_errors_fatal: true - tags: [join_master] - - - hosts: k8s_workers - roles: - - role: preinstall - - role: k8s - - role: join_worker - gather_facts: yes - any_errors_fatal: true - tags: [join_worker] - ``` - - There is a directory called "roles" which a sub-directory called "preinstall" (roles/preinstall): - "preinstall" has multiple sub-directories, so let's dive deeper into each its sub-directories: - - (preinstall/tasks): This path has two files called "basic.yml" and "main.yml". - - 1. Create "preinstall/tasks/basic.yml" and it must be as follows:" - ``` - - name: Set timezone to UTC - timezone: - name: Etc/UTC - - - name: Set hostname - command: hostnamectl set-hostname {inventory_hostname} - - - name: Remove symlink resolve.conf - file: - path: "/etc/resolv.conf" - state: absent - ignore_errors: true - when: use_iran == "true" - - - name: Configure resolv.conf - template: - src: "resolv.conf.j2" - dest: "/etc/resolv.conf" - mode: "0644" - when: use_iran == "true" - - - name: Add hostname - lineinfile: - path: /etc/hosts - regexp: '^127\.0\.0\.1' - line: "127.0.0.1 {inventory_hostname} localhost" - owner: root - group: root - mode: 0644 - - - name: Install necessary tools - apt: - state: latest - update_cache: true - name: - - vim - - sudo - - wget - - curl - - telnet - - nload - - s3cmd - - cron - - ipset - - lvm2 - - python3 - - python3-setuptools - - python3-pip - - python3-apt - - intel-microcode - - htop - - tcpdump - - net-tools - - screen - - tmux - - byobu - - iftop - - bmon - - iperf - - sysstat - - ethtool - - plocate - - thin-provisioning-tools - - conntrack - - stress - - cpufrequtils - - rsync - - xz-utils - - build-essential - - apt-transport-https - - ca-certificates - - software-properties-common - - gnupg-agent - - iptables-persistent - - open-iscsi - - nfs-common - - tzdata - - tree - - - name: Fix broken packages - apt: - state: fixed - ``` - - 2. Create preinstall/tasks/main.yml and it must be as follows:" - ``` - --- - - name: basic setup - include_tasks: basic.yml - ``` - - There is a directory called "roles" which a sub-directory called "k8s" (roles/k8s): - "k8s" has multiple sub-directories, so let's dive deeper into each its sub-directories: - - (k8s/tasks): This path has two files called "k8s.yml" and "main.yml". - - 1. Create k8s/tasks/k8s.yml and it must be as follows:" - ``` - - name: Disable SWAP since kubernetes can't work with swap enabled - shell: | - swapoff -a - - - name: Disable SWAP in fstab since kubernetes can't work with swap enabled - replace: - path: /etc/fstab - regexp: '^([^#].*?\sswap\s+sw\s+.*)$' - replace: '# \\1' - - - name: Check if ufw is installed - package_facts: - manager: "auto" - - - name: Disable ufw # just in Ubuntu - ufw: - state: disabled - when: "{ufw_in_task} in ansible_facts.packages" - - - name: Ensure kernel modules for containerd are enabled - lineinfile: - path: /etc/modules-load.d/containerd.conf - line: "{item_in_task}" - create: yes - state: present - loop: - - overlay - - br_netfilter - - - name: Load kernel modules - command: - cmd: "modprobe {item_in_task}" - loop: - - overlay - - br_netfilter - - - name: Ensure sysctl settings for Kubernetes are present - blockinfile: - path: /etc/sysctl.d/kubernetes.conf - block: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - net.ipv4.ip_forward = 1 - create: yes - marker: "# {{mark}} ANSIBLE MANAGED BLOCK" - owner: root - group: root - mode: '0644' - - - name: Reload sysctl settings - command: - cmd: sysctl --system - - - name: Update apt cache - apt: - update_cache: yes - - - name: Install required packages - apt: - pkg: - - ca-certificates - - curl - - gnupg - - lsb-release - - gpg - - state: present - update_cache: yes - - - name: Ensure the /etc/apt/keyrings directory exists - file: - path: /etc/apt/keyrings - state: directory - mode: '0755' # Adjust the permissions as necessary - owner: root # Set the owner, if required - group: root - - - name: Remove existing Docker GPG key if it exists - file: - path: '{docker_gpg_key_path_in_task}' - state: absent - - - name: Download Docker GPG key - shell: | - curl -fsSL {docker_gpg_key_url_in_task} | gpg --dearmor -o {docker_gpg_key_path_in_task} - - - name: Determine the architecture - command: dpkg --print-architecture - register: architecture - - - name: Determine the distribution codename - command: lsb_release -cs - register: distribution_codename - - - name: Add Docker APT repository - lineinfile: - path: /etc/apt/sources.list.d/docker.list - create: yes - line: "deb [arch={architecture_stdout_in_task} signed-by={docker_gpg_key_path_in_task}] {docker_apt_repo_in_task} {distribution_codename_stdout_in_task} stable" - state: present - - - name: Update apt cache - apt: - update_cache: yes - - - name: Install required packages (containerd) - apt: - pkg: - - containerd.io - state: present - - - name: Generate default containerd configuration - shell: - cmd: containerd config default > /etc/containerd/config.toml - - - name: Replace SystemdCgroup from false to true in containerd config - replace: - path: /etc/containerd/config.toml - regexp: 'SystemdCgroup = false' - replace: 'SystemdCgroup = true' - - - name: Restart containerd service - systemd: - name: containerd - state: restarted - daemon_reload: yes - - - name: Enable containerd service - systemd: - name: containerd - enabled: yes - - name: Delete the existing Kubernetes APT keyring file if it exists - file: - path: '{kubernetes_gpg_keyring_path_in_task}' - state: absent - - - name: Download Kubernetes GPG key - shell: | - curl -fsSL '{kubernetes_gpg_key_url_in_task}' | gpg --dearmor -o '{kubernetes_gpg_keyring_path_in_task}' - - - name: Add Kubernetes repo - apt_repository: - repo: "deb [signed-by={kubernetes_gpg_keyring_path_in_task}] {kubernetes_apt_repo_in_task} /" - state: present - filename: kubernetes.list - - - name: Update apt cache - apt: - update_cache: yes - - - name: Install Kubernetes packages - apt: - name: "{item_in_task}" - state: present - loop: - - kubeadm={k8s_version}.2-1.1 - - kubelet={k8s_version}.2-1.1 - - kubectl={k8s_version}.2-1.1 - - - name: Hold Kubernetes packages - dpkg_selections: - name: "{item_in_task}" - selection: hold - loop: - - kubeadm - - kubelet - - kubectl - - containerd.io - - - name: Configure node ip - lineinfile: - path: /etc/default/kubelet - line: KUBELET_EXTRA_ARGS=--node-ip={private_ip_in_task} - create: yes - state: present - notify: Restart kubelet - - - name: Add hosts to /etc/hosts - lineinfile: - path: /etc/hosts - line: "{hostvars_private_ip_in_task} {item_in_task} {item_in_task}.{domain_in_task}" - state: present - create: no - loop: "{groups_all_in_task}" - when: hostvars[item].private_ip is defined - - - name: Add apiserver_url to point to the masters temporary" - lineinfile: - dest: /etc/hosts - line: "{hostvars_groups_k8s_masters_private_ip_in_task} {apiserver_url_in_task}" - state: present - - - name: Pull Kubernetes images | If you got error check your dns and sanction - command: - cmd: kubeadm config images pull - ``` - 2. Create k8s/tasks/main.yml and it must be as follows:" - ``` - --- - - name: Install kubernetes packages - include_tasks: k8s.yml - ``` - - (k8s/handlers): This path has a file called "main.yml". - - 3. Create k8s/handlers/main.yml and it must be as follows:" - ``` - --- - # handlers file for k8s - - - name: Remove temporary GPG key file - file: - path: "/tmp/docker.list" - state: absent - - - name: Restart kubelet - service: - name: kubelet - state: restarted - ``` - - There is a directory called "roles" which a sub-directory called "init_k8s" (roles/init_k8s): - "init_k8s" has multiple sub-directories, so let's dive deeper into each its sub-directories: - - (init_k8s/tasks): This path has three files called "cni.yml", "initk8s.yml" and "main.yml". - - 1. Create init_k8s/tasks/cni.yml and it must be as follows:" - ``` - - block: - - name: Check if Calico CRDs exist - command: kubectl get crd felixconfigurations.crd.projectcalico.org - register: calico_crd_check - ignore_errors: true - delegate_to: "{groups_k8s_masters_in_task}" - - - block: - - name: Apply CNI plugin (Calico) - command: kubectl create -f {calico_operator_url_in_task} - retries: 3 - delay: 3 - - - name: Apply CNI plugin (Calico) - command: kubectl create -f {calico_crd_url_in_task} - retries: 3 - delay: 3 - delegate_to: "{groups_k8s_masters_in_task}" - when: calico_crd_check.rc != 0 - run_once: true - ``` - 2. Create init_k8s/tasks/initk8s.yml and it must be as follows:" - ``` - - name: Init cluster | Check if kubeadm has already run - stat: - path: "/var/lib/kubelet/config.yaml" - register: kubeadm_already_run - when: inventory_hostname == groups['k8s_masters'][0] - delegate_to: "{groups_k8s_masters_in_task}" - - - block: - - name: Init cluster | Copy kubeadmcnf.yaml - template: - src: kubeadmcnf.yml.j2 - dest: /root/kubeadmcnf.yaml - - - name: Init cluster | Initiate cluster on node groups['kube_master'][0] - shell: kubeadm init --config=/root/kubeadmcnf.yaml - register: kubeadm_init - # Retry is because upload config sometimes fails - until: kubeadm_init is succeeded or "field is immutable" in kubeadm_init.stderr - notify: Restart kubelet - - when: inventory_hostname == groups['k8s_masters'][0] and not kubeadm_already_run.stat.exists - delegate_to: "{groups_k8s_masters_in_task}" - - - block: - - name: Create kubectl directory - file: - path: /root/.kube - state: directory - - - name: Configure kubectl - copy: - src: /etc/kubernetes/admin.conf - dest: /root/.kube/config - remote_src: yes - - - name: Fetch kubeconfig - fetch: - src: /etc/kubernetes/admin.conf - dest: kubeconfig/ - flat: yes - when: inventory_hostname == groups['k8s_masters'][0] - delegate_to: "{groups_k8s_masters_in_task}" - - - name: Sleep for 300 seconds and reboot the Master1 server - wait_for: - timeout: 300 - delegate_to: localhost - - - name: Reboot the servers - command: reboot - async: 1 - poll: 0 - # ignore_errors: yes - delegate_to: "{groups_k8s_masters_in_task}" - - - name: Sleep for 300 seconds to Master1 up and running - wait_for: - timeout: 300 - delegate_to: localhost - # when: use_iran == "true" - - - name: Example Task After Reboot - debug: - msg: "Server back online and ready for tasks." - ``` - 3. Create init_k8s/tasks/main.yml and it must be as follows:" - ``` - --- - # tasks file for init_k8s - - - name: Initialize kubernetes cluster - include_tasks: initk8s.yml - - - name: Initialize Calico CNI - include_tasks: cni.yml - ``` - - There is a directory called "roles" which a sub-directory called "join_master" (roles/join_master): - "join_master" has multiple sub-directories, so let's dive deeper into each its sub-directories: - - (join_master/tasks): This path has two files called "join_master.yml" and "main.yml". - - 1. Create "join_master/tasks/join_master.yml" and it must be as follows:" - ``` - - name: Init cluster | Check if kubeadm has already run - stat: - path: "/var/lib/kubelet/config.yaml" - register: kubeadm_already_run - - - block: - - name: Generate join command - command: kubeadm token create --print-join-command - register: join_command - - - name: Print join command - debug: - msg: "{join_command_stdout_lines_in_task}" - - - name: Copy join command to local file - become: false - local_action: copy content="{join_command_stdout_lines_in_task} $@" dest="roles/join_master/files/join-command" - - - name: copy kubeadmcnf.yaml - template: - src: kubeadmcnf-join.yml.j2 - dest: /root/kubeadm-config.yaml - - when: - - inventory_hostname == groups['k8s_masters'][0] - delegate_to: "{groups_k8s_masters_in_task}" - - - block: - - name: Copy the join command to server location - copy: - src: roles/join_master/files/join-command - dest: /root/join-command.sh - mode: "0777" - - when: - - inventory_hostname != groups['k8s_masters'][0] - - inventory_hostname in groups['k8s_masters'] - - not kubeadm_already_run.stat.exists - - - block: - - name: get certificate key - shell: kubeadm init phase upload-certs --upload-certs --config=/root/kubeadm-config.yaml - register: kubeadm_cert_key - - - name: Print certificate key - debug: - msg: "{kubeadm_cert_key_stdout_lines_in_task}" - - - name: register the cert key - set_fact: - control_plane_certkey: "{kubeadm_cert_key_stdout_lines_in_task}" - - when: - - inventory_hostname in groups['k8s_masters'][0] - delegate_to: "{groups_k8s_masters_in_task}" - run_once: false - delegate_facts: true - - - name: Join | Join control-plane to cluster - command: "sh /root/join-command.sh --control-plane --certificate-key={hostvars_k8s_masters_control_plane_certkey_in_task} --cri-socket={cri_socket_in_task}" - when: - - inventory_hostname != groups['k8s_masters'][0] - - inventory_hostname in groups['k8s_masters'] - - not kubeadm_already_run.stat.exists - - - block: - - name: Create kubectl directory - file: - path: /root/.kube - state: directory - - - name: Configure kubectl - copy: - src: /etc/kubernetes/admin.conf - dest: /root/.kube/config - remote_src: yes - - - name: Fetch kubeconfig - fetch: - src: /etc/kubernetes/admin.conf - dest: kubeconfig/ - flat: yes - when: - - inventory_hostname != groups['k8s_masters'][0] - - inventory_hostname in groups['k8s_masters'] - - not kubeadm_already_run.stat.exists - - - name: remove apiserver_url to point to the masters temporary - lineinfile: - dest: /etc/hosts - line: "{hostvars_groups_k8s_masters_private_ip_in_task} {apiserver_url_in_task}" - state: absent - - - - name: Add apiserver_url to point to the masters" - lineinfile: - dest: /etc/hosts - line: "{private_ip_in_task} {apiserver_url_in_task}" - state: present - when: - - inventory_hostname in groups['k8s_masters'] - ``` - 2. Create join_master/tasks/main.yml and it must be as follows:" - ``` - --- - # tasks file for join_master - - - name: Join master(s) node to cluster - include_tasks: join_master.yml - - ``` - - There is a directory called "roles" which a sub-directory called "join_worker" (roles/join_worker): - "join_worker" has multiple sub-directories, so let's dive deeper into each its sub-directories: - - (join_worker/tasks): This path has two files called "join_worker.yml" and "main.yml". - - 1. Create "join_worker/tasks/join_worker.yml" and it must be as follows:" - ``` - - name: Init cluster | Check if kubeadm has already run - stat: - path: "/var/lib/kubelet/config.yaml" - register: kubeadm_already_run - - - block: - - name: Generate join command - command: kubeadm token create --print-join-command - register: join_command - - - name: Print join command - debug: - msg: "{join_command_stdout_lines_in_task}" - - - name: Copy join command to local file - become: false - local_action: copy content="{join_command_stdout_lines_in_task} $@" dest="roles/join_worker/files/join-command" - - when: - - inventory_hostname not in groups['k8s_masters'][0] - delegate_to: "{groups_k8s_masters_in_task}" - - - block: - - name: Copy the join command to server location - copy: - src: roles/join_worker/files/join-command - dest: /root/join-command.sh - mode: "0777" - - when: - - inventory_hostname not in groups['k8s_masters'] - - not kubeadm_already_run.stat.exists - - - name: Join | Join worker nodes to the cluster - command: sh /root/join-command.sh - when: - - inventory_hostname not in groups['k8s_masters'] - - not kubeadm_already_run.stat.exists - ``` - 2. Create join_worker/tasks/main.yml and it must be as follows:" - ``` - --- - # tasks file for join_worker - - - name: Join worker(s) node to cluster - include_tasks: join_worker.yml - ``` - finally just give me a python code without any note that can generate a project folder with the - given schema without ```python entry. and we dont need any base directory in the python code. - the final ansible template must work very well without any error! - - the python code you give me, must have structure like that: - - import os - project_name = "app/media/MyAnsible" - foo_dir = os.path.join(project_name, "bar") - x_dir = os.path.join(modules_dir, "y") - - # Create project directories - os.makedirs(ansible_dir, exist_ok=True) - - # Create main.tf - with open(os.path.join(project_name, "main.tf"), "w") as main_file: - # any thing you need - """ - return prompt From 67e63bb3f1848072529fe0315cabb04bd458529e Mon Sep 17 00:00:00 2001 From: abolfazl1381 Date: Wed, 4 Dec 2024 22:10:17 +0330 Subject: [PATCH 05/22] fix(compose): totally restructre docker compose generator --- app/directory_generators/compose_generator.py | 39 -------- app/media/MyCompose/docker-compose.yaml | 91 ++++++++++++++----- app/models/ansible_models.py | 2 +- app/models/compose_models.py | 78 +++++----------- app/routes/docker.py | 5 +- app/template_generators/docker/compose.py | 64 ++----------- 6 files changed, 102 insertions(+), 177 deletions(-) diff --git a/app/directory_generators/compose_generator.py b/app/directory_generators/compose_generator.py index 0627c345..e69de29b 100644 --- a/app/directory_generators/compose_generator.py +++ b/app/directory_generators/compose_generator.py @@ -1,39 +0,0 @@ -import os - -project_name = "app/media/MyCompose" -compose_file_path = os.path.join(project_name, "docker-compose.yaml") - -# Create project directories -os.makedirs(project_name, exist_ok=True) - -# Create docker-compose.yaml -with open(compose_file_path, "w") as compose_file: - compose_file.write("version: '3'\n") - compose_file.write("services:\n") - compose_file.write(" web_server:\n") - compose_file.write(" image: nginx:latest\n") - compose_file.write(" volumes:\n") - compose_file.write(" - ./nginx/nginx.conf:/etc/nginx/nginx.conf\n") - compose_file.write(" depends_on:\n") - compose_file.write(" - string\n") - compose_file.write(" ports:\n") - compose_file.write(" - '80:80'\n") - compose_file.write(" environment:\n") - compose_file.write(" - foo=bar\n") - compose_file.write(" networks:\n") - compose_file.write(" - app_network\n") - compose_file.write(" monitoring_server:\n") - compose_file.write(" image: grafana:latest\n") - compose_file.write(" volumes:\n") - compose_file.write(" - ./nginx/nginx.conf:/etc/nginx/nginx.conf\n") - compose_file.write(" depends_on:\n") - compose_file.write(" - string\n") - compose_file.write(" ports:\n") - compose_file.write(" - '82:80'\n") - compose_file.write(" environment:\n") - compose_file.write(" - foo=bar\n") - compose_file.write(" networks:\n") - compose_file.write(" - app_network\n") - compose_file.write("networks:\n") - compose_file.write(" app_network:\n") - compose_file.write(" driver: bridge\n") \ No newline at end of file diff --git a/app/media/MyCompose/docker-compose.yaml b/app/media/MyCompose/docker-compose.yaml index 1bbac8cc..f8882cd1 100644 --- a/app/media/MyCompose/docker-compose.yaml +++ b/app/media/MyCompose/docker-compose.yaml @@ -1,31 +1,78 @@ -version: '3' +networks: + additionalProp1: + driver: string + additionalProp2: + driver: string + additionalProp3: + driver: string services: - web_server: - image: nginx:latest - volumes: - - ./nginx/nginx.conf:/etc/nginx/nginx.conf + additionalProp1: + args: + additionalProp1: string + additionalProp2: string + additionalProp3: string + build: + context: string + dockerfile: string + command: string + container_name: string depends_on: - - string - ports: - - '80:80' + - string environment: - - foo=bar + additionalProp1: string + additionalProp2: string + additionalProp3: string + image: string networks: - - app_network - - monitoring_server: - image: grafana:latest + - string + ports: + - string volumes: - - ./nginx/nginx.conf:/etc/nginx/nginx.conf + - string + additionalProp2: + args: + additionalProp1: string + additionalProp2: string + additionalProp3: string + build: + context: string + dockerfile: string + command: string + container_name: string depends_on: - - string + - string + environment: + additionalProp1: string + additionalProp2: string + additionalProp3: string + image: string + networks: + - string ports: - - '82:80' + - string + volumes: + - string + additionalProp3: + args: + additionalProp1: string + additionalProp2: string + additionalProp3: string + build: + context: string + dockerfile: string + command: string + container_name: string + depends_on: + - string environment: - - foo=bar + additionalProp1: string + additionalProp2: string + additionalProp3: string + image: string networks: - - app_network - -networks: - app_network: - driver: bridge + - string + ports: + - string + volumes: + - string +version: string diff --git a/app/models/ansible_models.py b/app/models/ansible_models.py index 7d80f39c..f97c4c09 100644 --- a/app/models/ansible_models.py +++ b/app/models/ansible_models.py @@ -32,7 +32,7 @@ def validator_os(cls, value): return value - + class AnsibleInstallKuber(AnsibleBase): os: str = 'ubuntu' k8s_worker_nodes: List[str] diff --git a/app/models/compose_models.py b/app/models/compose_models.py index daa81ba8..137dc1f1 100644 --- a/app/models/compose_models.py +++ b/app/models/compose_models.py @@ -1,59 +1,27 @@ -from typing import List, Optional -from pydantic import BaseModel, validator, ValidationError, computed_field - -class Port(BaseModel): - machine_port:int = 80 - container_port:int = 80 - -class Network(BaseModel): - name:str = 'app_network' - -class EnvironmentVariable(BaseModel): - name:str = 'foo' - value:str = "bar" - - @computed_field - @property - def env_full(self) -> int: - return f"{self.name}:{self.value}" - -class Volume(BaseModel): - local_dir: str = './nginx/nginx.conf' - container_dir:str = '/etc/nginx/nginx.conf' - - @computed_field - @property - def volume(self) -> int: - return f"{self.local_dir}:{self.container_dir}" +from typing import Dict, List, Optional +from pydantic import BaseModel class Build(BaseModel): - context:str - dockerfile:str + context: str + dockerfile: str + class Service(BaseModel): - image:str = 'nginx' - name:str = 'web_server' - container_name:str = 'web_server' - build: Build | None = None - version:str = 'latest' - volumes:List[Volume] | None = None - depends_on:List[str] | None = None - ports:List[Port] - networks:List[Network] | None = None - environments:List[EnvironmentVariable] | None = None - - @computed_field - @property - def image_full(self) -> int: - return f"{self.image}:{self.version}" - - @computed_field - @property - def volumes_full(self) -> int: - return [i.volume for i in self.volumes] - - - + build: Optional[Build] = None + image: Optional[str] = None + container_name: Optional[str] = None + command: Optional[str] = None + volumes: Optional[List[str]] = None + environment: Optional[Dict[str, str]] = None + ports: Optional[List[str]] = None + networks: Optional[List[str]] = None + args: Optional[Dict[str, str]] = None + depends_on: Optional[List[str]] = None + +class Network(BaseModel): + driver: str + class DockerCompose(BaseModel): - services: List[Service] - network:Network - \ No newline at end of file + version: str + services: Dict[str, Service] + networks: Optional[Dict[str, Network]] = None + diff --git a/app/routes/docker.py b/app/routes/docker.py index 1c8b4e65..c62a487c 100644 --- a/app/routes/docker.py +++ b/app/routes/docker.py @@ -10,10 +10,7 @@ async def docker_compose_template(request:DockerCompose) -> Output: if os.environ.get("TEST"): return Output(output='output') - generated_prompt = docker_compose_generator(request) + docker_compose_generator(request) - output = gpt_service(generated_prompt) - edit_directory_generator("compose_generator",output) - execute_pythonfile("MyCompose","compose_generator") return Output(output='output') \ No newline at end of file diff --git a/app/template_generators/docker/compose.py b/app/template_generators/docker/compose.py index 23b7cf92..c2770d57 100644 --- a/app/template_generators/docker/compose.py +++ b/app/template_generators/docker/compose.py @@ -1,59 +1,11 @@ -def docker_compose_generator(input): - compose_network = input.network.name - compose_services = input.services - services = [i.name for i in compose_services] - images = [{i.name:i.image_full} for i in compose_services] - volumes = [{i.name:i.volumes_full} for i in compose_services] - depends_on = [{i.name:i.depends_on} for i in compose_services] - ports = [{i.name:i.ports} for i in compose_services] - env = [{i.name:i.environments} for i in compose_services] - networks = [{i.name:i.networks} for i in compose_services] - - - prompt = f""" - - generate a python code (with out any ```python entry or additionals) with generates a docker-compose.yaml file in the directory 'app/media/MyCompose' - - the docker-compose.yaml, must following there instructions: - the version must be = 3 - set services following this list: {services} - set images to serivce following this dict : {images} - set volumes to service following this dict : {volumes} - set depends_on to service following this dict : {depends_on} - set ports to service following this dict : {ports} - set environment to service following this dict : {env} - set netwotks to service following this dict : {networks} - - - finally, at the end of docker-compose file, add following block: - ``` - networks: - {compose_network}: - driver: bridge - - ``` - - - finally just give me a python code without any note that can generate a project folder with the - given schema without ```python entry. and we dont need any base directory in the python code. - the final ansible template must work very well without any error! - - - the python code you give me, must have structure like that: - - import os - project_name = "app/media/MyCompose" - foo_dir = os.path.join(project_name, "bar") - x_dir = os.path.join(modules_dir, "y") +import yaml +from app.models.compose_models import DockerCompose - # Create project directories - os.makedirs(compose_dir, exist_ok=True) - - # Create main.tf - with open(os.path.join(project_name, "main.tf"), "w") as main_file: - # any thing you need - +def docker_compose_generator(input): + compose_total = input.model_dump(mode="json") - """ - return prompt \ No newline at end of file + file=open("app/media/MyCompose/docker-compose.yaml","w") + yaml.dump(compose_total,file) + file.close() + \ No newline at end of file From c61adb30fe14c1f5ef5dfbdc64dbe8453026f446 Mon Sep 17 00:00:00 2001 From: abolfazl1381 Date: Wed, 4 Dec 2024 23:18:50 +0330 Subject: [PATCH 06/22] fix(compose): directory builder --- app/media/MyCompose/docker-compose.yaml | 2 +- app/template_generators/docker/compose.py | 12 ++++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/app/media/MyCompose/docker-compose.yaml b/app/media/MyCompose/docker-compose.yaml index f8882cd1..607ae626 100644 --- a/app/media/MyCompose/docker-compose.yaml +++ b/app/media/MyCompose/docker-compose.yaml @@ -44,7 +44,7 @@ services: environment: additionalProp1: string additionalProp2: string - additionalProp3: string + additionalProp3: shhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhh image: string networks: - string diff --git a/app/template_generators/docker/compose.py b/app/template_generators/docker/compose.py index c2770d57..2ef066a4 100644 --- a/app/template_generators/docker/compose.py +++ b/app/template_generators/docker/compose.py @@ -1,10 +1,18 @@ import yaml from app.models.compose_models import DockerCompose +import os def docker_compose_generator(input): - + dir = 'app/media/MyCompose' compose_total = input.model_dump(mode="json") - + if not os.path.exists(dir): + os.makedirs(dir) + os.path.join(dir, 'docker-compose.yaml') + + file=open("app/media/MyCompose/docker-compose.yaml","w") + yaml.dump(compose_total,file) + file.close() + file=open("app/media/MyCompose/docker-compose.yaml","w") yaml.dump(compose_total,file) file.close() From 7db40d2da6f34222d6c454ec32636ed8cc6317a0 Mon Sep 17 00:00:00 2001 From: abolfazl1381 Date: Thu, 5 Dec 2024 11:04:47 +0330 Subject: [PATCH 07/22] fix(compose): compelete compose generation allgorithm --- app/media/MyCompose/docker-compose.yaml | 94 ++++++++--------------- app/models/compose_models.py | 28 ++++--- app/template_generators/docker/compose.py | 13 +++- m.py | 29 +++++++ 4 files changed, 90 insertions(+), 74 deletions(-) create mode 100644 m.py diff --git a/app/media/MyCompose/docker-compose.yaml b/app/media/MyCompose/docker-compose.yaml index 607ae626..19cb7613 100644 --- a/app/media/MyCompose/docker-compose.yaml +++ b/app/media/MyCompose/docker-compose.yaml @@ -1,78 +1,50 @@ -networks: - additionalProp1: - driver: string - additionalProp2: - driver: string - additionalProp3: - driver: string +version: '3' services: - additionalProp1: - args: - additionalProp1: string - additionalProp2: string - additionalProp3: string - build: - context: string - dockerfile: string - command: string - container_name: string - depends_on: - - string + web1: + image: nginx:latest + container_name: web_server environment: - additionalProp1: string - additionalProp2: string - additionalProp3: string - image: string - networks: - - string + foo: bar ports: - - string - volumes: - - string - additionalProp2: - args: - additionalProp1: string - additionalProp2: string - additionalProp3: string + - 80:80 + networks: + - app_network + web2: build: - context: string - dockerfile: string + context: . + dockerfile: DockerFile + image: nginx:latest + container_name: web_server command: string - container_name: string - depends_on: + volumes: - string environment: - additionalProp1: string - additionalProp2: string - additionalProp3: shhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhh - image: string - networks: - - string + foo: bar ports: - - string - volumes: - - string - additionalProp3: + - 80:80 + networks: + - app_network args: additionalProp1: string additionalProp2: string additionalProp3: string + depends_on: + - string + web3: build: - context: string - dockerfile: string + context: . + dockerfile: DockerFile + image: nginx:latest + container_name: web_server command: string - container_name: string - depends_on: + volumes: - string environment: - additionalProp1: string - additionalProp2: string - additionalProp3: string - image: string - networks: - - string + foo: bar ports: - - string - volumes: - - string -version: string + - 80:80 + networks: + - app_network +networks: + app: + driver: bridge diff --git a/app/models/compose_models.py b/app/models/compose_models.py index 137dc1f1..62e409f7 100644 --- a/app/models/compose_models.py +++ b/app/models/compose_models.py @@ -1,27 +1,33 @@ from typing import Dict, List, Optional -from pydantic import BaseModel +from pydantic import BaseModel, model_validator class Build(BaseModel): - context: str - dockerfile: str + context: str = "." + dockerfile: str = "DockerFile" class Service(BaseModel): build: Optional[Build] = None - image: Optional[str] = None - container_name: Optional[str] = None + image: Optional[str] = "nginx:latest" + container_name: Optional[str] = "web_server" command: Optional[str] = None volumes: Optional[List[str]] = None - environment: Optional[Dict[str, str]] = None - ports: Optional[List[str]] = None - networks: Optional[List[str]] = None + environment: Optional[Dict[str, str]] = {"foo":"bar"} + ports: Optional[List[str]] = ["80:80"] + networks: Optional[List[str]] = ["app_network"] args: Optional[Dict[str, str]] = None depends_on: Optional[List[str]] = None + @model_validator(mode="after") + def validator(self): + if self.build == None and self.image == None: + raise ValueError(f"one of the build or image sections must be present!") + return self + class Network(BaseModel): - driver: str + driver: str = "bridge" class DockerCompose(BaseModel): - version: str + version: str = "3" services: Dict[str, Service] - networks: Optional[Dict[str, Network]] = None + networks: Optional[Dict[str, Network]] diff --git a/app/template_generators/docker/compose.py b/app/template_generators/docker/compose.py index 2ef066a4..7a11eb31 100644 --- a/app/template_generators/docker/compose.py +++ b/app/template_generators/docker/compose.py @@ -1,19 +1,28 @@ import yaml from app.models.compose_models import DockerCompose import os +def remove_none_values(d): + if isinstance(d, dict): + return {k: remove_none_values(v) for k, v in d.items() if v is not None} + elif isinstance(d, list): + return [remove_none_values(i) for i in d if i is not None] + return d + def docker_compose_generator(input): dir = 'app/media/MyCompose' + compose_total = input.model_dump(mode="json") + compose_total = remove_none_values(compose_total) if not os.path.exists(dir): os.makedirs(dir) os.path.join(dir, 'docker-compose.yaml') file=open("app/media/MyCompose/docker-compose.yaml","w") - yaml.dump(compose_total,file) + yaml.dump(compose_total,file,default_flow_style=False) file.close() file=open("app/media/MyCompose/docker-compose.yaml","w") - yaml.dump(compose_total,file) + yaml.dump(compose_total,file,default_flow_style=False,sort_keys=False) file.close() \ No newline at end of file diff --git a/m.py b/m.py new file mode 100644 index 00000000..976d59bd --- /dev/null +++ b/m.py @@ -0,0 +1,29 @@ +{ + "version": "3", + "services": { + "webserver": { + "build": { + "context": ".", + "dockerfile": "DockerFile" + }, + "image": null, + "container_name": "web_server", + "command": null, + "volumes": null, + "environment": { + "foo": "bar" + }, + "ports": [ + "80:80" + ], + "networks": [ + "app_network" + ], + "args": null, + "depends_on": null + + + } + +} +} \ No newline at end of file From b9e4ed16fb903bb69b4886784d5dc48f52b04cfb Mon Sep 17 00:00:00 2001 From: abolfazl1381 Date: Thu, 5 Dec 2024 11:22:05 +0330 Subject: [PATCH 08/22] fix(compose): edit default values for documentation --- app/media/MyCompose/docker-compose.yaml | 33 ++++++++++--------------- app/models/compose_models.py | 14 +++++------ 2 files changed, 20 insertions(+), 27 deletions(-) diff --git a/app/media/MyCompose/docker-compose.yaml b/app/media/MyCompose/docker-compose.yaml index 19cb7613..27640329 100644 --- a/app/media/MyCompose/docker-compose.yaml +++ b/app/media/MyCompose/docker-compose.yaml @@ -1,23 +1,14 @@ version: '3' services: - web1: - image: nginx:latest - container_name: web_server - environment: - foo: bar - ports: - - 80:80 - networks: - - app_network - web2: + web: build: context: . dockerfile: DockerFile image: nginx:latest container_name: web_server - command: string + command: command... volumes: - - string + - ./foo:bar environment: foo: bar ports: @@ -25,26 +16,28 @@ services: networks: - app_network args: - additionalProp1: string - additionalProp2: string - additionalProp3: string + foo: bar depends_on: - - string - web3: + - service 0 + web2: build: context: . dockerfile: DockerFile image: nginx:latest container_name: web_server - command: string + command: command... volumes: - - string + - ./foo:bar environment: foo: bar ports: - 80:80 networks: - app_network + args: + foo: bar + depends_on: + - service 0 networks: - app: + app_network: driver: bridge diff --git a/app/models/compose_models.py b/app/models/compose_models.py index 62e409f7..9f53a9f5 100644 --- a/app/models/compose_models.py +++ b/app/models/compose_models.py @@ -6,16 +6,16 @@ class Build(BaseModel): dockerfile: str = "DockerFile" class Service(BaseModel): - build: Optional[Build] = None + build: Optional[Build] = Build() image: Optional[str] = "nginx:latest" container_name: Optional[str] = "web_server" - command: Optional[str] = None - volumes: Optional[List[str]] = None + command: Optional[str] = "command..." + volumes: Optional[List[str]] = ["./foo:bar"] environment: Optional[Dict[str, str]] = {"foo":"bar"} ports: Optional[List[str]] = ["80:80"] networks: Optional[List[str]] = ["app_network"] - args: Optional[Dict[str, str]] = None - depends_on: Optional[List[str]] = None + args: Optional[Dict[str, str]] = {"foo":"bar"} + depends_on: Optional[List[str]] = ['service 0'] @model_validator(mode="after") def validator(self): @@ -28,6 +28,6 @@ class Network(BaseModel): class DockerCompose(BaseModel): version: str = "3" - services: Dict[str, Service] - networks: Optional[Dict[str, Network]] + services: Dict[str, Service] = {"web":Service(), "web2":Service()} + networks: Optional[Dict[str, Network]] = {"app_network": {"driver":"bridge"}} From 41c6fdeb533eacee97161f93dfd3ee33cc8182ad Mon Sep 17 00:00:00 2001 From: abolfazl1381 Date: Thu, 5 Dec 2024 12:06:52 +0330 Subject: [PATCH 09/22] feat(compose): add union type input for networks --- app/media/MyCompose/docker-compose.yaml | 8 ++++---- app/models/compose_models.py | 13 ++++++++----- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/app/media/MyCompose/docker-compose.yaml b/app/media/MyCompose/docker-compose.yaml index 27640329..2bd21d8e 100644 --- a/app/media/MyCompose/docker-compose.yaml +++ b/app/media/MyCompose/docker-compose.yaml @@ -4,6 +4,8 @@ services: build: context: . dockerfile: DockerFile + args: + foo: bar image: nginx:latest container_name: web_server command: command... @@ -15,14 +17,14 @@ services: - 80:80 networks: - app_network - args: - foo: bar depends_on: - service 0 web2: build: context: . dockerfile: DockerFile + args: + foo: bar image: nginx:latest container_name: web_server command: command... @@ -34,8 +36,6 @@ services: - 80:80 networks: - app_network - args: - foo: bar depends_on: - service 0 networks: diff --git a/app/models/compose_models.py b/app/models/compose_models.py index 9f53a9f5..907a4ef8 100644 --- a/app/models/compose_models.py +++ b/app/models/compose_models.py @@ -1,10 +1,10 @@ -from typing import Dict, List, Optional +from typing import Dict, List, Optional,Union from pydantic import BaseModel, model_validator class Build(BaseModel): context: str = "." dockerfile: str = "DockerFile" - + args: Optional[Dict[str, str]] = {"foo":"bar"} class Service(BaseModel): build: Optional[Build] = Build() image: Optional[str] = "nginx:latest" @@ -14,7 +14,7 @@ class Service(BaseModel): environment: Optional[Dict[str, str]] = {"foo":"bar"} ports: Optional[List[str]] = ["80:80"] networks: Optional[List[str]] = ["app_network"] - args: Optional[Dict[str, str]] = {"foo":"bar"} + depends_on: Optional[List[str]] = ['service 0'] @model_validator(mode="after") @@ -26,8 +26,11 @@ def validator(self): class Network(BaseModel): driver: str = "bridge" +class PreCreatedNetwork(BaseModel): + name:str = "net1" + external:bool = True class DockerCompose(BaseModel): version: str = "3" services: Dict[str, Service] = {"web":Service(), "web2":Service()} - networks: Optional[Dict[str, Network]] = {"app_network": {"driver":"bridge"}} - + networks: Union[Optional[Dict[str, PreCreatedNetwork]],Optional[Dict[str, Network]]] = {"app_network": {"driver":"bridge"}} + From e4f71179f66b37e89dc5d928c59d0ef80d6c613f Mon Sep 17 00:00:00 2001 From: abolfazl1381 Date: Fri, 6 Dec 2024 21:20:53 +0330 Subject: [PATCH 10/22] fix(routes): add /api to all routes --- app/media/MyCompose/docker-compose.yaml | 39 ++++-------------------- app/media/MyCompose_zip.zip | Bin 0 -> 285 bytes app/models/compose_models.py | 3 +- app/routes/ansible.py | 6 ++-- app/routes/docker.py | 2 +- app/routes/helm.py | 2 +- app/routes/jcasc.py | 2 +- app/routes/terraform.py | 20 ++++++------ app/routes/utils.py | 2 +- 9 files changed, 25 insertions(+), 51 deletions(-) create mode 100644 app/media/MyCompose_zip.zip diff --git a/app/media/MyCompose/docker-compose.yaml b/app/media/MyCompose/docker-compose.yaml index 2bd21d8e..2eafd8d6 100644 --- a/app/media/MyCompose/docker-compose.yaml +++ b/app/media/MyCompose/docker-compose.yaml @@ -3,41 +3,14 @@ services: web: build: context: . - dockerfile: DockerFile - args: - foo: bar + dockerfile: Dockerfile + args: {} image: nginx:latest - container_name: web_server - command: command... - volumes: - - ./foo:bar - environment: - foo: bar + container_name: x ports: - 80:80 - networks: - - app_network - depends_on: - - service 0 - web2: - build: - context: . - dockerfile: DockerFile - args: - foo: bar - image: nginx:latest - container_name: web_server - command: command... - volumes: - - ./foo:bar - environment: - foo: bar - ports: - - 80:80 - networks: - - app_network - depends_on: - - service 0 + - 90:90 + - 70:70 networks: - app_network: + p: driver: bridge diff --git a/app/media/MyCompose_zip.zip b/app/media/MyCompose_zip.zip new file mode 100644 index 0000000000000000000000000000000000000000..66b5179797380b71eaa466b0d75c41882d82f9f9 GIT binary patch literal 285 zcmWIWW@Zs#U|`^2khE)ytjR4}GZn~t3dF)doRXiMom!-uoS$2eU!1B}nV6en+iT0$ zY{298ea<22;ME?o4CnW{i*`syHZ;!>nXu~K-YN6_>i+%7@n6!pSV3g_A(cy=(Owx0 zX<2ow3n~lWm$PInPKh!)^p&F~$NG}`_m+bHpAAc-gbw}QywzsM1}~uqu{W{Hrkt(3 zv~hyn5#H4sa$ZW9F8}+QRjYfd?T&2`^JAiP)(V~~zLK~3c Output: if os.environ.get("TEST"): @@ -22,7 +22,7 @@ async def ansible_install_generation_nginx(request:AnsibleInstallNginx) -> Outpu return Output(output='output') -@app.post("/ansible-install/docker/") +@app.post("/api/ansible-install/docker/") async def ansible_install_generation_docker(request:AnsibleInstallDocker) -> Output: if os.environ.get("TEST"): @@ -35,7 +35,7 @@ async def ansible_install_generation_docker(request:AnsibleInstallDocker) -> Out return Output(output='output') -@app.post("/ansible-install/kuber/") +@app.post("/api/ansible-install/kuber/") async def ansible_install_generation_kuber(request:AnsibleInstallKuber) -> Output: if os.environ.get("TEST"): diff --git a/app/routes/docker.py b/app/routes/docker.py index c62a487c..727d9773 100644 --- a/app/routes/docker.py +++ b/app/routes/docker.py @@ -5,7 +5,7 @@ from app.template_generators.docker.compose import docker_compose_generator import os -@app.post("/docker-compose/") +@app.post("/api/docker-compose/") async def docker_compose_template(request:DockerCompose) -> Output: if os.environ.get("TEST"): diff --git a/app/routes/helm.py b/app/routes/helm.py index 7470024c..dde13d44 100644 --- a/app/routes/helm.py +++ b/app/routes/helm.py @@ -4,7 +4,7 @@ from app.models import (HelmTemplateGeneration,Output) from app.prompt_generators import (helm_template_generator) import os -@app.post("/Helm-template/") +@app.post("/api/Helm-template/") async def Helm_template_generation(request:HelmTemplateGeneration) -> Output: if os.environ.get("TEST"): return Output(output='output') diff --git a/app/routes/jcasc.py b/app/routes/jcasc.py index 99802d8f..5ad85ab6 100644 --- a/app/routes/jcasc.py +++ b/app/routes/jcasc.py @@ -5,7 +5,7 @@ from app.template_generators.jenkins.jcasc import jcasc_template_generator import os -@app.post("/jcasc-template/") +@app.post("/api/jcasc-template/") async def jcasc_template_generation(request:Jcasc) -> Output: if os.environ.get("TEST"): return Output(output='output') diff --git a/app/routes/terraform.py b/app/routes/terraform.py index 800e1da4..6efab360 100644 --- a/app/routes/terraform.py +++ b/app/routes/terraform.py @@ -31,7 +31,7 @@ from app.template_generators.terraform.aws.EFS import (IaC_template_generator_efs) import os -@app.post("/IaC-basic/") +@app.post("/api/IaC-basic/") async def IaC_basic_generation(request:IaCBasicInput) -> Output: if os.environ.get("TEST"): return Output(output='Terraform developed by hashicorp and it is very usefull') @@ -39,7 +39,7 @@ async def IaC_basic_generation(request:IaCBasicInput) -> Output: output = gpt_service(generated_prompt) return Output(output=output) -@app.post("/IaC-bugfix/") +@app.post("/api/IaC-bugfix/") async def IaC_bugfix_generation(request:IaCBugfixInput) -> Output: if os.environ.get("TEST"): return Output(output='fix this bug by adding x to the y') @@ -48,7 +48,7 @@ async def IaC_bugfix_generation(request:IaCBugfixInput) -> Output: return Output(output=output) -@app.post("/IaC-install/") +@app.post("/api/IaC-install/") async def IaC_install_generation(request:IaCInstallationInput) -> Output: if os.environ.get("TEST"): return Output(output='apt-get install xyz \n apt-get update (covert them to shell file output)') @@ -56,7 +56,7 @@ async def IaC_install_generation(request:IaCInstallationInput) -> Output: output = gpt_service(generated_prompt) return Output(output=output) -@app.post("/IaC-template/docker") +@app.post("/api/IaC-template/docker") async def IaC_template_generation_docker(request:IaCTemplateGenerationDocker) -> Output: if os.environ.get("TEST"): return Output(output='output (nothing special)') @@ -66,7 +66,7 @@ async def IaC_template_generation_docker(request:IaCTemplateGenerationDocker) -> execute_pythonfile("MyTerraform","terraform_generator") return Output(output='output') -@app.post("/IaC-template/aws/ec2") +@app.post("/api/IaC-template/aws/ec2") async def IaC_template_generation_aws_ec2(request:IaCTemplateGenerationEC2) -> Output: if os.environ.get("TEST"): return Output(output='output (nothing special)') @@ -77,7 +77,7 @@ async def IaC_template_generation_aws_ec2(request:IaCTemplateGenerationEC2) -> O execute_pythonfile("MyTerraform","terraform_generator") return Output(output='output') -@app.post("/IaC-template/aws/s3") +@app.post("/api/IaC-template/aws/s3") async def IaC_template_generation_aws_s3(request:IaCTemplateGenerationS3) -> Output: if os.environ.get("TEST"): return Output(output='output (nothing special)') @@ -87,7 +87,7 @@ async def IaC_template_generation_aws_s3(request:IaCTemplateGenerationS3) -> Out execute_pythonfile("MyTerraform","terraform_generator") return Output(output='output') -@app.post("/IaC-template/aws/iam") +@app.post("/api/IaC-template/aws/iam") async def IaC_template_generation_aws_iam(request:IaCTemplateGenerationIAM) -> Output: if os.environ.get("TEST"): return Output(output='output (nothing special)') @@ -98,7 +98,7 @@ async def IaC_template_generation_aws_iam(request:IaCTemplateGenerationIAM) -> O return Output(output='output') -@app.post("/IaC-template/argocd") +@app.post("/api/IaC-template/argocd") async def IaC_template_generation_argocd(request:IaCTemplateGenerationArgoCD) -> Output: if os.environ.get("TEST"): return Output(output='output (nothing special)') @@ -110,7 +110,7 @@ async def IaC_template_generation_argocd(request:IaCTemplateGenerationArgoCD) -> -@app.post("/IaC-template/aws/elb") +@app.post("/api/IaC-template/aws/elb") async def IaC_template_generation_aws_elb(request:IaCTemplateGenerationELB) -> Output: if os.environ.get("TEST"): return Output(output='output (nothing special)') @@ -120,7 +120,7 @@ async def IaC_template_generation_aws_elb(request:IaCTemplateGenerationELB) -> O execute_pythonfile("MyTerraform","terraform_generator") return Output(output='output') -@app.post("/IaC-template/aws/efs") +@app.post("/api/IaC-template/aws/efs") async def IaC_template_generation_aws_efs(request:IaCTemplateGenerationEFS) -> Output: if os.environ.get("TEST"): return Output(output='output (nothing special)') diff --git a/app/routes/utils.py b/app/routes/utils.py index a7a5d7b1..b0a62ef7 100644 --- a/app/routes/utils.py +++ b/app/routes/utils.py @@ -26,7 +26,7 @@ def add_files_to_folder(files:list,folder:str): shutil.copy(filename, destination_file) -@app.get("/download-folder{folder_name}/{source}") +@app.get("/api/download-folder{folder_name}/{source}") async def download_folder_MyHelm(folder_name: str,source:str): folder_path = f"app/media/{folder_name}" # Adjust the path as needed if not os.path.exists(folder_path): From fd2efe7fe4563c44f485ee795f5d90a2b090bd9a Mon Sep 17 00:00:00 2001 From: abolfazl1381 Date: Sun, 8 Dec 2024 11:20:54 +0330 Subject: [PATCH 11/22] fix(installation): fix terraform installation process and model --- Makefile | 6 ++-- .../Terraform/amazon_linux.sh | 8 +++++ .../Installation_base/Terraform/centos.sh | 8 +++++ .../Installation_base/Terraform/fedora.sh | 8 +++++ .../Installation_base/Terraform/ubuntu.sh | 15 ++++++++ app/media/MyCompose_zip.zip | Bin 285 -> 0 bytes app/models/compose_models.py | 1 - app/models/terraform_models.py | 22 ++++++------ app/routes/terraform.py | 6 ++-- .../terraform/Installation/__init__.py | 0 .../terraform/Installation/main.py | 33 ++++++++++++++++++ 11 files changed, 89 insertions(+), 18 deletions(-) create mode 100644 app/media/Installation_base/Terraform/amazon_linux.sh create mode 100644 app/media/Installation_base/Terraform/centos.sh create mode 100644 app/media/Installation_base/Terraform/fedora.sh create mode 100644 app/media/Installation_base/Terraform/ubuntu.sh delete mode 100644 app/media/MyCompose_zip.zip create mode 100644 app/template_generators/terraform/Installation/__init__.py create mode 100644 app/template_generators/terraform/Installation/main.py diff --git a/Makefile b/Makefile index 056f35a1..5b552a6f 100644 --- a/Makefile +++ b/Makefile @@ -4,11 +4,11 @@ releaseName ?= devopsgpt all: build up build: - docker-compose build + docker compose build up: - docker-compose up -d + docker compose up -d down: - docker-compose down -v + docker compose down -v helm-install: helm install $(releaseName) helm/ -f helm/values.yaml -n $(namespace) --create-namespace helm-uninstall: diff --git a/app/media/Installation_base/Terraform/amazon_linux.sh b/app/media/Installation_base/Terraform/amazon_linux.sh new file mode 100644 index 00000000..9b423b8c --- /dev/null +++ b/app/media/Installation_base/Terraform/amazon_linux.sh @@ -0,0 +1,8 @@ +#!bin/bash + + +sudo yum install -y yum-utils + +sudo yum-config-manager --add-repo https://rpm.releases.hashicorp.com/AmazonLinux/hashicorp.repo + +sudo yum -y install terraform \ No newline at end of file diff --git a/app/media/Installation_base/Terraform/centos.sh b/app/media/Installation_base/Terraform/centos.sh new file mode 100644 index 00000000..bf7e5745 --- /dev/null +++ b/app/media/Installation_base/Terraform/centos.sh @@ -0,0 +1,8 @@ +#!bin/bash + + +sudo yum install -y yum-utils + +sudo yum-config-manager --add-repo https://rpm.releases.hashicorp.com/RHEL/hashicorp.repo + +sudo yum -y install terraform \ No newline at end of file diff --git a/app/media/Installation_base/Terraform/fedora.sh b/app/media/Installation_base/Terraform/fedora.sh new file mode 100644 index 00000000..9851b25e --- /dev/null +++ b/app/media/Installation_base/Terraform/fedora.sh @@ -0,0 +1,8 @@ +#!bin/bash + + +sudo dnf install -y dnf-plugins-core + +sudo dnf config-manager --add-repo https://rpm.releases.hashicorp.com/fedora/hashicorp.repo + +sudo dnf -y install terraform \ No newline at end of file diff --git a/app/media/Installation_base/Terraform/ubuntu.sh b/app/media/Installation_base/Terraform/ubuntu.sh new file mode 100644 index 00000000..d3bd0505 --- /dev/null +++ b/app/media/Installation_base/Terraform/ubuntu.sh @@ -0,0 +1,15 @@ +#!bin/bash + +sudo apt-get update && sudo apt-get install -y gnupg software-properties-common + +wget -O- https://apt.releases.hashicorp.com/gpg | \ +gpg --dearmor | \ +sudo tee /usr/share/keyrings/hashicorp-archive-keyring.gpg > /dev/null + +echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] \ +https://apt.releases.hashicorp.com $(lsb_release -cs) main" | \ +sudo tee /etc/apt/sources.list.d/hashicorp.list + +sudo apt update + +sudo apt-get install terraform \ No newline at end of file diff --git a/app/media/MyCompose_zip.zip b/app/media/MyCompose_zip.zip deleted file mode 100644 index 66b5179797380b71eaa466b0d75c41882d82f9f9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 285 zcmWIWW@Zs#U|`^2khE)ytjR4}GZn~t3dF)doRXiMom!-uoS$2eU!1B}nV6en+iT0$ zY{298ea<22;ME?o4CnW{i*`syHZ;!>nXu~K-YN6_>i+%7@n6!pSV3g_A(cy=(Owx0 zX<2ow3n~lWm$PInPKh!)^p&F~$NG}`_m+bHpAAc-gbw}QywzsM1}~uqu{W{Hrkt(3 zv~hyn5#H4sa$ZW9F8}+QRjYfd?T&2`^JAiP)(V~~zLK~3c Output: async def IaC_install_generation(request:IaCInstallationInput) -> Output: if os.environ.get("TEST"): return Output(output='apt-get install xyz \n apt-get update (covert them to shell file output)') - generated_prompt = IaC_installation_generator(request) - output = gpt_service(generated_prompt) - return Output(output=output) + selected_script = select_install(request) + return Output(output=selected_script) @app.post("/api/IaC-template/docker") async def IaC_template_generation_docker(request:IaCTemplateGenerationDocker) -> Output: diff --git a/app/template_generators/terraform/Installation/__init__.py b/app/template_generators/terraform/Installation/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/template_generators/terraform/Installation/main.py b/app/template_generators/terraform/Installation/main.py new file mode 100644 index 00000000..b4d3bc78 --- /dev/null +++ b/app/template_generators/terraform/Installation/main.py @@ -0,0 +1,33 @@ + +def select_install(input): + + match input.os: + + case "Ubuntu": + with open("app/media/Installation_base/Terraform/ubuntu.sh", 'r') as file: + file_content = file.read() + + return file_content + + case "Fedora": + with open("app/media/Installation_base/Terraform/fedora.sh", 'r') as file: + file_content = file.read() + + return file_content + + case "Centos": + with open("app/media/Installation_base/Terraform/centos.sh", 'r') as file: + file_content = file.read() + + return file_content + + case "Amazon_linux": + with open("app/media/Installation_base/Terraform/amazon_linux.sh", 'r') as file: + file_content = file.read() + + return file_content + case _: + raise ValueError() + + + \ No newline at end of file From 26c3bd89c8053c9677d2e783f7fd621114ad91b9 Mon Sep 17 00:00:00 2001 From: abolfazl1381 Date: Sun, 8 Dec 2024 12:16:15 +0330 Subject: [PATCH 12/22] fix(installation): create MyBash for scripts --- app/media/MyBash/bash.sh | 15 +++++++ app/routes/terraform.py | 6 +-- .../terraform/Installation/main.py | 44 ++++++++++++------- 3 files changed, 46 insertions(+), 19 deletions(-) create mode 100644 app/media/MyBash/bash.sh diff --git a/app/media/MyBash/bash.sh b/app/media/MyBash/bash.sh new file mode 100644 index 00000000..d3bd0505 --- /dev/null +++ b/app/media/MyBash/bash.sh @@ -0,0 +1,15 @@ +#!bin/bash + +sudo apt-get update && sudo apt-get install -y gnupg software-properties-common + +wget -O- https://apt.releases.hashicorp.com/gpg | \ +gpg --dearmor | \ +sudo tee /usr/share/keyrings/hashicorp-archive-keyring.gpg > /dev/null + +echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] \ +https://apt.releases.hashicorp.com $(lsb_release -cs) main" | \ +sudo tee /etc/apt/sources.list.d/hashicorp.list + +sudo apt update + +sudo apt-get install terraform \ No newline at end of file diff --git a/app/routes/terraform.py b/app/routes/terraform.py index 1a21a437..20ebbe03 100644 --- a/app/routes/terraform.py +++ b/app/routes/terraform.py @@ -52,9 +52,9 @@ async def IaC_bugfix_generation(request:IaCBugfixInput) -> Output: @app.post("/api/IaC-install/") async def IaC_install_generation(request:IaCInstallationInput) -> Output: if os.environ.get("TEST"): - return Output(output='apt-get install xyz \n apt-get update (covert them to shell file output)') - selected_script = select_install(request) - return Output(output=selected_script) + return Output(output='nothing special') + select_install(request) + return Output(output="pk") @app.post("/api/IaC-template/docker") async def IaC_template_generation_docker(request:IaCTemplateGenerationDocker) -> Output: diff --git a/app/template_generators/terraform/Installation/main.py b/app/template_generators/terraform/Installation/main.py index b4d3bc78..ffdb14e1 100644 --- a/app/template_generators/terraform/Installation/main.py +++ b/app/template_generators/terraform/Installation/main.py @@ -1,31 +1,43 @@ +import os +import shutil + +def create_MyBash_directory(): + + dir = 'app/media/MyBash' + + + if not os.path.exists(dir): + os.makedirs(dir) + os.path.join(dir, 'bash.sh') + + + def select_install(input): + create_MyBash_directory() match input.os: + case "Ubuntu": - with open("app/media/Installation_base/Terraform/ubuntu.sh", 'r') as file: - file_content = file.read() - - return file_content + source = 'app/media/Installation_base/Terraform/ubuntu.sh' + dest = 'app/media/MyBash/bash.sh' + shutil.copyfile(source, dest) case "Fedora": - with open("app/media/Installation_base/Terraform/fedora.sh", 'r') as file: - file_content = file.read() - - return file_content + source = 'app/media/Installation_base/Terraform/fedora.sh' + dest = 'app/media/MyBash/bash.sh' + shutil.copyfile(source, dest) case "Centos": - with open("app/media/Installation_base/Terraform/centos.sh", 'r') as file: - file_content = file.read() - - return file_content + source = 'app/media/Installation_base/Terraform/centos.sh' + dest = 'app/media/MyBash/bash.sh' + shutil.copyfile(source, dest) case "Amazon_linux": - with open("app/media/Installation_base/Terraform/amazon_linux.sh", 'r') as file: - file_content = file.read() - - return file_content + source = 'app/media/Installation_base/Terraform/amazon_linux.sh' + dest = 'app/media/MyBash/bash.sh' + shutil.copyfile(source, dest) case _: raise ValueError() From 90bece3e2864b3cfe9fdaa54376e844955b9fdf8 Mon Sep 17 00:00:00 2001 From: abolfazl1381 Date: Sun, 8 Dec 2024 13:07:20 +0330 Subject: [PATCH 13/22] fix(bash): edit bi/bash --- app/media/Installation_base/Terraform/amazon_linux.sh | 2 +- app/media/Installation_base/Terraform/centos.sh | 2 +- app/media/Installation_base/Terraform/fedora.sh | 2 +- app/media/Installation_base/Terraform/ubuntu.sh | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/app/media/Installation_base/Terraform/amazon_linux.sh b/app/media/Installation_base/Terraform/amazon_linux.sh index 9b423b8c..fa26a32b 100644 --- a/app/media/Installation_base/Terraform/amazon_linux.sh +++ b/app/media/Installation_base/Terraform/amazon_linux.sh @@ -1,4 +1,4 @@ -#!bin/bash +#!/bin/bash sudo yum install -y yum-utils diff --git a/app/media/Installation_base/Terraform/centos.sh b/app/media/Installation_base/Terraform/centos.sh index bf7e5745..094b3079 100644 --- a/app/media/Installation_base/Terraform/centos.sh +++ b/app/media/Installation_base/Terraform/centos.sh @@ -1,4 +1,4 @@ -#!bin/bash +#!/bin/bash sudo yum install -y yum-utils diff --git a/app/media/Installation_base/Terraform/fedora.sh b/app/media/Installation_base/Terraform/fedora.sh index 9851b25e..673dbbdf 100644 --- a/app/media/Installation_base/Terraform/fedora.sh +++ b/app/media/Installation_base/Terraform/fedora.sh @@ -1,4 +1,4 @@ -#!bin/bash +#!/bin/bash sudo dnf install -y dnf-plugins-core diff --git a/app/media/Installation_base/Terraform/ubuntu.sh b/app/media/Installation_base/Terraform/ubuntu.sh index d3bd0505..6dc8998a 100644 --- a/app/media/Installation_base/Terraform/ubuntu.sh +++ b/app/media/Installation_base/Terraform/ubuntu.sh @@ -1,4 +1,4 @@ -#!bin/bash +#!/bin/bash sudo apt-get update && sudo apt-get install -y gnupg software-properties-common From 54d507ff495c61a2c0e79f5155b2a18793bed2b1 Mon Sep 17 00:00:00 2001 From: abolfazl1381 Date: Sun, 8 Dec 2024 14:10:15 +0330 Subject: [PATCH 14/22] fix(docker install): fix it --- app/media/Installation_base/Docker/RHEL.sh | 19 ++++++++ app/media/Installation_base/Docker/centos.sh | 16 +++++++ app/media/Installation_base/Docker/fedora.sh | 19 ++++++++ app/media/Installation_base/Docker/ubuntu.sh | 16 +++++++ app/media/MyBash/bash.sh | 25 ++++++----- app/models/__init__.py | 3 +- app/models/docker_installation_models.py | 20 +++++++++ app/routes/docker.py | 13 ++++-- app/template_generators/docker/__init__.py | 0 .../docker/installation.py | 45 +++++++++++++++++++ 10 files changed, 159 insertions(+), 17 deletions(-) create mode 100644 app/media/Installation_base/Docker/RHEL.sh create mode 100644 app/media/Installation_base/Docker/centos.sh create mode 100644 app/media/Installation_base/Docker/fedora.sh create mode 100644 app/media/Installation_base/Docker/ubuntu.sh create mode 100644 app/models/docker_installation_models.py create mode 100644 app/template_generators/docker/__init__.py create mode 100644 app/template_generators/docker/installation.py diff --git a/app/media/Installation_base/Docker/RHEL.sh b/app/media/Installation_base/Docker/RHEL.sh new file mode 100644 index 00000000..6613df87 --- /dev/null +++ b/app/media/Installation_base/Docker/RHEL.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +sudo dnf remove -y \ + docker \ + docker-client \ + docker-client-latest \ + docker-common \ + docker-latest \ + docker-latest-logrotate \ + docker-logrotate \ + docker-engine \ + podman \ + runc + + +sudo dnf -y install dnf-plugins-core +sudo dnf config-manager --add-repo https://download.docker.com/linux/rhel/docker-ce.repo + +sudo dnf install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin \ No newline at end of file diff --git a/app/media/Installation_base/Docker/centos.sh b/app/media/Installation_base/Docker/centos.sh new file mode 100644 index 00000000..b29d7708 --- /dev/null +++ b/app/media/Installation_base/Docker/centos.sh @@ -0,0 +1,16 @@ +#!/bin/bash +sudo dnf remove -y \ + docker \ + docker-client \ + docker-client-latest \ + docker-common \ + docker-latest \ + docker-latest-logrotate \ + docker-logrotate \ + docker-engine + + +sudo dnf -y install dnf-plugins-core +sudo dnf config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo + +sudo dnf -y install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin \ No newline at end of file diff --git a/app/media/Installation_base/Docker/fedora.sh b/app/media/Installation_base/Docker/fedora.sh new file mode 100644 index 00000000..bbd9cff3 --- /dev/null +++ b/app/media/Installation_base/Docker/fedora.sh @@ -0,0 +1,19 @@ +#!/bin/bash +sudo dnf remove -y \ + docker \ + docker-client \ + docker-client-latest \ + docker-common \ + docker-latest \ + docker-latest-logrotate \ + docker-logrotate \ + docker-selinux \ + docker-engine-selinux \ + docker-engine + + +sudo dnf -y install dnf-plugins-core +sudo dnf-3 config-manager --add-repo https://download.docker.com/linux/fedora/docker-ce.repo + + +sudo dnf -y install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin \ No newline at end of file diff --git a/app/media/Installation_base/Docker/ubuntu.sh b/app/media/Installation_base/Docker/ubuntu.sh new file mode 100644 index 00000000..b223db9f --- /dev/null +++ b/app/media/Installation_base/Docker/ubuntu.sh @@ -0,0 +1,16 @@ +#!/bin/bash +sudo apt-get update -y +sudo apt-get install ca-certificates curl -y +sudo install -m 0755 -d /etc/apt/keyrings +sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc +sudo chmod a+r /etc/apt/keyrings/docker.asc + +# Add the repository to Apt sources: +echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \ + $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \ + sudo tee /etc/apt/sources.list.d/docker.list > /dev/null +sudo apt-get update -y + + +sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin \ No newline at end of file diff --git a/app/media/MyBash/bash.sh b/app/media/MyBash/bash.sh index d3bd0505..b223db9f 100644 --- a/app/media/MyBash/bash.sh +++ b/app/media/MyBash/bash.sh @@ -1,15 +1,16 @@ -#!bin/bash +#!/bin/bash +sudo apt-get update -y +sudo apt-get install ca-certificates curl -y +sudo install -m 0755 -d /etc/apt/keyrings +sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc +sudo chmod a+r /etc/apt/keyrings/docker.asc -sudo apt-get update && sudo apt-get install -y gnupg software-properties-common +# Add the repository to Apt sources: +echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \ + $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \ + sudo tee /etc/apt/sources.list.d/docker.list > /dev/null +sudo apt-get update -y -wget -O- https://apt.releases.hashicorp.com/gpg | \ -gpg --dearmor | \ -sudo tee /usr/share/keyrings/hashicorp-archive-keyring.gpg > /dev/null -echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] \ -https://apt.releases.hashicorp.com $(lsb_release -cs) main" | \ -sudo tee /etc/apt/sources.list.d/hashicorp.list - -sudo apt update - -sudo apt-get install terraform \ No newline at end of file +sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin \ No newline at end of file diff --git a/app/models/__init__.py b/app/models/__init__.py index f206434e..9f97ab17 100644 --- a/app/models/__init__.py +++ b/app/models/__init__.py @@ -3,4 +3,5 @@ from .utils import * from .ansible_models import * from .jcasc import * -from .compose_models import * \ No newline at end of file +from .compose_models import * +from .docker_installation_models import * \ No newline at end of file diff --git a/app/models/docker_installation_models.py b/app/models/docker_installation_models.py new file mode 100644 index 00000000..f64b9f58 --- /dev/null +++ b/app/models/docker_installation_models.py @@ -0,0 +1,20 @@ +from typing import Dict, List, Optional,Union +from pydantic import BaseModel, model_validator,validator + +class DockerInstallationInput(BaseModel): + os:str = "Ubuntu" + environment:str = "Linux" + + @validator("os") + def validate_os(cls, value): + allowed_os = ['Ubuntu', 'Centos', 'Fedora', 'RHEL'] + if value not in allowed_os: + raise ValueError(f"OS must be one of {allowed_os}.") + return value + + @validator("environment") + def validate_environment(cls, value): + allowed_os = ['Linux'] + if value not in allowed_os: + raise ValueError(f"Environment must be one of {allowed_os}.") + return value \ No newline at end of file diff --git a/app/routes/docker.py b/app/routes/docker.py index 727d9773..04b831aa 100644 --- a/app/routes/docker.py +++ b/app/routes/docker.py @@ -1,8 +1,7 @@ from app.app_instance import app -from app.gpt_services import gpt_service -from app.services import (write_installation,edit_directory_generator,execute_pythonfile) -from app.models import (DockerCompose,Output) +from app.models import (DockerCompose,DockerInstallationInput,Output) from app.template_generators.docker.compose import docker_compose_generator +from app.template_generators.docker.installation import docker_installation_selection import os @app.post("/api/docker-compose/") @@ -13,4 +12,10 @@ async def docker_compose_template(request:DockerCompose) -> Output: docker_compose_generator(request) return Output(output='output') - \ No newline at end of file + +@app.post("/api/docker/installation") +async def docker_installation(request:DockerInstallationInput) -> Output: + + docker_installation_selection(request) + + return Output(output='output') \ No newline at end of file diff --git a/app/template_generators/docker/__init__.py b/app/template_generators/docker/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/template_generators/docker/installation.py b/app/template_generators/docker/installation.py new file mode 100644 index 00000000..10adcad5 --- /dev/null +++ b/app/template_generators/docker/installation.py @@ -0,0 +1,45 @@ +import os +import shutil + + +def create_MyBash_directory(): + + dir = 'app/media/MyBash' + + + if not os.path.exists(dir): + os.makedirs(dir) + os.path.join(dir, 'bash.sh') + + + +def docker_installation_selection(input): + + create_MyBash_directory() + + match input.os: + + case "Ubuntu": + + source = 'app/media/Installation_base/Docker/ubuntu.sh' + dest = 'app/media/MyBash/bash.sh' + + shutil.copyfile(source, dest) + + + case "Fedora": + source = 'app/media/Installation_base/Docker/fedora.sh' + dest = 'app/media/MyBash/bash.sh' + shutil.copyfile(source, dest) + + case "Centos": + source = 'app/media/Installation_base/Docker/centos.sh' + dest = 'app/media/MyBash/bash.sh' + shutil.copyfile(source, dest) + + case "RHEL": + source = 'app/media/Installation_base/Docker/RHEL.sh' + dest = 'app/media/MyBash/bash.sh' + shutil.copyfile(source, dest) + case _: + raise ValueError() \ No newline at end of file From 3dcf0ee96e8bd1199569618877fc3042d821f19d Mon Sep 17 00:00:00 2001 From: abolfazl1381 Date: Mon, 9 Dec 2024 13:10:03 +0330 Subject: [PATCH 15/22] feat(install): add jenkins and gitlab installation --- app/main.py | 4 +- .../Gitlab/docker-compose.yaml | 31 +++++++++++ app/media/Installation_base/Jenkins/RHEL.sh | 9 +++ .../Jenkins/docker-compose.yml | 15 +++++ app/media/Installation_base/Jenkins/fedora.sh | 9 +++ app/media/Installation_base/Jenkins/ubuntu.sh | 12 ++++ app/media/MyBash/bash.sh | 24 ++++---- app/media/MyCompose/docker-compose.yaml | 31 +++++++++++ app/models/__init__.py | 4 +- app/models/gitlab_models.py | 17 ++++++ app/models/jenkins.py | 24 ++++++++ app/routes/docker.py | 1 + app/routes/gitlab.py | 14 +++++ app/routes/jenkins.py | 14 +++++ .../gitlab/installation.py | 35 ++++++++++++ .../jenkins/installation.py | 55 +++++++++++++++++++ .../terraform/Installation/main.py | 8 +-- 17 files changed, 287 insertions(+), 20 deletions(-) create mode 100644 app/media/Installation_base/Gitlab/docker-compose.yaml create mode 100644 app/media/Installation_base/Jenkins/RHEL.sh create mode 100644 app/media/Installation_base/Jenkins/docker-compose.yml create mode 100644 app/media/Installation_base/Jenkins/fedora.sh create mode 100644 app/media/Installation_base/Jenkins/ubuntu.sh create mode 100644 app/models/gitlab_models.py create mode 100644 app/models/jenkins.py create mode 100644 app/routes/gitlab.py create mode 100644 app/routes/jenkins.py create mode 100644 app/template_generators/gitlab/installation.py create mode 100644 app/template_generators/jenkins/installation.py diff --git a/app/main.py b/app/main.py index ca950b8b..1980475c 100644 --- a/app/main.py +++ b/app/main.py @@ -3,4 +3,6 @@ from app.routes.helm import * from app.routes.ansible import * from app.routes.jcasc import * -from app.routes.docker import * \ No newline at end of file +from app.routes.docker import * +from app.routes.jenkins import * +from app.routes.gitlab import * \ No newline at end of file diff --git a/app/media/Installation_base/Gitlab/docker-compose.yaml b/app/media/Installation_base/Gitlab/docker-compose.yaml new file mode 100644 index 00000000..fe15d044 --- /dev/null +++ b/app/media/Installation_base/Gitlab/docker-compose.yaml @@ -0,0 +1,31 @@ +# sudo mkdir -p /srv/gitlab +# export GITLAB_HOME=/srv/gitlab + +version: '3.6' +services: + gitlab: + image: gitlab/gitlab-ee:-ee.0 + container_name: gitlab + restart: always + hostname: 'gitlab.example.com' + environment: + GITLAB_OMNIBUS_CONFIG: | + # Add any other gitlab.rb configuration here, each on its own line + external_url 'https://gitlab.example.com' + + # you can also use custom HTTP and SSH port. if you you want to do that, follow the below syntax + + # external_url 'http://gitlab.example.com:8929' + # gitlab_rails['gitlab_shell_ssh_port'] = 2424 + + ports: + # - '8929:8929' # Custom HTTP Port + # - '2424:22' # Custom SSH Port + - '80:80' + - '443:443' + - '22:22' + volumes: + - '$GITLAB_HOME/config:/etc/gitlab' + - '$GITLAB_HOME/logs:/var/log/gitlab' + - '$GITLAB_HOME/data:/var/opt/gitlab' + shm_size: '256m' \ No newline at end of file diff --git a/app/media/Installation_base/Jenkins/RHEL.sh b/app/media/Installation_base/Jenkins/RHEL.sh new file mode 100644 index 00000000..5ee7896f --- /dev/null +++ b/app/media/Installation_base/Jenkins/RHEL.sh @@ -0,0 +1,9 @@ +sudo wget -O /etc/yum.repos.d/jenkins.repo \ + https://pkg.jenkins.io/redhat-stable/jenkins.repo +sudo rpm --import https://pkg.jenkins.io/redhat-stable/jenkins.io-2023.key +sudo yum upgrade -y +# Add required dependencies for the jenkins package +sudo yum install -y fontconfig java-17-openjdk +sudo yum install -y jenkins +sudo systemctl daemon-reload +sudo systemctl enable --now jenkins \ No newline at end of file diff --git a/app/media/Installation_base/Jenkins/docker-compose.yml b/app/media/Installation_base/Jenkins/docker-compose.yml new file mode 100644 index 00000000..efc50a8b --- /dev/null +++ b/app/media/Installation_base/Jenkins/docker-compose.yml @@ -0,0 +1,15 @@ +version: '3.8' +services: + jenkins: + image: jenkins/jenkins:lts + privileged: true + user: root + ports: + - 8080:8080 + - 50000:50000 + container_name: jenkins + volumes: + - /home/${myname}/jenkins_compose/jenkins_configuration:/var/jenkins_home + - /var/run/docker.sock:/var/run/docker.sock + +# Replace "/home/${myname}/jenkins_compose/jenkins_configuration" with the path you want to use to store your jenkins data \ No newline at end of file diff --git a/app/media/Installation_base/Jenkins/fedora.sh b/app/media/Installation_base/Jenkins/fedora.sh new file mode 100644 index 00000000..0f2a543e --- /dev/null +++ b/app/media/Installation_base/Jenkins/fedora.sh @@ -0,0 +1,9 @@ +sudo wget -O /etc/yum.repos.d/jenkins.repo \ + https://pkg.jenkins.io/redhat-stable/jenkins.repo +sudo rpm --import https://pkg.jenkins.io/redhat-stable/jenkins.io-2023.key +sudo dnf upgrade -y +# Add required dependencies for the jenkins package +sudo dnf install -y fontconfig java-17-openjdk +sudo dnf install -y jenkins +sudo systemctl daemon-reload +sudo systemctl enable --now jenkins \ No newline at end of file diff --git a/app/media/Installation_base/Jenkins/ubuntu.sh b/app/media/Installation_base/Jenkins/ubuntu.sh new file mode 100644 index 00000000..5ff50a32 --- /dev/null +++ b/app/media/Installation_base/Jenkins/ubuntu.sh @@ -0,0 +1,12 @@ +sudo apt update -y +sudo apt install -y fontconfig openjdk-17-jre + + +sudo wget -O /usr/share/keyrings/jenkins-keyring.asc \ + https://pkg.jenkins.io/debian-stable/jenkins.io-2023.key +echo "deb [signed-by=/usr/share/keyrings/jenkins-keyring.asc]" \ + https://pkg.jenkins.io/debian-stable binary/ | sudo tee \ + /etc/apt/sources.list.d/jenkins.list > /dev/null +sudo apt-get update -y +sudo apt-get install -y jenkins +sudo systemctl enable --now jenkins \ No newline at end of file diff --git a/app/media/MyBash/bash.sh b/app/media/MyBash/bash.sh index b223db9f..5ff50a32 100644 --- a/app/media/MyBash/bash.sh +++ b/app/media/MyBash/bash.sh @@ -1,16 +1,12 @@ -#!/bin/bash -sudo apt-get update -y -sudo apt-get install ca-certificates curl -y -sudo install -m 0755 -d /etc/apt/keyrings -sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc -sudo chmod a+r /etc/apt/keyrings/docker.asc - -# Add the repository to Apt sources: -echo \ - "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \ - $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \ - sudo tee /etc/apt/sources.list.d/docker.list > /dev/null -sudo apt-get update -y +sudo apt update -y +sudo apt install -y fontconfig openjdk-17-jre -sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin \ No newline at end of file +sudo wget -O /usr/share/keyrings/jenkins-keyring.asc \ + https://pkg.jenkins.io/debian-stable/jenkins.io-2023.key +echo "deb [signed-by=/usr/share/keyrings/jenkins-keyring.asc]" \ + https://pkg.jenkins.io/debian-stable binary/ | sudo tee \ + /etc/apt/sources.list.d/jenkins.list > /dev/null +sudo apt-get update -y +sudo apt-get install -y jenkins +sudo systemctl enable --now jenkins \ No newline at end of file diff --git a/app/media/MyCompose/docker-compose.yaml b/app/media/MyCompose/docker-compose.yaml index e69de29b..fe15d044 100644 --- a/app/media/MyCompose/docker-compose.yaml +++ b/app/media/MyCompose/docker-compose.yaml @@ -0,0 +1,31 @@ +# sudo mkdir -p /srv/gitlab +# export GITLAB_HOME=/srv/gitlab + +version: '3.6' +services: + gitlab: + image: gitlab/gitlab-ee:-ee.0 + container_name: gitlab + restart: always + hostname: 'gitlab.example.com' + environment: + GITLAB_OMNIBUS_CONFIG: | + # Add any other gitlab.rb configuration here, each on its own line + external_url 'https://gitlab.example.com' + + # you can also use custom HTTP and SSH port. if you you want to do that, follow the below syntax + + # external_url 'http://gitlab.example.com:8929' + # gitlab_rails['gitlab_shell_ssh_port'] = 2424 + + ports: + # - '8929:8929' # Custom HTTP Port + # - '2424:22' # Custom SSH Port + - '80:80' + - '443:443' + - '22:22' + volumes: + - '$GITLAB_HOME/config:/etc/gitlab' + - '$GITLAB_HOME/logs:/var/log/gitlab' + - '$GITLAB_HOME/data:/var/opt/gitlab' + shm_size: '256m' \ No newline at end of file diff --git a/app/models/__init__.py b/app/models/__init__.py index 9f97ab17..a529a87f 100644 --- a/app/models/__init__.py +++ b/app/models/__init__.py @@ -4,4 +4,6 @@ from .ansible_models import * from .jcasc import * from .compose_models import * -from .docker_installation_models import * \ No newline at end of file +from .docker_installation_models import * +from .jenkins import * +from .gitlab_models import * \ No newline at end of file diff --git a/app/models/gitlab_models.py b/app/models/gitlab_models.py new file mode 100644 index 00000000..44bbe939 --- /dev/null +++ b/app/models/gitlab_models.py @@ -0,0 +1,17 @@ +from typing import List, Optional +from pydantic import BaseModel, validator, ValidationError + + + +class GitLabInstallation(BaseModel): + + + environment:str = 'Docker' + + + @validator("environment") + def validator_environment(cls, value): + env = ['Docker'] + if value not in env: + raise ValueError(f"your selected Environemnt must be in {env}") + return value \ No newline at end of file diff --git a/app/models/jenkins.py b/app/models/jenkins.py new file mode 100644 index 00000000..e84ca17e --- /dev/null +++ b/app/models/jenkins.py @@ -0,0 +1,24 @@ +from typing import List, Optional +from pydantic import BaseModel, validator, ValidationError + + + +class JenkinsInstallation(BaseModel): + + os: str = 'Ubuntu' + + environment:str = 'Linux' + + @validator("os") + def validator_os(cls, value): + valid_oss = ['Ubuntu','Fedora','RHEL'] + if value not in valid_oss: + raise ValueError(f"your selected OS must be in {valid_oss}") + return value + + @validator("environment") + def validator_environment(cls, value): + env = ['Linux','Docker'] + if value not in env: + raise ValueError(f"your selected Environemnt must be in {env}") + return value \ No newline at end of file diff --git a/app/routes/docker.py b/app/routes/docker.py index 04b831aa..913a6ded 100644 --- a/app/routes/docker.py +++ b/app/routes/docker.py @@ -13,6 +13,7 @@ async def docker_compose_template(request:DockerCompose) -> Output: return Output(output='output') + @app.post("/api/docker/installation") async def docker_installation(request:DockerInstallationInput) -> Output: diff --git a/app/routes/gitlab.py b/app/routes/gitlab.py new file mode 100644 index 00000000..ec8338bf --- /dev/null +++ b/app/routes/gitlab.py @@ -0,0 +1,14 @@ +from app.app_instance import app +from app.models import (GitLabInstallation,Output) +from app.template_generators.gitlab.installation import select_install_gitlab +import os + + + + +@app.post("/api/gitlab/installation") +async def gitlab_installation(request:GitLabInstallation) -> Output: + + select_install_gitlab(request) + + return Output(output='output') \ No newline at end of file diff --git a/app/routes/jenkins.py b/app/routes/jenkins.py new file mode 100644 index 00000000..5f1aa788 --- /dev/null +++ b/app/routes/jenkins.py @@ -0,0 +1,14 @@ +from app.app_instance import app +from app.models import (DockerCompose,JenkinsInstallation,Output) +from app.template_generators.jenkins.installation import select_install_jenkins +import os + + + + +@app.post("/api/jenkins/installation") +async def jenkins_installation(request:JenkinsInstallation) -> Output: + + select_install_jenkins(request) + + return Output(output='output') \ No newline at end of file diff --git a/app/template_generators/gitlab/installation.py b/app/template_generators/gitlab/installation.py new file mode 100644 index 00000000..a50c333e --- /dev/null +++ b/app/template_generators/gitlab/installation.py @@ -0,0 +1,35 @@ +import os +import shutil + + +def create_directory(folder:str,filename:str): + + dir = f"app/media/{folder}" + + + if not os.path.exists(dir): + os.makedirs(dir) + os.path.join(dir, filename) + + +def select_install_gitlab(input): + + create_directory("MyCompose","docker-compose.yaml") + + + match input.environment: + + case "Docker": + source = 'app/media/Installation_base/Gitlab/docker-compose.yaml' + dest = 'app/media/MyCompose/docker-compose.yaml' + shutil.copyfile(source, dest) + + + case _: + raise ValueError() + + + + + + \ No newline at end of file diff --git a/app/template_generators/jenkins/installation.py b/app/template_generators/jenkins/installation.py new file mode 100644 index 00000000..4177124e --- /dev/null +++ b/app/template_generators/jenkins/installation.py @@ -0,0 +1,55 @@ +import os +import shutil + + +def create_directory(folder:str,filename:str): + + dir = f"app/media/{folder}" + + + if not os.path.exists(dir): + os.makedirs(dir) + os.path.join(dir, filename) + + + +def select_install_jenkins(input): + + create_directory("MyBash",'bash.sh') + create_directory("MyCompose",'docker-compose.yaml') + + if input.environment == 'Docker': + + source = 'app/media/Installation_base/Jenkins/docker-compose.yml' + dest = 'app/media/MyCompose/docker-compose.yaml' + shutil.copyfile(source, dest) + + else: + + match input.os: + + + case "Ubuntu": + source = 'app/media/Installation_base/Jenkins/ubuntu.sh' + dest = 'app/media/MyBash/bash.sh' + shutil.copyfile(source, dest) + + case "Fedora": + source = 'app/media/Installation_base/Jenkins/fedora.sh' + dest = 'app/media/MyBash/bash.sh' + shutil.copyfile(source, dest) + + + case "RHEL": + source = 'app/media/Installation_base/Jenkins/RHEL.sh' + dest = 'app/media/MyBash/bash.sh' + shutil.copyfile(source, dest) + + case _: + raise ValueError() + + + + + + \ No newline at end of file diff --git a/app/template_generators/terraform/Installation/main.py b/app/template_generators/terraform/Installation/main.py index ffdb14e1..c4c479e4 100644 --- a/app/template_generators/terraform/Installation/main.py +++ b/app/template_generators/terraform/Installation/main.py @@ -2,19 +2,19 @@ import shutil -def create_MyBash_directory(): +def create_directory(folder:str,filename:str): - dir = 'app/media/MyBash' + dir = f"app/media/{folder}" if not os.path.exists(dir): os.makedirs(dir) - os.path.join(dir, 'bash.sh') + os.path.join(dir, filename) def select_install(input): - create_MyBash_directory() + create_directory("MyBash","bash.sh") match input.os: From fddb77c6f1aada3c7e6cee0b9a98e6d178973783 Mon Sep 17 00:00:00 2001 From: Abolfazl Andalib <79583121+abolfazl8131@users.noreply.github.com> Date: Mon, 9 Dec 2024 13:55:26 +0330 Subject: [PATCH 16/22] Update unit-test.yml --- .github/workflows/unit-test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index 4145ba88..529e1fdb 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -13,6 +13,7 @@ jobs: - name: Checkout code uses: actions/checkout@v4 with: + submodules: true ref: ${{ github.event.pull_request.head.ref }} repository: ${{ github.event.pull_request.head.repo.full_name }} From 55deb590c58da7e2f27cfc9bafb9c983840a1aaf Mon Sep 17 00:00:00 2001 From: abolfazl1381 Date: Tue, 10 Dec 2024 14:14:19 +0330 Subject: [PATCH 17/22] fix(install): fix jenkins and gitlab --- app/media/Installation_base/Gitlab/docker-compose.yaml | 2 +- app/media/Installation_base/Jenkins/RHEL.sh | 1 + app/media/Installation_base/Jenkins/fedora.sh | 1 + app/media/Installation_base/Jenkins/ubuntu.sh | 1 + 4 files changed, 4 insertions(+), 1 deletion(-) diff --git a/app/media/Installation_base/Gitlab/docker-compose.yaml b/app/media/Installation_base/Gitlab/docker-compose.yaml index fe15d044..2c3e366a 100644 --- a/app/media/Installation_base/Gitlab/docker-compose.yaml +++ b/app/media/Installation_base/Gitlab/docker-compose.yaml @@ -4,7 +4,7 @@ version: '3.6' services: gitlab: - image: gitlab/gitlab-ee:-ee.0 + image: gitlab/gitlab-ce:-ce.0 container_name: gitlab restart: always hostname: 'gitlab.example.com' diff --git a/app/media/Installation_base/Jenkins/RHEL.sh b/app/media/Installation_base/Jenkins/RHEL.sh index 5ee7896f..5c324160 100644 --- a/app/media/Installation_base/Jenkins/RHEL.sh +++ b/app/media/Installation_base/Jenkins/RHEL.sh @@ -1,3 +1,4 @@ +#!/bin/bash sudo wget -O /etc/yum.repos.d/jenkins.repo \ https://pkg.jenkins.io/redhat-stable/jenkins.repo sudo rpm --import https://pkg.jenkins.io/redhat-stable/jenkins.io-2023.key diff --git a/app/media/Installation_base/Jenkins/fedora.sh b/app/media/Installation_base/Jenkins/fedora.sh index 0f2a543e..aeaa6d7b 100644 --- a/app/media/Installation_base/Jenkins/fedora.sh +++ b/app/media/Installation_base/Jenkins/fedora.sh @@ -1,3 +1,4 @@ +#!/bin/bash sudo wget -O /etc/yum.repos.d/jenkins.repo \ https://pkg.jenkins.io/redhat-stable/jenkins.repo sudo rpm --import https://pkg.jenkins.io/redhat-stable/jenkins.io-2023.key diff --git a/app/media/Installation_base/Jenkins/ubuntu.sh b/app/media/Installation_base/Jenkins/ubuntu.sh index 5ff50a32..fcb3d783 100644 --- a/app/media/Installation_base/Jenkins/ubuntu.sh +++ b/app/media/Installation_base/Jenkins/ubuntu.sh @@ -1,3 +1,4 @@ +#!/bin/bash sudo apt update -y sudo apt install -y fontconfig openjdk-17-jre From 81895d21d2ddc93974de9b27cba0b9730209bea0 Mon Sep 17 00:00:00 2001 From: abolfazl1381 Date: Wed, 11 Dec 2024 17:06:55 +0330 Subject: [PATCH 18/22] fix(crawler): fix the crowler to crawl 2 aws urls --- admin-panel | 2 +- crawl/content_parser.py | 105 ------------ ...Amazon EC2 instance types - Amazon EC2.txt | 79 +++++++++ ...on EC2? - Amazon Elastic Compute Cloud.txt | 151 ++++++++++++++++++ crawl/main.py | 131 +++++---------- crawl/readme.md | 39 ----- crawl/urls.csv | 1 - 7 files changed, 272 insertions(+), 236 deletions(-) delete mode 100644 crawl/content_parser.py create mode 100644 crawl/crawled_data/Amazon EC2 instance types - Amazon EC2.txt create mode 100644 crawl/crawled_data/What is Amazon EC2? - Amazon Elastic Compute Cloud.txt delete mode 100644 crawl/readme.md delete mode 100644 crawl/urls.csv diff --git a/admin-panel b/admin-panel index 5b9c0c12..bfa06012 160000 --- a/admin-panel +++ b/admin-panel @@ -1 +1 @@ -Subproject commit 5b9c0c123018e42b185681bb955c7a8b48b6b7f8 +Subproject commit bfa06012cc943bdb1a59fde5fe235be06840005d diff --git a/crawl/content_parser.py b/crawl/content_parser.py deleted file mode 100644 index 9e03e97c..00000000 --- a/crawl/content_parser.py +++ /dev/null @@ -1,105 +0,0 @@ -import requests -from bs4 import BeautifulSoup -from requests.adapters import HTTPAdapter -from requests.packages.urllib3.util.retry import Retry - -class WebContentParser: - def __init__(self, url): - self.url = url - self.headers = { - 'User-Agent': ( - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) ' - 'AppleWebKit/537.36 (KHTML, like Gecko) ' - 'Chrome/50.0.2661.102 Safari/537.36' - ) - } - self.session = self._initialize_session() - self.main_response = None - self.all_page_data = [] - - def _initialize_session(self): - """Set up the session with retry strategy.""" - retry_strategy = Retry( - total=5, - backoff_factor=8, - ) - adapter = HTTPAdapter(max_retries=retry_strategy) - adapter.max_retries.respect_retry_after_header = False - - session = requests.Session() - session.mount("https://", adapter) - session.mount("http://", adapter) - return session - - def fetch_content(self): - """Fetch the main content from the URL.""" - try: - self.main_response = self.session.get( - self.url, verify=False, timeout=30, headers=self.headers - ) - print(f'URL fetched: {self.url}') - return self.main_response - except requests.RequestException as e: - print(f"Failed to fetch the URL: {e}") - return None - - def parse_content(self): - """Parse the fetched HTML content.""" - if not self.main_response: - print("No response available to parse.") - return [] - - main_soup = BeautifulSoup(self.main_response.content, 'html.parser') - datas = main_soup.find('main', {'id': 'main'}) - if not datas: - print("No 'main' element found.") - return [] - - all_tag = datas.find_all(['h1', 'h2', 'h3', 'p', 'blockquote', 'ul']) - each_title_data = {} - - for tag in all_tag: - if tag.name in ['h1', 'h2']: - if each_title_data: - self.all_page_data.append(each_title_data) - each_title_data = {} - each_title_data['metadata'] = tag.text.strip() - - elif tag.name == 'h3': - if tag.text.strip() == 'Resources': - each_title_data[tag.text.strip()] = '' - else: - if each_title_data: - self.all_page_data.append(each_title_data) - each_title_data = {} - each_title_data['metadata'] = tag.text.strip() - - elif tag.name in ['p', 'blockquote']: - num = len(each_title_data) - key = f'content {num}' - if tag.text.strip(): - each_title_data[key] = tag.text.strip() - - elif tag.name == 'ul': - text = ' '.join( - li.text.strip() - for li in tag.find_all('li', {'class': 'mdx-lists_listItem__nkqhg'}) - ) - if 'Resources' in each_title_data: - each_title_data['Resources'] = text - else: - num = len(each_title_data) - key = f'content {num}' - if text: - each_title_data[key] = text - - if each_title_data: - self.all_page_data.append(each_title_data) - - return self.all_page_data - - def get_data(self): - """Main method to fetch and parse content.""" - self.fetch_content() - return self.parse_content() - diff --git a/crawl/crawled_data/Amazon EC2 instance types - Amazon EC2.txt b/crawl/crawled_data/Amazon EC2 instance types - Amazon EC2.txt new file mode 100644 index 00000000..861c4cda --- /dev/null +++ b/crawl/crawled_data/Amazon EC2 instance types - Amazon EC2.txt @@ -0,0 +1,79 @@ +Title: Amazon EC2 instance types - Amazon EC2 + +When you launch an EC2 instance, the instance type that you specify + determines the hardware of the host computer used for your instance. Each instance type + offers different compute, memory, and storage capabilities, and is grouped in an instance + family based on these capabilities. Select an instance type based on the requirements of the + application or software that you plan to run on your instance. +Amazon EC2 dedicates some resources of the host computer, such as CPU, memory, and instance + storage, to a particular instance. Amazon EC2 shares other resources of the host computer, such + as the network and the disk subsystem, among instances. If each instance on a host computer + tries to use as much of one of these shared resources as possible, each receives an equal + share of that resource. However, when a resource is underused, an instance can consume a + higher share of that resource while it's available. +Each instance type provides higher or lower minimum performance from a shared resource. + For example, instance types with high I/O performance have a larger allocation of shared resources. + Allocating a larger share of shared resources also reduces the variance of I/O performance. + For most applications, moderate I/O performance is more than enough. However, for + applications that require greater or more consistent I/O performance, consider + an instance type with higher I/O performance. +Current generation instances +Previous generation instances +Amazon EC2 instance type naming conventions +Amazon EC2 instance type specifications +Instances built on the AWS Nitro System +Amazon EC2 instance type quotas +For the best performance, we recommend that you use the following instance types + when you launch new instances. For more information, see Amazon EC2 Instance Types. +General purpose: M5 | M5a | M5ad | M5d | M5dn | M5n | M5zn | M6a | M6g | M6gd | M6i | M6id | M6idn | M6in | M7a | M7g | M7gd | M7i | M7i-flex | M8g | Mac1 | Mac2 | Mac2-m1ultra | Mac2-m2 | Mac2-m2pro | T2 | T3 | T3a | T4g +Compute optimized: C5 | C5a | C5ad | C5d | C5n | C6a | C6g | C6gd | C6gn | C6i | C6id | C6in | C7a | C7g | C7gd | C7gn | C7i | C7i-flex | C8g +Memory optimized: R5 | R5a | R5ad | R5b | R5d | R5dn | R5n | R6a | R6g | R6gd | R6i | R6idn | R6in | R6id | R7a | R7g | R7gd | R7i | R7iz | R8g | U-3tb1 | U-6tb1 | U-9tb1 | U-12tb1 | U-18tb1 | U-24tb1 | U7i-6tb | U7i-8tb | U7i-12tb | U7in-16tb | U7in-24tb | U7in-32tb | X1 | X1e | X2gd | X2idn | X2iedn | X2iezn | X8g | z1d +Storage optimized: D2 | D3 | D3en | H1 | I3 | I3en | I4g | I4i | I7ie | I8g | Im4gn | Is4gen +Accelerated computing: DL1 | DL2q | F1 | G4ad | G4dn | G5 | G5g | G6 | G6e | Gr6 | Inf1 | Inf2 | P2 | P3 | P3dn | P4d | P4de | P5 | P5e | P5en | Trn1 | Trn1n | Trn2 | Trn2u | VT1 +High-performance computing: Hpc6a | Hpc6id | Hpc7a | Hpc7g +Amazon Web Services offers previous generation instance types for users who have optimized their + applications around them and have yet to upgrade. We encourage you to use current generation + instance types to get the best performance, but we continue to support the following previous + generation instance types. For more information about which current + generation instance type would be a suitable upgrade, see + Previous Generation Instances. +General purpose: A1 | M1 | M2 | M3 | M4 | T1 +Compute optimized: C1 | C3 | C4 +Memory optimized: R3 | R4 +Storage optimized: I2 +Accelerated computing: G3 +Fixed performance instances provide fixed CPU resources. These instances can + deliver and sustain full CPU performance at any time, and for as long as a workload + needs it. If you need consistently high CPU performance for applications such as + video encoding, high volume websites, or HPC applications, we recommend that you use + fixed performance instances. +Burstable performance (T) instances provide a baseline level of CPU + performance with the ability to burst above the baseline. The baseline CPU is + designed to meet the needs of the majority of general purpose workloads, such as + large-scale micro-services, web servers, small and medium databases, data logging, + code repositories, virtual desktops, and development and test environments. +The baseline utilization and ability to burst are governed by CPU credits. Each + burstable performance instance continuously earns credits when it stays below the CPU + baseline, and continuously spends credits when it bursts above the baseline. For more + information, see Burstable + performance instances in the Amazon EC2 User Guide. +M7i-flex and C7i-flex instances offer a balance of compute, memory, and network + resources, and they provide the most cost-effective way to run a broad spectrum of + general purpose applications. These instances provide reliable CPU resources to + deliver a baseline CPU performance of 40 percent, which is designed to meet the + compute requirements for a majority of general purpose workloads. When more + performance is needed, these instances provide the ability to exceed the baseline + CPU performance and deliver up to 100 percent CPU performance for 95 percent of the + time over a 24-hour window. +M7i-flex and C7i-flex instances running at a high CPU utilization that is consistently + above the baseline for long periods of time might see a gradual reduction in the maximum + burst CPU throughput. For more information, see M7i-flex instances and C7i-flex instances. +For pricing information, see Amazon EC2 Pricing. + Javascript is disabled or is unavailable in your browser. +To use the Amazon Web Services Documentation, Javascript must be enabled. Please refer to your browser's Help pages for instructions. +Thanks for letting us know we're doing a good job! +If you've got a moment, please tell us what we did right so we can do more of it. + +Thanks for letting us know this page needs work. We're sorry we let you down. +If you've got a moment, please tell us how we can make the documentation better. + diff --git a/crawl/crawled_data/What is Amazon EC2? - Amazon Elastic Compute Cloud.txt b/crawl/crawled_data/What is Amazon EC2? - Amazon Elastic Compute Cloud.txt new file mode 100644 index 00000000..d0e78fd3 --- /dev/null +++ b/crawl/crawled_data/What is Amazon EC2? - Amazon Elastic Compute Cloud.txt @@ -0,0 +1,151 @@ +Title: What is Amazon EC2? - Amazon Elastic Compute Cloud + +Amazon Elastic Compute Cloud (Amazon EC2) provides on-demand, scalable computing capacity in the Amazon Web + Services (AWS) Cloud. Using Amazon EC2 reduces hardware costs so you can develop and deploy + applications faster. You can use Amazon EC2 to launch as many or as few virtual servers as you + need, configure security and networking, and manage storage. You can add capacity (scale up) + to handle compute-heavy tasks, such as monthly or yearly processes, or spikes in website + traffic. When usage decreases, you can reduce capacity (scale down) again. +An EC2 instance is a virtual server in the AWS Cloud. When you launch an EC2 instance, + the instance type that you specify determines the hardware available to your instance. + Each instance type offers a different balance of compute, memory, network, and storage + resources. For more information, see the Amazon EC2 Instance Types Guide. +Amazon EC2 provides the following high-level features: +Virtual servers. +Preconfigured templates for your instances that package the components you + need for your server (including the operating system and additional + software). +Various configurations of CPU, memory, storage, networking capacity, and + graphics hardware for your instances. +Persistent storage volumes for your data using Amazon Elastic Block Store (Amazon EBS). +Storage volumes for temporary data that is deleted when you stop, + hibernate, or terminate your instance. +Secure login information for your instances. AWS stores the public key + and you store the private key in a secure place. +A virtual firewall that allows you to specify the protocols, ports, and + source IP ranges that can reach your instances, and the destination IP + ranges to which your instances can connect. +Amazon EC2 supports the processing, storage, and transmission +of credit card data by a merchant or service provider, and has been +validated as being compliant with Payment Card Industry (PCI) Data Security Standard (DSS). +For more information about PCI DSS, including how to request a copy of the AWS PCI Compliance Package, +see PCI DSS Level 1. + +You can use other AWS services with the instances that you deploy using Amazon EC2. +Helps ensure you have the correct number of Amazon EC2 instances available to + handle the load for your application. +Automate backing up your Amazon EC2 instances and the Amazon EBS volumes attached to + them. +Monitor your instances and Amazon EBS volumes. +Automatically distribute incoming application traffic across multiple + instances. +Detect potentially unauthorized or malicious use of your EC2 instances. +Automate the creation, management, and deployment of customized, secure, and + up-to-date server images. +Size, configure, and deploy AWS resources for third-party applications + without having to manually identify and provision individual AWS + resources. +Perform operations at scale on EC2 instances with this secure end-to-end + management solution. +You can launch instances using another AWS compute service instead of using Amazon EC2. +Build websites or web applications using Amazon Lightsail, a cloud platform + that provides the resources that you need to deploy your project quickly, for + a low, predictable monthly price. To compare Amazon EC2 and Lightsail, see + Amazon Lightsail or Amazon EC2. +Deploy, manage, and scale containerized applications on a cluster of EC2 + instances. For more information, see Choosing an AWS container service. +Run your Kubernetes applications on AWS. For more information, see + Choosing an AWS container service. +You can create and manage your Amazon EC2 instances using the following interfaces: +A simple web interface to create and manage Amazon EC2 instances and resources. + If you've signed up for an AWS account, you can access the Amazon EC2 console + by signing into the AWS Management Console and selecting EC2 from + the console home page. +Enables you to interact with AWS services using commands in your command-line shell. It + is supported on Windows, Mac, and Linux. For more information about the + AWS CLI , see AWS Command Line Interface User Guide. You can find the Amazon EC2 commands in the AWS CLI Command Reference. +Amazon EC2 supports creating resources using AWS CloudFormation. You create a template, in JSON or YAML + format, that describes your AWS resources, and AWS CloudFormation provisions and + configures those resources for you. You can reuse your CloudFormation + templates to provision the same resources multiple times, whether in the + same Region and account or in multiple Regions and accounts. For more + information about supported resource types and properties for Amazon EC2, see + EC2 resource type + reference in the AWS CloudFormation User Guide. +If you prefer to build applications using language-specific APIs instead + of submitting a request over HTTP or HTTPS, AWS provides libraries, sample + code, tutorials, and other resources for software developers. These + libraries provide basic functions that automate tasks such as + cryptographically signing your requests, retrying requests, and handling + error responses, making it easier for you to get started. For more + information, see + Tools to Build + on AWS. +A set of PowerShell modules that are built on the functionality exposed by + the AWS SDK for .NET. The Tools for PowerShell enable you to script operations on your AWS + resources from the PowerShell command line. To get started, see the + AWS Tools for Windows PowerShell User Guide. You can find the cmdlets for Amazon EC2, in the AWS Tools for PowerShell Cmdlet Reference. +Amazon EC2 provides a Query API. These requests are HTTP or HTTPS requests that + use the HTTP verbs GET or POST and a Query parameter named + Action. For more information about the API actions for + Amazon EC2, see Actions in the + Amazon EC2 API Reference. +Amazon EC2 provides the following pricing options: +You can get started with Amazon EC2 for free. To explore the Free Tier options, + see AWS Free Tier. +Pay for the instances that you use by the second, with a minimum of 60 + seconds, with no long-term commitments or upfront payments. +You can reduce your Amazon EC2 costs by making a commitment to a consistent + amount of usage, in USD per hour, for a term of 1 or 3 years. +You can reduce your Amazon EC2 costs by making a commitment to a specific + instance configuration, including instance type and Region, for a term of 1 + or 3 years. +Request unused EC2 instances, which can reduce your Amazon EC2 costs + significantly. +Reduce costs by using a physical EC2 server that is fully dedicated for + your use, either On-Demand or as part of a Savings Plan. You can use your + existing server-bound software licenses and get help meeting compliance + requirements. +Reserve compute capacity for your EC2 instances in a specific Availability + Zone for any duration of time. +Removes the cost of unused minutes and seconds from your bill. +For a complete list of charges and prices for Amazon EC2 and more information about the purchase + models, see Amazon EC2 pricing. +To create estimates for your AWS use cases, use the AWS Pricing Calculator. +To estimate the cost of transforming Microsoft + workloads to a modern architecture that uses open source and + cloud-native services deployed on AWS, use the AWS + Modernization Calculator for Microsoft Workloads. +To see your bill, go to the Billing and Cost Management + Dashboard in the AWS Billing and Cost Management + console. Your bill contains links to usage reports that provide details + about your bill. To learn more about AWS account billing, see AWS Billing and Cost Management User + Guide. +If you have questions concerning AWS billing, accounts, and events, contact AWS Support. +To calculate the cost of a sample provisioned + environment, see Cloud Economics + Center. When calculating the cost of a provisioned + environment, remember to include incidental costs such as snapshot storage for EBS + volumes. +You can optimize the cost, security, and performance of your AWS environment + using AWS Trusted Advisor. +You can use AWS Cost Explorer to analyze the cost and usage of your EC2 instances. You can view + data up to the last 13 months, and forecast how much you are likely to spend for the next + 12 months. For more information, see + Analyzing your costs with + AWS Cost Explorer in the AWS Cost Management User Guide. +Amazon EC2 features +AWS re:Post +AWS Skill Builder +AWS Support +Hands-on Tutorials +Web Hosting +Windows on AWS + Javascript is disabled or is unavailable in your browser. +To use the Amazon Web Services Documentation, Javascript must be enabled. Please refer to your browser's Help pages for instructions. +Thanks for letting us know we're doing a good job! +If you've got a moment, please tell us what we did right so we can do more of it. + +Thanks for letting us know this page needs work. We're sorry we let you down. +If you've got a moment, please tell us how we can make the documentation better. + diff --git a/crawl/main.py b/crawl/main.py index f86e632e..3a4621e3 100644 --- a/crawl/main.py +++ b/crawl/main.py @@ -1,92 +1,43 @@ - -import argparse -import csv -import logging import requests from bs4 import BeautifulSoup -from requests.adapters import HTTPAdapter -from requests.packages.urllib3.util.retry import Retry -from content_parser import WebContentParser - - -def setup_logging(): - logging.basicConfig( - level=logging.INFO, - format='%(asctime)s - %(levelname)s - %(message)s', - handlers=[logging.StreamHandler()] - ) - - -def setup_http_session(): - retry_strategy = Retry( - total=5, - backoff_factor=8, - ) - adapter = HTTPAdapter(max_retries=retry_strategy) - adapter.max_retries.respect_retry_after_header = False - session = requests.Session() - session.mount("https://", adapter) - session.mount("http://", adapter) - return session - - -def process_urls(file_path, save_result): - http = setup_http_session() - headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36'} - - with open(file_path, 'r') as file: - csv_reader = csv.reader(file) - for row in csv_reader: - if row: # Check if the row is not empty - main_url = row[0] - try: - main_response = http.get(main_url, verify=False, timeout=30, headers=headers) - logging.info(f'Fetched URL: {main_url}') - except requests.RequestException as e: - logging.error(f"Failed to fetch URL {main_url}: {e}") - continue - - main_soup = BeautifulSoup(main_response.content, 'html.parser') - products = main_soup.find('div', {'class': 'marketing-content_root__DE3hU'}).find_all('div', {'class': 'card-grid-block_root__yDdm_'}) - logging.info(f'Found {len(products)} products on page: {main_url}') - all_data = [] - for product in products: - # Get org title - title = product.find('h2').text - sub_content_link=[] - all_sub_title = product.find_all('li') - for res in all_sub_title: - sub_part_content = {} - sub_part_content['main_title'] = title - sub_title = res.find('span', {'class': 'card-title_text__F97Wj'}).get_text() - sub_part_content['sub_title'] = sub_title - sub_title_link = 'https://developer.hashicorp.com' + res.find('a').attrs['href'] - sub_part_content['sub_title_link'] = sub_title_link - - parser = WebContentParser(sub_title_link) - data = parser.get_data() - sub_part_content['all_data_info'] = data - - logging.info(f'Parsed content for sub-title: {sub_title}') - sub_content_link.append(sub_part_content) - all_data.append(sub_content_link) - if save_result: - # Logic to save sub_part_content goes here (e.g., writing to a file or database) - logging.info(f'Saving result for: {all_data}') - else: - print(all_data) - - -def main(): - setup_logging() - - parser = argparse.ArgumentParser(description='Process URLs from a CSV file.') - parser.add_argument('--csv_path', type=str, default='./urls.csv', help='Path to the CSV file containing URLs') - parser.add_argument('--save_result', type=bool, default=False, help='Flag to indicate if the results should be saved') - args = parser.parse_args() - - process_urls(args.csv_path, args.save_result) - - -if __name__ == '__main__': - main() +import os + +# List of URLs to crawl +urls = [ + "https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/concepts.html", + "https://docs.aws.amazon.com/ec2/latest/instancetypes/instance-types.html#current-gen-instances" +] + +# Directory to save the files +save_dir = "crawled_data" +os.makedirs(save_dir, exist_ok=True) + +def fetch_and_save(url): + try: + response = requests.get(url) + response.raise_for_status() # Check if the request was successful + + # Parse the HTML content + soup = BeautifulSoup(response.text, 'html.parser') + + # For demonstration, we are fetching the page title and all paragraphs + title = soup.title.string if soup.title else "no_title" + paragraphs = soup.find_all('p') + + # Prepare the file name + file_name = os.path.join(save_dir, f"{title}.txt") + + # Write the content to the file + with open(file_name, 'w', encoding='utf-8') as file: + file.write(f"Title: {title}\n\n") + for para in paragraphs: + file.write(para.get_text() + "\n") + + print(f"Saved content from {url} to {file_name}") + + except requests.RequestException as e: + print(f"Failed to fetch {url}: {e}") + +# Fetch and save data from each URL +for url in urls: + fetch_and_save(url) diff --git a/crawl/readme.md b/crawl/readme.md deleted file mode 100644 index 93e44d57..00000000 --- a/crawl/readme.md +++ /dev/null @@ -1,39 +0,0 @@ -# Documentation for Web Content Scraper - -## Overview -This script is designed to scrape data from a list of URLs provided in a CSV file. It fetches the content, extracts specific product information, and logs the operations performed. Optionally, the extracted content can also be saved. The script utilizes various libraries such as `requests`, `BeautifulSoup`, and `argparse` to ensure efficient and robust operation. - -## Prerequisites -Make sure the following Python packages are installed: -- `requests` -- `beautifulsoup4` -- `urllib3` - -To install the dependencies, run the following command: -```sh -pip install requests beautifulsoup4 -``` -## How to Use -Arguments -The script accepts command-line arguments that allow customization of behavior: ---csv_path: The path to the CSV file containing URLs to scrape. The default value is ./urls.csv. ---save_result: A boolean flag indicating whether to save the scraped results. The default value is False. -## Running the Script -You can run the script by using the following command: - -```sh -Copy code -python main.py --csv_path --save_result -``` -For example: -```sh -Copy code -python main.py --csv_path ./urls.csv --save_result True -``` -## CSV File Format -The CSV file should contain a list of URLs, with each URL on a new line. Here is an example: -``` -https://example.com/page1 -https://example.com/page2 -``` - diff --git a/crawl/urls.csv b/crawl/urls.csv deleted file mode 100644 index 46e1afd1..00000000 --- a/crawl/urls.csv +++ /dev/null @@ -1 +0,0 @@ -https://developer.hashicorp.com/terraform/docs \ No newline at end of file From 1818b9896349cdd17b6f67fe8734eaeba7ad4b0b Mon Sep 17 00:00:00 2001 From: abolfazl1381 Date: Fri, 13 Dec 2024 10:50:05 +0330 Subject: [PATCH 19/22] feat(aws_data_crowl): add more documents --- ... Issues and Their Solutions - Sematext.txt | 63 +++ ...erview - Amazon Simple Storage Service.txt | 225 +++++++++ ...zon S3 - Amazon Simple Storage Service.txt | 65 +++ ...for S3 - Amazon Simple Storage Service.txt | 15 + ...tpoint - Amazon Simple Storage Service.txt | 117 +++++ ...bucket - Amazon Simple Storage Service.txt | 226 +++++++++ ...ueries - Amazon Simple Storage Service.txt | 40 ++ ... rules - Amazon Simple Storage Service.txt | 115 +++++ ...tpoint - Amazon Simple Storage Service.txt | 94 ++++ ...for S3 - Amazon Simple Storage Service.txt | 15 + ...tables - Amazon Simple Storage Service.txt | 21 + ...ctions - Amazon Simple Storage Service.txt | 60 +++ ...bjects - Amazon Simple Storage Service.txt | 123 +++++ ...rmance - Amazon Simple Storage Service.txt | 20 + ...schema - Amazon Simple Storage Service.txt | 146 ++++++ ...for S3 - Amazon Simple Storage Service.txt | 346 ++++++++++++++ ...tables - Amazon Simple Storage Service.txt | 45 ++ ...s and How to Solve Them | BrowserStack.txt | 84 ++++ ...S SDKs - Amazon Simple Storage Service.txt | 28 ++ ...for S3 - Amazon Simple Storage Service.txt | 22 + ... class - Amazon Simple Storage Service.txt | 27 ++ ...zon S3 - Amazon Simple Storage Service.txt | 344 ++++++++++++++ ...for S3 - Amazon Simple Storage Service.txt | 35 ++ ...uckets - Amazon Simple Storage Service.txt | 229 ++++++++++ ...on S3? - Amazon Simple Storage Service.txt | 431 ++++++++++++++++++ crawl/main.py | 51 ++- 26 files changed, 2985 insertions(+), 2 deletions(-) create mode 100644 crawl/crawled_data/10 Biggest DevOps Challenges & Issues and Their Solutions - Sematext.txt create mode 100644 crawl/crawled_data/Buckets overview - Amazon Simple Storage Service.txt create mode 100644 crawl/crawled_data/Common bucket patterns for building applications on Amazon S3 - Amazon Simple Storage Service.txt create mode 100644 crawl/crawled_data/Configuring Storage Browser for S3 - Amazon Simple Storage Service.txt create mode 100644 crawl/crawled_data/Configuring and using Mountpoint - Amazon Simple Storage Service.txt create mode 100644 crawl/crawled_data/Creating a bucket - Amazon Simple Storage Service.txt create mode 100644 crawl/crawled_data/Example metadata table queries - Amazon Simple Storage Service.txt create mode 100644 crawl/crawled_data/General purpose bucket naming rules - Amazon Simple Storage Service.txt create mode 100644 crawl/crawled_data/Installing Mountpoint - Amazon Simple Storage Service.txt create mode 100644 crawl/crawled_data/Installing Storage Browser for S3 - Amazon Simple Storage Service.txt create mode 100644 crawl/crawled_data/Joining custom metadata with S3 metadata tables - Amazon Simple Storage Service.txt create mode 100644 crawl/crawled_data/Metadata table limitations and restrictions - Amazon Simple Storage Service.txt create mode 100644 crawl/crawled_data/Naming Amazon S3 objects - Amazon Simple Storage Service.txt create mode 100644 crawl/crawled_data/Optimizing metadata table query performance - Amazon Simple Storage Service.txt create mode 100644 crawl/crawled_data/S3 Metadata tables schema - Amazon Simple Storage Service.txt create mode 100644 crawl/crawled_data/Setting up Storage Browser for S3 - Amazon Simple Storage Service.txt create mode 100644 crawl/crawled_data/Setting up permissions for configuring metadata tables - Amazon Simple Storage Service.txt create mode 100644 crawl/crawled_data/Top DevOps Challenges and its SolutionsTop Challenges in DevOps and How to Solve Them | BrowserStack.txt create mode 100644 crawl/crawled_data/Tracking a multipart upload with the AWS SDKs - Amazon Simple Storage Service.txt create mode 100644 crawl/crawled_data/Troubleshooting Storage Browser for S3 - Amazon Simple Storage Service.txt create mode 100644 crawl/crawled_data/Uploading a directory using the high-level .NET TransferUtility class - Amazon Simple Storage Service.txt create mode 100644 crawl/crawled_data/Uploading and copying objects using multipart upload in Amazon S3 - Amazon Simple Storage Service.txt create mode 100644 crawl/crawled_data/Using Storage Browser for S3 - Amazon Simple Storage Service.txt create mode 100644 crawl/crawled_data/Virtual hosting of buckets - Amazon Simple Storage Service.txt create mode 100644 crawl/crawled_data/What is Amazon S3? - Amazon Simple Storage Service.txt diff --git a/crawl/crawled_data/10 Biggest DevOps Challenges & Issues and Their Solutions - Sematext.txt b/crawl/crawled_data/10 Biggest DevOps Challenges & Issues and Their Solutions - Sematext.txt new file mode 100644 index 00000000..691dc388 --- /dev/null +++ b/crawl/crawled_data/10 Biggest DevOps Challenges & Issues and Their Solutions - Sematext.txt @@ -0,0 +1,63 @@ +Title: 10 Biggest DevOps Challenges & Issues and Their Solutions - Sematext + +Registration is open - Live, Instructor-led Online Classes - Elasticsearch in March - Solr in April - OpenSearch in May. See all classes +Solr / Elasticsearch Experts – Search & Big Data Analytics +Close +Upcoming Solr, Elasticsearch, and OpenSearch Online Training Classes See Online Classes +Check the latest product updates +Compare Sematext with other solutions according to reviews on G2.com. +Explore all DevOps terms and definitions in our Glossary +See all Videos +See All → +Table of Contents +To enhance their continuous delivery initiative, companies have long started to switch to a DevOps culture. By adapting best practices such as CI/CD pipeline, continuous testing or continuous security, teams get faster and more creative in scaling their operations. +On the other hand, the companies that are yet to take the plunge are under pressure to comply if they want to get or maintain a competitive edge. +If you are planning to implement DevOps practices, know that even though you’ll benefit greatly from it, the adoption process is not without its fair share of issues. +We asked engineers what were the biggest DevOps challenges they encountered when implementing the model and their solutions. Here’s what they shared with us. +If you are planning to adopt DevOps practices, here are the top DevOps challenges that you must address in the process and how to overcome them: +Determining the metrics that are the most relevant and useful for a particular organization is one of the top challenges in transitioning to the DevOps model. That is because a wide range of metrics is available to measure the effectiveness and efficiency of a DevOps process, such as deployment frequency, lead time for changes, mean time to recover from failures, and defect escape rate. However, not all of these metrics may be applicable or valuable for a given organization, and identifying the metrics important to your organization can be challenging. +You could adopt a data-driven approach to identify and track DevOps metrics. This can involve using tools such as analytics platforms or dashboards to collect and visualize data on various aspects of the DevOps process. This can help organizations find the right metrics, identify the patterns and trends in their data, and focus on areas where they may be able to improve their processes or where they may be experiencing bottlenecks or other issues. +Security in DevOps ensures that security needs are adequately addressed throughout the development process, as traditional approaches to security can be a barrier to rapid development and deployment. It can be tough in organizations that have adopted a continuous delivery model, where code changes are made and deployed frequently. +You could use the DevSecOps approach to resolve this challenge. DevSecOps integrates security into the DevOps process from the outset. It involves the collaboration of development, security, and operations teams to build and deploy secure systems continuously and automatedly. +To put it in a better way, Radu Tizu, CTO of Multicode Plus, said, ‘Dev(Sec)Ops comes as a requirement in the current ecosystem of unlimited auto-scalable number of worker nodes – for consistency, trackability (and auditing) but also fast response times. ‘Data center is on fire?’ run this command and change the target region, and you get a brand new infrastructure deployment; ‘moving to a new region with strict data retention policies?’ do the same and replicate the infrastructure, ‘Black Friday is this Friday?’ DevSecOps can triple your capacity in no time.’ +To implement a DevSecOps approach, organizations can adopt tools and processes that enable developers to identify and fix security issues early in the development cycle, such as static analysis tools and automated testing frameworks. +Using microservices in a DevOps environment increases the complexity of managing a large number of independently deployable service components. This can make it more difficult to track dependencies between services, troubleshoot issues, and ensure that the overall system functions correctly. +To tackle this DevOps challenge, tools and practices that enable organizations to manage and monitor their microservices environment effectively can be adopted. This can involve using tools such as service meshes or orchestration platforms to manage communication between services and implementing monitoring and logging systems to track the health and performance of individual services. +Managing changes in a DevOps environment calls for balancing rapid deployment with the need to ensure the stability and reliability of the system. It can be difficult because changing a system introduces new risks and vulnerabilities, and the more frequent changes are made, the greater the potential for issues. +One solution to this challenge is to adopt a culture of continuous testing and monitoring, in which changes are thoroughly tested and validated before being deployed to production. It usually means implementing automated testing frameworks and using tools such as monitoring and logging platforms to track the performance and behavior of the system after changes are made. +However, there is an even bigger challenge to implementing testing frameworks and tools for monitoring. Neil Douek, Director of DevOps / Agility at Dops Digital says – “In my experience you can’t implement a culture, but rather nurture it into existence over a period of time. I always look towards fundamentals, such as branching strategy and repository hygiene, and then develop sustainable standards that can support CI/CD.” So, you need to ensure that your team members understand what to test (or log) in their code and how to make tests (or generate logs). Only after you and your team are satisfied with the fundamentals, you should introduce automated testing and log management rather than directly leapfrogging into it. +Just like the metrics, the DevOps market is filled with several DevOps tools. But, there are no standard tools that apply to all organizations because each organization may have different requirements. So, selecting and integrating the right tools into the development and deployment process is critical to achieving DevOps success within an organization. +One way to overcome this challenge is to adopt a DevOps toolchain, a set of integrated tools that work together to support the development and deployment process. A DevOps toolchain includes tools for version control, continuous integration, testing, deployment, and monitoring. You should also follow these 4 best practices for choosing DevOps tools and simplify your suite. +Cross-functional teams in a DevOps environment ensure that team members have the necessary skills and knowledge to collaborate effectively and work together. But establishing a team with the necessary skills can be challenging because DevOps requires diverse skills and expertise, including development, operations, security, and testing. +However, organizations can provide training and resources to help DevOps teams build the necessary skills and knowledge. It involves providing training on relevant tools and technologies, as well as on practices such as agile methodologies and continuous delivery. +Another solution to this issue is to adopt a culture of collaboration and ownership within the organization. According to Lewis Stevens, Senior Cloud DevOps and Infrastructure Engineer of DigiTickets – lk message, ‘The main challenge I have found trying to implement a DevOps culture is that the Developers or the Operations team struggle to work on each other’s repositories. To overcome this, I will typically focus on creating a new small project where a developer and operations engineer is assigned solely to the project and can take ownership of it.’ +While these solutions can help in creating a cross functional team, Rafal Krol, a Cloud Site Reliability Engineer at Chaos Gears also warns that change can be overwhelming for members of an organization, so the organization should introduce change gradually. He says – “If I were to name one challenge of embracing DevOps culture, I’d doubt the lack of a general agreement on the importance of learning. A cultural change takes time to ripe…” +Floor Nooitgedagt, Principal Recruitment Consultant for DevOps, Cloud & Data at Apollo Solutions argues – “…If your CIO or CTO doesn’t truly advocate DevOps, it’ll be very difficult to successfully implement a strong DevOps culture across the business. And it all starts with senior management and a clear vision.” +Organizations with clear vision can implement a center of excellence (COE) model in a DevOps environment to ensure that DevOps practices are successfully implanted. COE effectively supports and facilitates the development and deployment process across the organization. But as the COE must coordinate with multiple teams and functions and may need to juggle competing priorities and demands, this can become challenging. +One way to solve this DevOps challenge is to adopt a collaborative and consultative approach to the COE model, in which the COE works closely with teams across the organization to understand their needs and goals and provides guidance and support as needed. This ensures that the COE can effectively support the development and deployment process while respecting the autonomy and independence of individual teams. +Strict governance strives for the system’s quality and stability by imposing clear policies and procedures for development and deployment. But on the other hand, overly strict governance can become a barrier to rapid development and deployment, as it may require teams to follow rigid processes and procedures that can slow down the development and deployment process. So finding the right balance between control and necessary oversight for flexibility and agility can become a challenge of the DevOps adoption process. +One solution to this challenge is to adopt a flexible and agile approach to governance, in which policies and procedures are designed to support and enable the development and deployment process rather than hinder it. +A DevOps environment has to manage multiple environments. But, ensuring the different environments are kept in sync, and that code changes and deployments are properly tested and validated before being promoted to production can be difficult. Different environments may have different configurations and dependencies. +One way to conquer this challenge is to adopt a continuous integration and delivery (CI/CD) approach, in which code changes are automatically built, tested, and deployed through a series of automated DevOps pipeline stages. +Organizations need to ensure that DevOps environments have the budget allocation aligned with the needs and goals of the organization. However, DevOps involves a wide range of activities and tools, and it can be challenging to determine which areas are most important and should be prioritized. +To solve this DevOps challenge, you could involve stakeholders from across the organization in the budgeting process and establish clear policies and procedures for allocating resources. This ensures that budget allocation is based on the needs and goals of the organization and that resources are being used effectively to support the development and deployment process. +In the rapidly evolving landscape of microservices, the challenges of ensuring their continuous availability and optimal performance are paramount. Manual testing and deploying new functionalities across an ever-growing number of microservices can be complex and time-consuming. DevOps teams face the pressing need to replicate microservices in test environments while maintaining their stability. This is where Sematext Cloud comes into play with its array of advanced features designed to improve your DevOps processes and address potential challenges. +By seamlessly integrating Sematext agents, microservices can be monitored as they dynamically come and go, even on-the-fly. This proactive approach helps swiftly identify instability or performance issues, ensuring the reliability and efficiency of the microservices ecosystem. +Furthermore, the embrace of an active CI/CD pipeline helps both developers and operations. This collaboration facilitates rapid product patching and the introduction of new functionalities, sometimes on a daily basis. However, fully harnessing the power of CI/CD takes time and effort as both teams learn its intricacies, enhance it through plugins, and grasp its limitations. +Providing real-time monitoring and analytics capabilities, Sematext makes it easier to achieve operational performance.From monitoring microservices in real-time to refining the CI/CD pipeline, Sematext equips teams with the tools needed to enhance efficiency, minimize downtime, and deliver superior products. +Watch the video below to learn more about Sematext Cloud or start your 14-day free trial to experience the power of enhanced monitoring. + +Getting Started with Sematext Cloud +DevOps is a powerful approach to software development and deployment that can help organizations deliver value to customers faster and more reliably. However, implementing DevOps also brings with it several challenges and issues that teams must address to succeed. +Challenges like selecting and integrating the right tools, managing security risks, and coordinating across teams and functions can pressure companies trying to adopt DevOps practices. However, by understanding these challenges and adopting the right solutions, organizations can overcome these hurdles and build a successful DevOps practice that drives business value. +Start Free Trial +See all jobs +Write for us +Get tips, how-tos, and news about Elastic / ELK Stack, Observability, Solr, and Sematext Cloud news and updates. + +© Sematext Group. All rights reserved +Terms Of Service · Privacy Policy + +Apache Lucene, Apache Solr and their respective logos are trademarks of the Apache Software Foundation. +Elasticsearch, Kibana, Logstash, and Beats are trademarks of Elasticsearch BV, registered in the U.S. +and in other countries. Sematext Group, Inc. is not affiliated with Elasticsearch BV. diff --git a/crawl/crawled_data/Buckets overview - Amazon Simple Storage Service.txt b/crawl/crawled_data/Buckets overview - Amazon Simple Storage Service.txt new file mode 100644 index 00000000..1c41cddf --- /dev/null +++ b/crawl/crawled_data/Buckets overview - Amazon Simple Storage Service.txt @@ -0,0 +1,225 @@ +Title: Buckets overview - Amazon Simple Storage Service + +To upload your data (photos, videos, documents, etc.) to Amazon S3, you must first create an S3 + bucket in one of the AWS Regions. +There are two types of Amazon S3 buckets, general purpose buckets and directory buckets. Choose + the bucket type that best fits your application and performance requirements: +General purpose buckets are the original S3 + bucket type and are recommended for most use cases and access patterns. General + purpose buckets also allow objects that are stored across all storage classes, + except S3 Express One Zone. +Directory buckets use the S3 Express One Zone storage + class, which is recommended if your application is performance sensitive and + benefits from single-digit millisecond PUT and GET + latencies, see S3 Express One Zone and Working with directory buckets. +The following sections provide more information about general purpose buckets, including bucket + naming rules, quotas, and bucket configuration details. For a list of restriction and + limitations related to Amazon S3 buckets see, Bucket quotas, limitations, and restrictions. +A general purpose bucket is a container for objects stored in Amazon S3. You can store any number of + objects in a bucket and all accounts have a default bucket quota of 10,000 general purpose buckets. To see your bucket utilization, bucket quota, or request an + increase to this quota, visit the Service Quotas + console. +General purpose bucket quotas for commercial Regions can only be viewed and managed + from US East (N. Virginia). +General purpose bucket quotas for AWS GovCloud (US) can only be viewed and managed from + AWS GovCloud (US-West). +Every object is contained in a bucket. For example, if the object named + photos/puppy.jpg is stored in the + amzn-s3-demo-bucket bucket in the US West (Oregon) + Region, then it is addressable by using the URL + https://amzn-s3-demo-bucket.s3.us-west-2.amazonaws.com/photos/puppy.jpg. + For more information, see Accessing a + Bucket. +In terms of implementation, buckets and objects are AWS resources, and Amazon S3 provides + APIs for you to manage them. For example, you can create a bucket and upload objects using + the Amazon S3 API. You can also use the Amazon S3 console to perform these operations. The console + uses the Amazon S3 APIs to send requests to Amazon S3. +This section describes how to work with buckets. For information about working with + objects, see Amazon S3 objects overview. + Amazon S3 supports global buckets, which means that each bucket name must be unique across all + AWS accounts in all the AWS Regions within a partition. A partition is a grouping of + Regions. AWS currently has three partitions: aws (Standard Regions), + aws-cn (China Regions), and aws-us-gov (AWS GovCloud (US)). +After a bucket is created, the name of that bucket cannot be used by another AWS account + in the same partition until the bucket is deleted. You should not depend on specific bucket + naming conventions for availability or security verification purposes. For bucket naming + guidelines, see General purpose bucket naming rules. +Amazon S3 creates buckets in a Region that you specify. To reduce latency, minimize costs, or + address regulatory requirements, choose any AWS Region that is geographically close to + you. For example, if you reside in Europe, you might find it advantageous to create buckets + in the Europe (Ireland) or Europe (Frankfurt) Regions. For a list of Amazon S3 Regions, see + Regions and + Endpoints in the AWS General Reference. +Objects that belong to a bucket that you create in a specific AWS Region never leave + that Region, unless you explicitly transfer them to another Region. For example, objects + that are stored in the Europe (Ireland) Region never leave it. +When you build applications on Amazon S3, you can use unique general purpose buckets to separate + different datasets or workloads. Depending on your use case, there are different design + patterns and best practices for using buckets. For more information, see Common bucket patterns for building applications on + Amazon S3. +You can use your AWS account root user credentials to create a bucket and perform any other Amazon S3 + operation. However, we recommend that you do not use the root user credentials of your + AWS account to make requests, such as to create a bucket. Instead, create an AWS Identity and Access Management + (IAM) user, and grant that user full access (users by default have no permissions). +These users are referred to as administrators. You + can use the administrator user credentials, instead of the root user credentials of your + account, to interact with AWS and perform tasks, such as create a bucket, create + users, and grant them permissions. +For more information, see AWS account root user + credentials and IAM user credentials in the AWS + General Reference and Security best practices in IAM in the + IAM User Guide. +The AWS account that creates a resource owns that resource. For example, if you + create an IAM user in your AWS account and grant the user permission to create a + bucket, the user can create a bucket. But the user does not own the bucket; the + AWS account that the user belongs to owns the bucket. The user needs additional + permission from the resource owner to perform any other bucket operations. For more + information about managing permissions for your Amazon S3 resources, see Identity and Access Management for Amazon S3. +Public access is granted to buckets and objects through bucket policies, access + control lists (ACLs), or both. To help you manage public access to Amazon S3 resources, Amazon S3 + provides settings to block public access. Amazon S3 Block Public Access settings can override + ACLs and bucket policies so that you can enforce uniform limits on public access to + these resources. You can apply Block Public Access settings to individual buckets or to + all buckets in your account. +To ensure that all of your Amazon S3 buckets and objects have their public access blocked, + all four settings for Block Public Access are enabled by default when you create a new + bucket. We recommend that you turn on all four settings for Block Public Access for your + account too. These settings block all public access for all current and future + buckets. +Before applying these settings, verify that your applications will work correctly + without public access. If you require some level of public access to your buckets or + objects—for example, to host a static website, as described at Hosting a static website using Amazon S3—you can customize + the individual settings to suit your storage use cases. For more information, see Blocking public access to your Amazon S3 + storage. + However, we highly recommend keeping Block Public Access enabled. If you want to keep + all four Block Public Access settings enabled and host a static website, you can use + Amazon CloudFront origin access control (OAC). Amazon CloudFront provides the capabilities required to set + up a secure static website. Amazon S3 static websites support only HTTP endpoints. Amazon CloudFront + uses the durable storage of Amazon S3 while providing additional security headers, such as + HTTPS. HTTPS adds security by encrypting a normal HTTP request and protecting against + common cyberattacks. +For more information, see Getting started with a secure static website in the Amazon CloudFront Developer Guide. +If you see an Error when you list your buckets and their public + access settings, you might not have the required permissions. Make sure that you + have the following permissions added to your user or role policy: +In some rare cases, requests can also fail because of an AWS Region + outage. +Amazon S3 supports various options for you to configure your bucket. For example, you can + configure your bucket for website hosting, add a configuration to manage the lifecycle + of objects in the bucket, and configure the bucket to log all access to the bucket. Amazon S3 + supports subresources for you to store and manage the bucket configuration information. + You can use the Amazon S3 API to create and manage these subresources. However, you can also + use the console or the AWS SDKs. +There are also object-level configurations. For example, you can configure + object-level permissions by configuring an access control list (ACL) specific to + that object. +These are referred to as subresources because they exist in the context of a specific + bucket or object. The following table lists subresources that enable you to manage + bucket-specific configurations. + +cors (cross-origin resource sharing) + You can configure your bucket to allow cross-origin + requests. +For more information, see Using cross-origin resource sharing (CORS). + +event notification + +You can enable your bucket to send you notifications of specified + bucket events. +For more information, see Amazon S3 Event Notifications. +You can define lifecycle rules for objects in your bucket that + have a well-defined lifecycle. For example, you can define a rule to + archive objects one year after creation, or delete an object 10 + years after creation. +For more information, see Managing the lifecycle of objects. + +location + + When you create a bucket, you specify the AWS Region where you + want Amazon S3 to create the bucket. Amazon S3 stores this information in the + location subresource and provides an API for you to retrieve this + information. + +logging + +Logging enables you to track requests for access to your bucket. + Each access log record provides details about a single access + request, such as the requester, bucket name, request time, request + action, response status, and error code, if any. Access log + information can be useful in security and access audits. It can also + help you learn about your customer base and understand your Amazon S3 + bill.   +For more information, see Logging requests with server access logging. + +object locking + +To use S3 Object Lock, you must enable it for a bucket. You can + also optionally configure a default retention mode and period that + applies to new objects that are placed in the bucket. +For more information, see Locking objects with Object Lock. + +policy and ACL (access + control list) +All your resources (such as buckets and objects) are private by + default. Amazon S3 supports both bucket policy and access control list + (ACL) options for you to grant and manage bucket-level permissions. + Amazon S3 stores the permission information in the + policy and acl + subresources. +For more information, see Identity and Access Management for Amazon S3. + +replication + +Replication is the automatic, asynchronous copying of objects + across buckets in different or the same AWS Regions. For more + information, see Replicating objects within and across Regions. + +requestPayment + +By default, the AWS account that creates the bucket (the bucket + owner) pays for downloads from the bucket. Using this subresource, + the bucket owner can specify that the person requesting the download + will be charged for the download. Amazon S3 provides an API for you to + manage this subresource. +For more information, see Using Requester Pays buckets for storage + transfers and usage. + +tagging + +You can add cost allocation tags to your bucket to categorize and + track your AWS costs. Amazon S3 provides the + tagging subresource to store and manage + tags on a bucket. Using tags you apply to your bucket, AWS + generates a cost allocation report with usage and costs aggregated + by your tags. +For more information, see Billing and usage reporting for Amazon S3. + +transfer acceleration + +Transfer Acceleration enables fast, easy, and secure transfers of files + over long distances between your client and an S3 bucket. + Transfer Acceleration takes advantage of the globally distributed edge + locations of Amazon CloudFront. +For more information, see Configuring fast, secure file transfers using + Amazon S3 Transfer Acceleration. +Versioning helps you recover accidental overwrites and deletes. +We recommend versioning as a best practice to recover objects from + being deleted or overwritten by mistake. +For more information, see Retaining multiple versions of objects with S3 Versioning. +You can configure your bucket for static website hosting. Amazon S3 + stores this configuration by creating a website + subresource. +For more information, see Hosting a static website using Amazon S3. +The high availability engineering of Amazon S3 is focused on get, put, list, and delete operations. Because + bucket operations work against a centralized, global resource space, we recommend that you + don't create, delete, or configure buckets on the high availability code path + of your application. It's better to create, delete, or configure buckets in a separate + initialization or setup routine that you run less often. + Javascript is disabled or is unavailable in your browser. +To use the Amazon Web Services Documentation, Javascript must be enabled. Please refer to your browser's Help pages for instructions. +Thanks for letting us know we're doing a good job! +If you've got a moment, please tell us what we did right so we can do more of it. + +Thanks for letting us know this page needs work. We're sorry we let you down. +If you've got a moment, please tell us how we can make the documentation better. + diff --git a/crawl/crawled_data/Common bucket patterns for building applications on Amazon S3 - Amazon Simple Storage Service.txt b/crawl/crawled_data/Common bucket patterns for building applications on Amazon S3 - Amazon Simple Storage Service.txt new file mode 100644 index 00000000..1de5b92a --- /dev/null +++ b/crawl/crawled_data/Common bucket patterns for building applications on Amazon S3 - Amazon Simple Storage Service.txt @@ -0,0 +1,65 @@ +Title: Common bucket patterns for building applications on Amazon S3 - Amazon Simple Storage Service + +When you build applications on Amazon S3, you can use unique general + purpose buckets to separate different datasets or workloads. When you build applications + that serve end users or different user groups, use our best practices design + patterns to build applications that can best take advantage of Amazon S3 features and + scalability. + +We recommend that you create bucket names that are not predictable. Do not write code + assuming your chosen bucket name is available unless you have already created the + bucket. One method for creating bucket names that are not predictable is to append a + Globally Unique Identifier (GUID) to your bucket name, for example, + amzn-s3-demo-bucket-a1b2c3d4-5678-90ab-cdef-EXAMPLE11111. + For more information about general purpose bucket naming rules, see General purpose bucket naming rules. +With multi-tenant buckets, you create a single bucket for a team or workload. You use + unique S3 prefixes to organize the objects + that you store in the bucket. A prefix is a string of characters at the beginning of the + object key name. A prefix can be any length, subject to the maximum length of the object + key name (1,024 bytes). You can think of prefixes as a way to organize your data in a + similar way to directories. However, prefixes are not directories. +For example, to store information about cities, you might organize it by continent, + then by country, then by province or state. Because these names don't usually contain + punctuation, you might use slash (/) as the delimiter. The following examples shows + prefixes being used to organize city names by continent, country, and then province or + state, using a slash (/) delimiter. +Europe/France/NouvelleA-Aquitaine/Bordeaux +North America/Canada/Quebec/Montreal +North America/USA/Washington/Bellevue +North America/USA/Washington/Seattle +This pattern scales well when you have hundreds of unique datasets within a bucket. + With prefixes, you can easily organize and group these datasets. +However, one potential drawback to the multi-tenant bucket pattern is that many S3 + bucket-level features like default bucket + encryption, S3 Versioning, and + S3 Requester Pays are set at the + bucket-level and not the prefix-level. If the different datasets within the multi-tenant + bucket have unique requirements, the fact that you can't configure many S3 bucket-level + features at the prefix-level can make it difficult for you to specify the correct + settings for each dataset. Additionally, in a multi-tenant bucket, cost allocation can become complex as you work to + understand the storage, requests, and data transfer associated with specific prefixes. +With the bucket-per-use pattern, you create a bucket for each distinct dataset, end + user, or team. Because you can configure S3 bucket-level features for each of these + buckets, you can use this pattern to configure unique bucket-level settings. For + example, you can configure features like default + bucket encryption, S3 Versioning, + and S3 Requester Pays in a way that is + customized to the dataset in each bucket. Using one bucket for each distinct dataset, + end user, or team can also help you simplify both your access management and cost + allocation strategies. +A potential drawback to this strategy is that you will need to manage potentially + thousands of buckets. All AWS accounts have a default bucket quota of 10,000 general + purpose buckets. You can increase the bucket quota for an account by submitting a quota + increase request. To request an increase for general purpose buckets, visit the Service Quotas console. +To manage your bucket-per-use pattern and simplify your infrastructure management, you + can use AWS CloudFormation. You can create a custom AWS CloudFormation template for your pattern that + already defines all of your desired settings for your S3 buckets so that you can easily + deploy and track any changes to your infrastructure. For more information, see AWS::S3::Bucket in the AWS CloudFormation User Guide. + Javascript is disabled or is unavailable in your browser. +To use the Amazon Web Services Documentation, Javascript must be enabled. Please refer to your browser's Help pages for instructions. +Thanks for letting us know we're doing a good job! +If you've got a moment, please tell us what we did right so we can do more of it. + +Thanks for letting us know this page needs work. We're sorry we let you down. +If you've got a moment, please tell us how we can make the documentation better. + diff --git a/crawl/crawled_data/Configuring Storage Browser for S3 - Amazon Simple Storage Service.txt b/crawl/crawled_data/Configuring Storage Browser for S3 - Amazon Simple Storage Service.txt new file mode 100644 index 00000000..68b5ff98 --- /dev/null +++ b/crawl/crawled_data/Configuring Storage Browser for S3 - Amazon Simple Storage Service.txt @@ -0,0 +1,15 @@ +Title: Configuring Storage Browser for S3 - Amazon Simple Storage Service + +To allow Storage Browser for S3 access to S3 buckets, the Storage Browser component + makes the REST API calls to Amazon S3. By default, cross-origin resource sharing (CORS) isn’t + enabled on S3 buckets. As a result, you must enable CORS for each S3 bucket that Storage Browser is accessing data from. +For example, to enable CORS on your S3 bucket, you can update your CORS policy like + this: + Javascript is disabled or is unavailable in your browser. +To use the Amazon Web Services Documentation, Javascript must be enabled. Please refer to your browser's Help pages for instructions. +Thanks for letting us know we're doing a good job! +If you've got a moment, please tell us what we did right so we can do more of it. + +Thanks for letting us know this page needs work. We're sorry we let you down. +If you've got a moment, please tell us how we can make the documentation better. + diff --git a/crawl/crawled_data/Configuring and using Mountpoint - Amazon Simple Storage Service.txt b/crawl/crawled_data/Configuring and using Mountpoint - Amazon Simple Storage Service.txt new file mode 100644 index 00000000..3d07c048 --- /dev/null +++ b/crawl/crawled_data/Configuring and using Mountpoint - Amazon Simple Storage Service.txt @@ -0,0 +1,117 @@ +Title: Configuring and using Mountpoint - Amazon Simple Storage Service + +To use Mountpoint for Amazon S3, your host needs valid AWS credentials with access to the bucket or + buckets that you would like to mount. For different ways to authenticate, see + Mountpoint AWS Credentials on GitHub. +For example, you can create a new AWS Identity and Access Management (IAM) user and role for this purpose. Make + sure that this role has access to the bucket or buckets that you would like to mount. You can + pass the + IAM role to your Amazon EC2 instance with an instance profile. +Use Mountpoint for Amazon S3 to do the following: +Mount buckets with the mount-s3 command. +In the following example, replace + amzn-s3-demo-bucket with the name of your S3 + bucket, and replace ~/mnt with the directory on + your host where you want your S3 bucket to be mounted. +Because the Mountpoint client runs in the background by default, the + ~/mnt directory now gives you access to the + objects in your S3 bucket. +Access the objects in your bucket through Mountpoint. +After you mount your bucket locally, you can use common Linux + commands, such as cat or ls, to work with your S3 objects. + Mountpoint for Amazon S3 interprets keys in your S3 bucket as file system paths by splitting them on + the forward slash (/) character. For example, if you have the object key + Data/2023-01-01.csv in your bucket, you will have a directory named + Data in your Mountpoint file system, with a file named + 2023-01-01.csv inside it. +Mountpoint for Amazon S3 intentionally does not implement the full POSIX standard specification for + file systems. Mountpoint is optimized for workloads that need high-throughput + read and write access to data stored in Amazon S3 through a file system interface, but that + otherwise do not rely on file system features. For more information, see Mountpoint for Amazon S3 + file + system behavior on GitHub. Customers that need richer file + system semantics should consider other AWS file services, such as Amazon Elastic File System (Amazon EFS) or Amazon FSx. + +Unmount your bucket by using the umount command. This command unmounts + your S3 bucket and exits Mountpoint. +To use the following example command, replace + ~/mnt with the directory on your host where + your S3 bucket is mounted. +To get a list of options for this command, run umount --help. +For additional Mountpoint configuration details, see S3 bucket configuration, and file system configuration on GitHub. +Mountpoint for Amazon S3 supports different types of data caching. To accelerate repeated read + requests, you can opt in to the following: +Local cache – You can use a local cache in + your Amazon EC2 instance storage or an Amazon Elastic Block Store volume. If you repeatedly read the same data + from the same compute instance and if you have unused space in your local instance + storage for the repeatedly read dataset, you should opt in to a local cache. +Shared cache – You can use a shared cache on + S3 Express One Zone. If you repeatedly read small objects from multiple compute instances or if + you do not know the size of your repeatedly read dataset and want to benefit from + elasticity of cache size, you should opt in to the shared cache. Once you opt in, + Mountpoint retains objects with sizes up to one megabyte in a directory bucket + that uses S3 Express One Zone. +Combined local and shared cache – If you + have unused space in your local cache but also want a shared cache across multiple + instances, you can opt in to both a local cache and shared cache. +Caching in Mountpoint is ideal for use cases where you repeatedly read the same + data that doesn’t change during the multiple reads. For example, you can use caching with + machine learning training jobs that need to read a training dataset multiple times to + improve model accuracy. +For more information about how to configure caching in Mountpoint, see the + following examples. +You can opt in to a local cache with the --cache + CACHE_PATH flag. In the following example, replace + CACHE_PATH with the filepath to the directory + that you want to cache your data in. Replace + amzn-s3-demo-bucket with the name of your S3 + bucket, and replace ~/mnt with the directory on + your host where you want your S3 bucket to be mounted. +When you opt in to local caching while mounting an S3 bucket, Mountpoint creates an + empty sub-directory at the configured cache location, if that sub-directory doesn’t + already exist. When you first mount a bucket and when you unmount, Mountpoint deletes the + contents of the local cache. +If you enable local caching, Mountpoint will persist unencrypted object + content from your mounted S3 bucket at the local cache location provided at mount. In + order to protect your data, you should restrict access to the data cache location by + using file system access control mechanisms. +If you repeatedly read small objects (up to 1 MB) from multiple compute instances or + the size of the dataset that you repeatedly read often exceeds the size of your local + cache, you should use a shared cache in S3 Express One Zone. When you read the same + data repeatedly from multiple instances, this improves latency by avoiding redundant + requests to your mounted S3 bucket. +Once you opt in to the shared cache, you pay for the data cached in your + directory bucket in S3 Express One Zone. You also pay for requests made against your data in the + directory bucket in S3 Express One Zone. For more information, see Amazon S3 pricing. Mountpoint never deletes cached objects + from directory buckets. To manage your storage costs, you should set up a Lifecycle + policy on your directory bucket so that Amazon S3 expires the cached data in + S3 Express One Zone after a period of time that you specify. For more information, see Mountpoint for Amazon S3 caching configuration on GitHub. +To opt in to caching in S3 Express One Zone when you mount a general purpose bucket to your compute + instance, use the --cache-xz flag and specify a directory bucket as your + cache location. In the following example, replace the user input + placeholders. +If you have unused space on your instance but you also want to use a shared cache + across multiple instances, you can opt in to both a local cache and shared cache. With + this caching configuration, you can avoid redundant read requests from the same instance + to the shared cache in directory bucket when the required data is cached in local + storage. This can reduce request costs and improve performance. + To opt in to both a local cache and shared cache when you mount an S3 bucket, you + specify both cache locations by using the --cache and --cache-xz + flags. To use the following example to opt into both a local and shared cache, replace the + user input placeholders. +For more information, Mountpoint for Amazon S3 caching configuration on GitHub. +If you enable shared caching, Mountpoint will copy object content from your + mounted S3 bucket into the S3 directory bucket that you provide as your shared cache + location, making it accessible to any caller with access to the S3 directory bucket. To + protect your cached data, you should follow the Security best practices for Amazon S3 to ensure that your buckets use the correct + policies and are not publicly accessible. You should use a directory bucket dedicated + to Mountpoint shared caching and grant access only to Mountpoint + clients. + Javascript is disabled or is unavailable in your browser. +To use the Amazon Web Services Documentation, Javascript must be enabled. Please refer to your browser's Help pages for instructions. +Thanks for letting us know we're doing a good job! +If you've got a moment, please tell us what we did right so we can do more of it. + +Thanks for letting us know this page needs work. We're sorry we let you down. +If you've got a moment, please tell us how we can make the documentation better. + diff --git a/crawl/crawled_data/Creating a bucket - Amazon Simple Storage Service.txt b/crawl/crawled_data/Creating a bucket - Amazon Simple Storage Service.txt new file mode 100644 index 00000000..752878be --- /dev/null +++ b/crawl/crawled_data/Creating a bucket - Amazon Simple Storage Service.txt @@ -0,0 +1,226 @@ +Title: Creating a bucket - Amazon Simple Storage Service + +To upload your data to Amazon S3, you must first create an Amazon S3 bucket in one of the + AWS Regions. The AWS account that creates the bucket owns it. When you create a bucket, you must choose a bucket name and Region. You can + optionally choose other storage management options for the bucket. After you create a + bucket, you cannot change the bucket name or Region. For information about naming buckets, + see General purpose bucket naming rules. + +By default, you can create up to 10,000 general purpose buckets + per AWS account. To request a quota increase for general purpose buckets, + visit the Service Quotas + console. +You can store any number of objects in a bucket. For a list of restriction and limitations + related to Amazon S3 buckets see, Bucket quotas, limitations, and restrictions. + S3 Object Ownership is an Amazon S3 bucket-level setting that you can use both to control + ownership of objects that are uploaded to your bucket and to disable or enable access control lists (ACLs). By default, + Object Ownership is set to the Bucket owner enforced setting, and all ACLs are disabled. With ACLs disabled, the bucket owner + owns every object in the bucket and manages access to data exclusively by using policies. + For more information, see Controlling ownership of objects and disabling ACLs + for your bucket. +Server-side encryption with Amazon S3 managed keys (SSE-S3) + is the base level of encryption configuration for every bucket in Amazon S3. All new objects uploaded to an S3 bucket +are automatically encrypted with SSE-S3 as the base level of encryption setting. If you want to use a different type of default encryption, you can also specify server-side encryption with AWS Key Management Service + (AWS KMS) keys (SSE-KMS) or customer-provided keys (SSE-C) to encrypt your data. +For more information, see Setting default server-side encryption behavior for Amazon S3 + buckets. +You can use the Amazon S3 console, Amazon S3 APIs, AWS CLI, or AWS SDKs to create a bucket. For more + information about the permissions required to create a bucket, see CreateBucket in the Amazon Simple Storage Service API Reference. +Sign in to the AWS Management Console and open the Amazon S3 console at + https://console.aws.amazon.com/s3/. +In the navigation bar on the top of the page, choose the name of the currently displayed AWS Region. Next, choose the Region in which you want to create a bucket. + +To minimize latency and costs and address regulatory requirements, choose a Region + close to you. Objects stored in a Region never leave that Region unless you explicitly + transfer them to another Region. For a list of Amazon S3 AWS Regions, see AWS service endpoints in the + Amazon Web Services General Reference. +In the left navigation pane, choose Buckets. +Choose Create bucket. +The Create bucket page opens. +Under General configuration, view the AWS Region where your bucket will be created. + Under Bucket type, choose General purpose. +For Bucket name, enter a name for your bucket. +The bucket name must: +Be unique within a partition. A partition is a grouping of Regions. AWS + currently has three partitions: aws (Standard Regions), + aws-cn (China Regions), and aws-us-gov + (AWS GovCloud (US) Regions). +Be between 3 and 63 characters long. +Consist only of lowercase letters, numbers, dots (.), and hyphens (-). For + best compatibility, we recommend that you avoid using dots (.) in bucket names, + except for buckets that are used only for static website hosting. +Begin and end with a letter or number. +After you create the bucket, you cannot change its name. The AWS account that creates the bucket owns it. For more information about + naming buckets, see General purpose bucket naming rules. +Avoid including sensitive information, such as account numbers, in the bucket + name. The bucket name is visible in the URLs that point to the objects in the + bucket. +AWS Management Console allows you to copy an existing bucket's settings to your new bucket. If you do not want to copy the settings of an existing bucket, skip to the next step. +This option: +Is not available in the AWS CLI and is only available in console +Is not available for directory buckets +Does not copy the bucket policy from the existing bucket to the new bucket + To copy an existing bucket's settings, under Copy settings from existing bucket, select Choose bucket. The Choose bucket window opens. Find the bucket with the settings that you would like to copy, and select Choose bucket. The Choose bucket window closes, and the Create bucket window re-opens. +Under Copy settings from existing bucket, you will now see the name of the bucket you selected. You will also see a Restore defaults option that you can use to remove the copied bucket settings. Review the remaining bucket settings, on the Create bucket page. You will see that they now match the settings of the bucket that you selected. You can skip to the final step. +Under Object Ownership, to disable or enable ACLs and control + ownership of objects uploaded in your bucket, choose one of the following + settings: + Bucket owner enforced (default) – + ACLs are disabled, and the bucket owner automatically owns and has full control over every object in the bucket. ACLs + no longer affect access permissions to data in the S3 bucket. The bucket uses policies exclusively to define access control. +By default, ACLs are disabled. A majority of modern use cases in Amazon S3 no + longer require the use of ACLs. We recommend that you keep ACLs disabled, except + in unusual circumstances where you must control access for each object + individually. For more information, see Controlling ownership of objects and disabling ACLs + for your bucket. +Bucket owner preferred – The bucket owner owns and + has full control over new objects that other accounts write to the bucket with + the bucket-owner-full-control canned ACL. +If you apply the Bucket owner preferred setting, to + require all Amazon S3 uploads to include the bucket-owner-full-control + canned ACL, you can add a + bucket policy that allows only object uploads that use this + ACL. +Object writer – The AWS account that uploads an + object owns the object, has full control over it, and can grant other users + access to it through ACLs. +The default setting is Bucket owner enforced. To apply the + default setting and keep ACLs disabled, only the s3:CreateBucket + permission is needed. To enable ACLs, you must have the + s3:PutBucketOwnershipControls permission. +Under Block Public Access settings for this bucket, choose the + Block Public Access settings that you want to apply to the bucket. +By default, all four Block Public Access settings are enabled. We recommend that you + keep all settings enabled, unless you know that you need to turn off one or more of them + for your specific use case. For more information about blocking public access, see Blocking public access to your Amazon S3 + storage. +To enable all Block Public Access settings, only the s3:CreateBucket permission + is required. To turn off any Block Public Access settings, you must have the + s3:PutBucketPublicAccessBlock permission. +(Optional) Under Bucket Versioning, you can choose if you wish to keep + variants of objects in your bucket. For more information about versioning, + see Retaining multiple versions of objects with S3 Versioning. +To disable or enable versioning on your + bucket, choose either Disable or Enable. +(Optional) Under Tags, you can choose to add tags to your bucket. + Tags are key-value pairs used to categorize storage. +To add a bucket tag, enter a Key and optionally a + Value and choose Add Tag. +Under Default encryption, choose + Edit. +To configure default encryption, under Encryption type, + choose one of the following: +Amazon S3 managed key (SSE-S3) +AWS Key Management Service key (SSE-KMS) +If you use the SSE-KMS option for your default encryption configuration, you are + subject to the requests per second (RPS) quota of AWS KMS. For more information + about AWS KMS quotas and how to request a quota increase, see Quotas in the AWS Key Management Service Developer Guide. +Buckets and new objects are encrypted with server-side encryption with an Amazon S3 managed key as the base level of encryption configuration. For more + information about default encryption, see Setting default server-side encryption behavior for Amazon S3 + buckets. +For more information about using Amazon S3 server-side encryption to encrypt your data, + see Using server-side encryption with Amazon S3 managed keys + (SSE-S3). +If you chose AWS Key Management Service key (SSE-KMS), do the following: +Under AWS KMS key, specify your KMS key in one of the following ways: +To choose from a list of available KMS keys, choose Choose from + your AWS KMS keys, and choose your + KMS key from the list of available keys. +Both the AWS managed key (aws/s3) and your customer managed keys appear in this + list. For more information about customer managed keys, see Customer keys and + AWS keys in the AWS Key Management Service Developer Guide. +To enter the KMS key ARN, choose Enter AWS KMS key + ARN, and enter your KMS key ARN in the field that appears. + +To create a new customer managed key in the AWS KMS console, choose Create a + KMS key. +For more information about creating an AWS KMS key, see Creating keys in the AWS Key Management Service Developer Guide. +You can use only KMS keys that are available in the same AWS Region as the + bucket. The Amazon S3 console lists only the first 100 KMS keys in the same Region as the bucket. + To use a KMS key that is not listed, you must enter your KMS key ARN. If you want to use + a KMS key that is owned by a different account, you must first have permission to use the + key and then you must enter the KMS key ARN. For more information on cross account permissions for KMS keys, see Creating KMS keys that other accounts can use in the AWS Key Management Service Developer Guide. For more information on SSE-KMS, see Specifying server-side encryption with AWS KMS + (SSE-KMS). +When you use an AWS KMS key for server-side encryption in Amazon S3, you must + choose a symmetric encryption KMS key. Amazon S3 supports only symmetric encryption KMS keys and not + asymmetric KMS keys. For more information, see Identifying symmetric and + asymmetric KMS keys in the AWS Key Management Service Developer Guide. +For more information about creating an AWS KMS key, see Creating keys in the + AWS Key Management Service Developer Guide. For more information about using AWS KMS with + Amazon S3, see Using server-side encryption with AWS KMS keys + (SSE-KMS). +When you configure your bucket to use default encryption with SSE-KMS, you can also + enable S3 Bucket Keys. S3 Bucket Keys lower the cost of encryption by decreasing request traffic from + Amazon S3 to AWS KMS. For more information, see Reducing the cost of SSE-KMS with Amazon S3 Bucket Keys. +To use S3 Bucket Keys, under Bucket Key, choose + Enable. +(Optional) If you want to enable S3 Object Lock, do the + following: +Choose Advanced settings. +Enabling Object Lock also enables versioning for + the bucket. After enabling you must configure the Object Lock default + retention and legal hold settings to protect new objects from being deleted + or overwritten. +If you want to enable Object Lock, choose + Enable, read the warning that appears, and acknowledge it. +For more information, see Locking objects with Object Lock. +To create an Object Lock enabled bucket, you must have the following permissions: s3:CreateBucket, s3:PutBucketVersioning and s3:PutBucketObjectLockConfiguration. +Choose Create bucket. +When you use the AWS SDKs to create a bucket, you must create a client and then + use the client to send a request to create a bucket. As a best practice, you should + create your client and bucket in the same AWS Region. If you don't specify a + Region when you create a client or a bucket, Amazon S3 uses the default Region, + US East (N. Virginia). If you want to constrain the bucket creation to a specific + AWS Region, use the LocationConstraint condition key. +To create a client to access a dual-stack endpoint, you must specify an + AWS Region. For more information, see Using Amazon S3 dual-stack endpoints + in the Amazon S3 API Reference + . For a list of + available AWS Regions, see Regions and endpoints in the + AWS General Reference. +When you create a client, the Region maps to the Region-specific endpoint. The + client uses this endpoint to communicate with Amazon S3: + s3.region.amazonaws.com. If your + Region launched after March 20, 2019, your client and bucket must be in the same + Region. However, you can use a client in the US East (N. Virginia) Region to create a bucket in + any Region that launched before March 20, 2019. For more information, see Legacy endpoints. +These AWS SDK code examples perform the following tasks: +Create a client by explicitly specifying an + AWS Region – In the example, the client uses the + s3.us-west-2.amazonaws.com endpoint to communicate with + Amazon S3. You can specify any AWS Region. For a list of AWS Regions, see + Regions and + endpoints in the AWS General Reference. + +Send a create bucket request by specifying only a + bucket name – The client sends a request to Amazon S3 to + create the bucket in the Region where you created a client. +Retrieve information about the location of the + bucket – Amazon S3 stores bucket location information in + the location subresource that is associated with the + bucket. +The following example shows you how to create a bucket with a GUID at the end + of the bucket name in US East (N. Virginia) Region (us-east-1;) by using the + AWS SDK for Java. For information about other AWS SDKs, see Tools to Build on AWS. +This example shows you how to create an Amazon S3 bucket using the AWS SDK for Java. + For instructions on creating and testing a working sample, see Getting + Started in the AWS SDK for Java Developer Guide. +For information about how to create and test a working sample, see + AWS + SDK for .NET Version 3 API Reference. +For information about how to create and test a working sample, see + AWS SDK for + Ruby - Version 3. +The following AWS CLI example creates a bucket in the US West (N. California) Region + (us-west-1) Region with an example bucket name that uses a globally + unique identifier (GUID). +For more + information, see create-bucket in the AWS CLI Command Reference. + Javascript is disabled or is unavailable in your browser. +To use the Amazon Web Services Documentation, Javascript must be enabled. Please refer to your browser's Help pages for instructions. +Thanks for letting us know we're doing a good job! +If you've got a moment, please tell us what we did right so we can do more of it. + +Thanks for letting us know this page needs work. We're sorry we let you down. +If you've got a moment, please tell us how we can make the documentation better. + diff --git a/crawl/crawled_data/Example metadata table queries - Amazon Simple Storage Service.txt b/crawl/crawled_data/Example metadata table queries - Amazon Simple Storage Service.txt new file mode 100644 index 00000000..0c13f2fd --- /dev/null +++ b/crawl/crawled_data/Example metadata table queries - Amazon Simple Storage Service.txt @@ -0,0 +1,40 @@ +Title: Example metadata table queries - Amazon Simple Storage Service + +The S3 Metadata feature is in preview release for Amazon S3 and is subject to change. +The following examples show how you can get different types information from your S3 + Metadata tables by using standard SQL queries. +Remember when using these examples: +The examples are written to work with Amazon Athena. You might have to modify the examples + to work with a different query engine. +Make sure that you understand how to optimize your + queries. +Replace amzn-s3-demo-bucket with the name of the S3 table bucket that's + storing your metadata table. +Replace my_metadata_table with the name of the + metadata table that you're querying. +For a full list of supported columns, see the S3 Metadata tables schema. +The following query returns objects with a specific file extension (.jpg in + this case). +The following query returns object deletion events, including the AWS account ID or + AWS service principal that made the request. +The following query returns the ARNs of the AWS Key Management Service (AWS KMS) keys encrypting your + objects. +The following query returns objects that aren't encrypted with AWS KMS keys. +Some AWS services (such as Amazon Bedrock), + upload objects to Amazon S3. You can query the object metadata provided by these services. For + example, the following query includes the user_metadata column to determine if + there are objects uploaded by Amazon Bedrock to a general purpose bucket. +If Amazon Bedrock uploaded an object to your bucket, the user_metadata column will + display the following metadata associated with the object in the query result: +The following query can help you determine the current state of your objects. The query + identifies the most recent version of each object, filters out deleted objects, and marks + the latest version of each object based on sequence numbers. Results are ordered by the + bucket, key, and sequence_number columns. + Javascript is disabled or is unavailable in your browser. +To use the Amazon Web Services Documentation, Javascript must be enabled. Please refer to your browser's Help pages for instructions. +Thanks for letting us know we're doing a good job! +If you've got a moment, please tell us what we did right so we can do more of it. + +Thanks for letting us know this page needs work. We're sorry we let you down. +If you've got a moment, please tell us how we can make the documentation better. + diff --git a/crawl/crawled_data/General purpose bucket naming rules - Amazon Simple Storage Service.txt b/crawl/crawled_data/General purpose bucket naming rules - Amazon Simple Storage Service.txt new file mode 100644 index 00000000..735132ff --- /dev/null +++ b/crawl/crawled_data/General purpose bucket naming rules - Amazon Simple Storage Service.txt @@ -0,0 +1,115 @@ +Title: General purpose bucket naming rules - Amazon Simple Storage Service + +When you create a general purpose bucket, you choose its name and the AWS Region to create it in. After + you create a general purpose bucket, you can't change its name or Region. The following sections provide information + about general purpose bucket naming, including naming rules, best practices, and an example for creating a general purpose bucket + with a name that includes a globally unique identifier (GUID). +For directory bucket naming rules, see Directory bucket naming rules. + For information on object key names, see Creating object key names. +The following naming rules apply for general purpose buckets. +Bucket names must be between 3 (min) and 63 (max) characters long. +Bucket names can consist only of lowercase letters, numbers, dots (.), and + hyphens (-). +Bucket names must begin and end with a letter or number. +Bucket names must not contain two adjacent periods. +Bucket names must not be formatted as an IP address (for example, + 192.168.5.4). +Bucket names must not start with the prefix xn--. +Bucket names must not start with the prefix sthree-. +Bucket names must not start with the prefix + sthree-configurator. +Bucket names must not start with the prefix + amzn-s3-demo-. +Bucket names must not end with the suffix -s3alias. This suffix + is reserved for access point alias names. For more information, see Using a bucket-style alias for your S3 bucket + access point. +Bucket names must not end with the suffix --ol-s3. This suffix is + reserved for Object Lambda Access Point alias names. For more information, see How to use a bucket-style alias for your S3 bucket + Object Lambda Access Point. +Bucket names must not end with the suffix .mrap. This suffix is + reserved for Multi-Region Access Point names. For more information, see Rules for naming Amazon S3 Multi-Region Access Points. +Bucket names must not end with the suffix --x-s3. This suffix is + reserved for directory buckets. For more information, see Directory bucket naming rules. +Bucket names must be unique across all AWS accounts in all the AWS Regions + within a partition. A partition is a grouping of Regions. AWS currently has + three partitions: aws (Standard Regions), aws-cn + (China Regions), and aws-us-gov (AWS GovCloud (US)). +A bucket name cannot be used by another AWS account in the same partition + until the bucket is deleted. +Buckets used with Amazon S3 Transfer Acceleration can't have dots (.) in their names. For + more information about Transfer Acceleration, see Configuring fast, secure file transfers using + Amazon S3 Transfer Acceleration. +Bucket names must be unique across all AWS accounts in all the + AWS Regions within a partition. A partition is a grouping of Regions. + AWS currently has three partitions: aws (Standard Regions), + aws-cn (China Regions), and aws-us-gov + (AWS GovCloud (US)). +A bucket name cannot be used by another AWS account in the same + partition until the bucket is deleted. After you delete a bucket, + be aware that another AWS account in the same partition can use the + same bucket name. +Before March 1, 2018, buckets created in the US East (N. Virginia) Region could have + names that were up to 255 characters long and included uppercase letters and + underscores. Beginning March 1, 2018, new buckets in US East (N. Virginia) must conform + to the same rules applied in all other Regions. +The following example bucket names are valid and follow the recommended naming + guidelines for general purpose buckets: +docexamplebucket-1a1b2c3d4-5678-90ab-cdef-EXAMPLEaaaaa +amzn-s3-demo-bucket1-a1b2c3d4-5678-90ab-cdef-EXAMPLE11111 +amzn-s3-demo-bucket-a1b2c3d4-5678-90ab-cdef-EXAMPLE22222 +amzn-s3-demo-bucket2 +The following example bucket names are valid but not recommended for uses other than + static website hosting: +example.com +www.example.com +my.example.s3.bucket +The following example bucket names are not + valid: +amzn_s3_demo_bucket (contains underscores) +AmznS3DemoBucket (contains uppercase letters) +amzn-s3-demo-bucket- (ends with a hyphen) +When naming your buckets, consider the following bucket naming best practices. +If your application automatically creates buckets, choose a bucket naming scheme + that is unlikely to cause naming conflicts. Ensure that your application logic will + choose a different bucket name if a bucket name is already taken. +We recommend that you create bucket names that are not predictable. Do not write + code assuming your chosen bucket name is available unless you have already created + the bucket. One method for creating bucket names that are not predictable is to + append a Globally Unique Identifier (GUID) to your bucket name, for example, + amzn-s3-demo-bucket-a1b2c3d4-5678-90ab-cdef-EXAMPLE11111. +For best compatibility, we recommend that you avoid using dots (.) in bucket + names, except for buckets that are used only for static website hosting. If you + include dots in a bucket's name, you can't use virtual-host-style addressing over + HTTPS, unless you perform your own certificate validation. This is because the + security certificates used for virtual hosting of buckets don't work for buckets + with dots in their names. +This limitation doesn't affect buckets used for static website hosting, because static + website hosting is only available over HTTP. For more information about + virtual-host-style addressing, see Virtual hosting of buckets. For more information about static website hosting, + see Hosting a static website using Amazon S3. +When you name a bucket, we recommend that you + choose a name that is relevant to you or your business. Avoid using names associated with others. For + example, you should avoid using AWS or Amazon in your bucket name. +If a bucket is empty, you can delete it. After a bucket is deleted, the name becomes + available for reuse. However, after you delete the bucket, you might not be able to + reuse the name for various reasons. +For example, when you delete the bucket and the name becomes available for reuse, + another AWS account might create a bucket with that name. In addition, some time might + pass before you can reuse the name of a deleted bucket. If you want to use the same + bucket name, we recommend that you don't delete the bucket. +The following examples show you how to create a general purpose bucket that uses a GUID at + the end of the bucket name. +The following AWS CLI example creates a bucket in the US West (N. California) Region + (us-west-1) Region with an example bucket name that uses a globally + unique identifier (GUID). +The following example shows you how to create a bucket with a GUID at the end + of the bucket name in US East (N. Virginia) Region (us-east-1;) by using the + AWS SDK for Java. For information about other AWS SDKs, see Tools to Build on AWS. + Javascript is disabled or is unavailable in your browser. +To use the Amazon Web Services Documentation, Javascript must be enabled. Please refer to your browser's Help pages for instructions. +Thanks for letting us know we're doing a good job! +If you've got a moment, please tell us what we did right so we can do more of it. + +Thanks for letting us know this page needs work. We're sorry we let you down. +If you've got a moment, please tell us how we can make the documentation better. + diff --git a/crawl/crawled_data/Installing Mountpoint - Amazon Simple Storage Service.txt b/crawl/crawled_data/Installing Mountpoint - Amazon Simple Storage Service.txt new file mode 100644 index 00000000..4a32a17b --- /dev/null +++ b/crawl/crawled_data/Installing Mountpoint - Amazon Simple Storage Service.txt @@ -0,0 +1,94 @@ +Title: Installing Mountpoint - Amazon Simple Storage Service + +You can download and install prebuilt packages of Mountpoint for Amazon S3 by using the command line. + The instructions for downloading and installing Mountpoint vary, depending on which + Linux operating system that you're using. +Copy the following download URL for your architecture. +x86_64: +ARM64 (Graviton): +Download the Mountpoint for Amazon S3 package. Replace + download-link with the appropriate download + URL from the preceding step. +(Optional) Verify the authenticity and integrity of the downloaded file. First, copy + the appropriate signature URL for your architecture. +x86_64: +ARM64 (Graviton): +Next, see Verifying the signature of the + Mountpoint for Amazon S3 package. +Install the package by using the following command: +Verify that Mountpoint is successfully installed by entering the following + command: +You should see output similar to the following: +Copy the download URL for your architecture. +x86_64: +ARM64 (Graviton): +Download the Mountpoint for Amazon S3 package. Replace + download-link with the appropriate download + URL from the preceding step. +(Optional) Verify the authenticity and integrity of the downloaded file. First, copy + the signature URL for your architecture. +x86_64: +ARM64 (Graviton): +Next, see Verifying the signature of the + Mountpoint for Amazon S3 package. +Install the package by using the following command: +Verify that Mountpoint for Amazon S3 is successfully installed by running the following + command: +You should see output similar to the following: +Consult your operating system documentation to install the FUSE and + libfuse2 packages, which are required. +Copy the download URL for your architecture. +x86_64: +ARM64 + (Graviton): +Download the Mountpoint for Amazon S3 package. Replace + download-link with the appropriate download + URL from the preceding step. +(Optional) Verify the authenticity and integrity of the downloaded file. First, copy + the signature URL for your architecture. +x86_64: +ARM64 (Graviton): +Next, see Verifying the signature of the + Mountpoint for Amazon S3 package. +Install the package by using the following command: +Add the mount-s3 binary to your PATH environment variable. + In your $HOME/.profile file, append the following line: +Save the .profile file, and run the following command: +Verify that Mountpoint for Amazon S3 is successfully installed by running the following + command: +You should see output similar to the following: +Install GnuPG (the gpg command). It is required to + verify the authenticity and integrity of a downloaded Mountpoint for Amazon S3 package. + GnuPG is installed by default on Amazon Linux Amazon Machine Images (AMIs). + After you installGnuPG, proceed to step 2. +Download the Mountpoint public key by running the following command: +Import the Mountpoint public key into your keyring by running the following + command: +Verify the fingerprint of the Mountpoint public key by running the following + command: +Confirm that the displayed fingerprint string matches the following: +If the fingerprint string doesn't match, do not finish installing + Mountpoint, and contact AWS Support. +Download the package signature file. Replace + signature-link with the appropriate + signature link from the preceding sections. +Verify the signature of the downloaded package by running the following command. + Replace signature-filename with the file name + from the previous step. +For example, on RPM-based distributions, including Amazon Linux, enter the following + command: +The output should include the phrase Good signature. If the output + includes the phrase BAD signature, redownload the Mountpoint + package file and repeat these steps. If the issue persists, do not finish installing + Mountpoint, and contact AWS Support. +The output may include a warning about a trusted signature. This does not indicate a + problem. It only means that you have not independently verified the Mountpoint + public key. + Javascript is disabled or is unavailable in your browser. +To use the Amazon Web Services Documentation, Javascript must be enabled. Please refer to your browser's Help pages for instructions. +Thanks for letting us know we're doing a good job! +If you've got a moment, please tell us what we did right so we can do more of it. + +Thanks for letting us know this page needs work. We're sorry we let you down. +If you've got a moment, please tell us how we can make the documentation better. + diff --git a/crawl/crawled_data/Installing Storage Browser for S3 - Amazon Simple Storage Service.txt b/crawl/crawled_data/Installing Storage Browser for S3 - Amazon Simple Storage Service.txt new file mode 100644 index 00000000..8eaa8871 --- /dev/null +++ b/crawl/crawled_data/Installing Storage Browser for S3 - Amazon Simple Storage Service.txt @@ -0,0 +1,15 @@ +Title: Installing Storage Browser for S3 - Amazon Simple Storage Service + +You can install Storage Browser for S3 from the latest version of + aws-amplify/ui-react-storage and aws-amplify packages in the + aws-amplify GitHub repository. When installing Storage Browser for S3, make + sure to add the following dependencies to your package.json file: +Alternatively, you can add the dependencies using Node Package Manager (NPM): + Javascript is disabled or is unavailable in your browser. +To use the Amazon Web Services Documentation, Javascript must be enabled. Please refer to your browser's Help pages for instructions. +Thanks for letting us know we're doing a good job! +If you've got a moment, please tell us what we did right so we can do more of it. + +Thanks for letting us know this page needs work. We're sorry we let you down. +If you've got a moment, please tell us how we can make the documentation better. + diff --git a/crawl/crawled_data/Joining custom metadata with S3 metadata tables - Amazon Simple Storage Service.txt b/crawl/crawled_data/Joining custom metadata with S3 metadata tables - Amazon Simple Storage Service.txt new file mode 100644 index 00000000..7612f810 --- /dev/null +++ b/crawl/crawled_data/Joining custom metadata with S3 metadata tables - Amazon Simple Storage Service.txt @@ -0,0 +1,21 @@ +Title: Joining custom metadata with S3 metadata tables - Amazon Simple Storage Service + +The S3 Metadata feature is in preview release for Amazon S3 and is subject to change. +You can analyze data across your S3 managed metadata tables and customer (self-managed) + metadata tables. By using a standard SQL JOIN operator, you can query data from + these + multiple + sources. +The following example SQL query finds matching records between an S3 managed metadata table + (my_s3_metadata_table) and a self-managed metadata + table (my_self_managed_metadata_table). The query also + filters informations based on CREATE events, which indicate that a new object (or a + new version of the object) was written to the bucket. (For more information, see the S3 Metadata tables schema.) + Javascript is disabled or is unavailable in your browser. +To use the Amazon Web Services Documentation, Javascript must be enabled. Please refer to your browser's Help pages for instructions. +Thanks for letting us know we're doing a good job! +If you've got a moment, please tell us what we did right so we can do more of it. + +Thanks for letting us know this page needs work. We're sorry we let you down. +If you've got a moment, please tell us how we can make the documentation better. + diff --git a/crawl/crawled_data/Metadata table limitations and restrictions - Amazon Simple Storage Service.txt b/crawl/crawled_data/Metadata table limitations and restrictions - Amazon Simple Storage Service.txt new file mode 100644 index 00000000..726aebd2 --- /dev/null +++ b/crawl/crawled_data/Metadata table limitations and restrictions - Amazon Simple Storage Service.txt @@ -0,0 +1,60 @@ +Title: Metadata table limitations and restrictions - Amazon Simple Storage Service + +The S3 Metadata feature is in preview release for Amazon S3 and is subject to change. +Before creating a metadata table configuration, be aware of the following limitations and + restrictions: +S3 Metadata is currently available only in the US East (N. Virginia), US East (Ohio), + and US West (Oregon) Regions. +S3 Metadata supports all storage classes, except for the following: +The S3 Express One Zone storage class +The S3 One Zone-Infrequent Access (S3 One Zone-IA; Z-IA) storage class in directory buckets in Local Zones +For the S3 Intelligent-Tiering storage class, the specific tier isn't shown in the + metadata table. +To create a metadata table configuration, you must create or specify an S3 table + bucket to store your metadata table in. This table bucket must be in the same AWS Region + and AWS account as your general purpose bucket. +S3 Metadata isn't supported for directory buckets or table buckets. You can create + metadata table configurations only for general purpose buckets. +S3 Metadata doesn't apply to any objects that already existed in your general purpose + bucket before you created your metadata table configuration. In other words, S3 Metadata + only captures metadata for change events (such as uploads, updates, and deletes) that + happen after you have created your metadata table configuration. +S3 Metadata is designed to continuously append to the metadata table as you make + changes to your general purpose bucket. Each update creates a snapshot—a new version of the metadata table. Because of the + read-only nature of the metadata table, you can't delete records in the metadata table. + You also can't use the snapshot expiration capability of S3 Tables to expire old snapshots + of your metadata table. +To help minimize your costs, you can periodically delete your metadata table + configuration and your metadata tables, and then recreate them. For more information, see + Deleting metadata table + configurations and Deleting metadata tables. +When you're creating or updating table bucket or table policies, make sure that you + don't restrict Amazon S3 from writing to your table bucket or your metadata table. If Amazon S3 is + unable to write to your table bucket or your metadata table, you must create a new + metadata table by deleting your metadata table configuration and your metadata table, and + then creating a new configuration. +Before you can delete a metadata table, you must first delete the associated metadata + table configuration on your general purpose bucket. +You can create a metadata table configuration only for an entire general purpose + bucket. You can't apply a metadata table configuration at the prefix level. +You can't pause and resume updates to a metadata table. Instead, you can stop a + metadata table from updating by deleting its associated metadata table configuration. To + start receiving updates again, you must create a new metadata table configuration, which + creates a new metadata table. +Metadata tables don't contain all the same metadata as is available through S3 + Inventory or through the Amazon S3 REST API. For example, the following information isn't + available in metadata tables: +S3 Lifecycle expiration or transition status +Object Lock retention period or governance mode +Object access control list (ACL) information +Replication status +You can't adjust the partitioning or sorting for metadata tables. As a result, some + queries might require table scans and therefore might be less efficient. + Javascript is disabled or is unavailable in your browser. +To use the Amazon Web Services Documentation, Javascript must be enabled. Please refer to your browser's Help pages for instructions. +Thanks for letting us know we're doing a good job! +If you've got a moment, please tell us what we did right so we can do more of it. + +Thanks for letting us know this page needs work. We're sorry we let you down. +If you've got a moment, please tell us how we can make the documentation better. + diff --git a/crawl/crawled_data/Naming Amazon S3 objects - Amazon Simple Storage Service.txt b/crawl/crawled_data/Naming Amazon S3 objects - Amazon Simple Storage Service.txt new file mode 100644 index 00000000..7470c8bc --- /dev/null +++ b/crawl/crawled_data/Naming Amazon S3 objects - Amazon Simple Storage Service.txt @@ -0,0 +1,123 @@ +Title: Naming Amazon S3 objects - Amazon Simple Storage Service + +The object key (or key name) uniquely identifies the object in an + Amazon S3 bucket. When you create an object, you specify the key name. For example, on the Amazon S3 + console, when you select a bucket, a list of objects in your bucket appears. + These names are the object keys. +The object key name is a sequence of Unicode characters with UTF-8 encoding of up to 1,024 + bytes long. Object key names are case sensitive. The following section will provide + limitations on object key names and guidance on choosing key names. +Object key names with the value "soap" aren't supported for virtual-hosted-style requests. For object key name values where "soap" is used, a + path-style + URL must be used instead. +The Amazon S3 data model is a flat structure: You create a bucket, and the bucket stores + objects. There is no hierarchy of subbuckets or subfolders. However, you can infer logical + hierarchy using key name prefixes and delimiters as the Amazon S3 console does. The Amazon S3 console + supports a concept of folders. For more information about how to edit metadata from the Amazon S3 + console, see Editing object metadata in the Amazon S3 console. +Suppose that your bucket (admin-created) has four objects with the + following object keys: +Development/Projects.xls +Finance/statement1.pdf +Private/taxdocument.pdf +s3-dg.pdf +The console uses the key name prefixes (Development/, + Finance/, and Private/) and delimiter ('/') to + present a folder structure. The s3-dg.pdf key does not have a prefix, so its object appears + directly at the root level of the bucket. If you open the Development/ + folder, you see the Projects.xlsx object in it. +Amazon S3 supports buckets and objects, and there is no hierarchy. However, by using + prefixes and delimiters in an object key name, the Amazon S3 console and the AWS SDKs can infer + hierarchy and introduce the concept of folders. +The Amazon S3 console implements folder object creation by creating a zero-byte object with the folder + prefix and delimiter value as the key. These folder objects don't appear in the console. + Otherwise they behave like any other objects and can be viewed and manipulated through the REST API, AWS CLI, and AWS SDKs. +You can use any UTF-8 character in an object key name. However, using certain + characters in key names can cause problems with some applications and protocols. The + following guidelines help you maximize compliance with DNS, web-safe characters, XML + parsers, and other APIs. +The following character sets are generally safe for use in key names. +0-9 +a-z +A-Z +Exclamation point (!) +Hyphen (-) +Underscore (_) +Period (.) +Asterisk (*) +Single quote (') +Open parenthesis (() +Close parenthesis ()) +The following are examples of valid object key names: +4my-organization +my.great_photos-2014/jan/myvacation.jpg +videos/2014/birthday/video1.wmv +Objects with key names ending with period(s) "." downloaded + using the Amazon S3 console will have the period(s) "." removed from the + key name of the downloaded object. To download an object with the key + name ending in period(s) "." retained in the downloaded object, + you must use the AWS Command Line Interface (AWS CLI), AWS SDKs, or REST API. +In addition, be aware of the following prefix limitations: +Objects with a prefix of "./" must be uploaded or downloaded + with the AWS Command Line Interface (AWS CLI), AWS SDKs, or REST API. You cannot + use the Amazon S3 console. +Objects with a prefix of "../" cannot be uploaded using the + AWS Command Line Interface (AWS CLI) or Amazon S3 console. +The following characters in a key name might require additional code handling and + likely need to be URL encoded or referenced as HEX. Some of these are non-printable + characters that your browser might not handle, which also requires special + handling: +Ampersand ("&") +Dollar ("$") +ASCII character ranges 00–1F hex (0–31 decimal) and 7F (127 decimal) + +'At' symbol ("@") +Equals ("=") +Semicolon (";") +Forward slash ("/") +Colon (":") +Plus ("+") +Space – Significant sequences of spaces might be lost in some uses + (especially multiple spaces) +Comma (",") +Question mark ("?") +We recommend that you don't use the following characters in a key name because of significant special character handling, which isn't consistent across all applications. +Backslash ("\") +Left curly brace ("{") +Non-printable ASCII characters (128–255 decimal characters) +Caret ("^") +Right curly brace ("}") +Percent character ("%") +Grave accent / back tick ("`") +Right square bracket ("]") +Quotation marks +'Greater Than' symbol (">") +Left square bracket ("[") +Tilde ("~") +'Less Than' symbol ("<") +'Pound' character ("#") +Vertical bar / pipe ("|") +As specified by the XML standard on end-of-line handling, + all XML text is normalized such that single carriage returns (ASCII code 13) and carriage returns immediately followed by a + line feed (ASCII code 10) are replaced by a single line feed character. To ensure the correct parsing of object keys in XML requests, + carriage returns and other special characters must be replaced with their + equivalent XML entity code when they are inserted within XML tags. The following is a list of such special + characters and their equivalent entity codes: +' as ' +” as " +& as & +< as < +> as > +\r as or +\n as or +The following example illustrates the use of an XML entity code as a substitution for a carriage return. + This DeleteObjects request deletes an object with the key parameter: + /some/prefix/objectwith\rcarriagereturn (where the \r is the carriage return). + Javascript is disabled or is unavailable in your browser. +To use the Amazon Web Services Documentation, Javascript must be enabled. Please refer to your browser's Help pages for instructions. +Thanks for letting us know we're doing a good job! +If you've got a moment, please tell us what we did right so we can do more of it. + +Thanks for letting us know this page needs work. We're sorry we let you down. +If you've got a moment, please tell us how we can make the documentation better. + diff --git a/crawl/crawled_data/Optimizing metadata table query performance - Amazon Simple Storage Service.txt b/crawl/crawled_data/Optimizing metadata table query performance - Amazon Simple Storage Service.txt new file mode 100644 index 00000000..b940df62 --- /dev/null +++ b/crawl/crawled_data/Optimizing metadata table query performance - Amazon Simple Storage Service.txt @@ -0,0 +1,20 @@ +Title: Optimizing metadata table query performance - Amazon Simple Storage Service + +The S3 Metadata feature is in preview release for Amazon S3 and is subject to change. +Since S3 Metadata is based on the Apache Iceberg table format, you can optimize the + performance and cost of + your metadata table queries by using specific time ranges. +For example, the following SQL query provides the sensitivity level of new objects in an + S3 general purpose bucket: +This query scans the entire metadata table, which might take a long time to run. To + improve performance, you can include the record_timestamp column to focus on a + specific time range. Here's an updated version of the previous query that looks at new objects + from the past month: + Javascript is disabled or is unavailable in your browser. +To use the Amazon Web Services Documentation, Javascript must be enabled. Please refer to your browser's Help pages for instructions. +Thanks for letting us know we're doing a good job! +If you've got a moment, please tell us what we did right so we can do more of it. + +Thanks for letting us know this page needs work. We're sorry we let you down. +If you've got a moment, please tell us how we can make the documentation better. + diff --git a/crawl/crawled_data/S3 Metadata tables schema - Amazon Simple Storage Service.txt b/crawl/crawled_data/S3 Metadata tables schema - Amazon Simple Storage Service.txt new file mode 100644 index 00000000..83d87e76 --- /dev/null +++ b/crawl/crawled_data/S3 Metadata tables schema - Amazon Simple Storage Service.txt @@ -0,0 +1,146 @@ +Title: S3 Metadata tables schema - Amazon Simple Storage Service + +The S3 Metadata feature is in preview release for Amazon S3 and is subject to change. +Amazon S3 metadata tables contain rows and columns. Each row represents a mutation event that has + created, updated, or deleted an object in your general purpose bucket. Most of these events are + the result of various user actions, but some of these events are the result of actions taken by + Amazon S3 on your behalf, such as S3 Lifecycle expirations or storage class transitions. +S3 Metadata is an event-processing pipeline that is designed to keep the metadata table + eventually consistent with what changes have occurred in your general purpose bucket. Be aware + that by the time that S3 Metadata is notified that an object is created or updated, that object + might already have been overwritten or deleted in the bucket. To create a row for an event, the + object must still exist in the bucket at the time that S3 Metadata is notified of the event. +The following is an example of a metadata table for a general purpose bucket named + amzn-s3-demo-bucket: +Metadata tables have the following schema: +bucket +key +sequence_number +The sequence number, which is an ordinal that's included in the records for a given + object. To order records of the same bucket and key, you can sort on + sequence_number. For a given bucket and key, a lexicographically larger + sequence_number value implies that the record was introduced to the + bucket more recently. +record_type +The type of this record, one of CREATE, UPDATE_METADATA, + or DELETE. +CREATE records indicate that a new object (or a new version of the + object) was written to the bucket. +UPDATE_METADATA records capture changes to mutable metadata for an + existing object, such as the storage class or tags. +DELETE records indicate that this object (or this version of the + object) has been deleted. When versioning is enabled, DELETE records + represent either a delete marker or a permanent delete. Delete markers have a + record_type value of DELETE and an + is_delete_marker value of True. Permanent delete records + have null values in all other columns except bucket, key, + sequence_number, record_type, + record_timestamp, and version_id. For more information, + see Deleting object versions from a + versioning-enabled bucket. +record_timestamp +The timestamp that's associated with this record. +version_id +The object's version ID. When you enable versioning on a bucket, Amazon S3 assigns a + version number to objects that are added to the bucket. For more information, see + Retaining multiple versions of objects with S3 Versioning. +Objects that are stored in your bucket before you set the versioning state have a + version ID of null. +is_delete_marker +The object's delete marker status. If the object is a delete marker, this value is + True. Otherwise, it's False. For more information, see + Working with delete markers. +Rows that are added for delete markers have a record_type value of + DELETE, not UPDATE_METADATA. If the delete marker is + created as the result of an S3 Lifecycle expiration, the requester + value is s3.amazonaws.com. +size +The object size in bytes, not including the size of incomplete multipart uploads or + object metadata. If is_delete_marker is True, the size is + 0. For more information, see System-defined object metadata. +last_modified_date +The object creation date or the last modified date, whichever is the latest. For + multipart uploads, the object creation date is the date when the multipart upload is + initiated. For more information, see System-defined object metadata. +e_tag +The entity tag (ETag), which is a hash of the object. The ETag reflects changes + only to the contents of an object, not to its metadata. The ETag can be an MD5 digest of + the object data. Whether the ETag is an MD5 digest depends on how the object was created + and how it's encrypted. For more information, see Object in the + Amazon S3 API Reference. +storage_class +The storage class that’s used for storing the object. One of STANDARD, + REDUCED_REDUNDANCY, STANDARD_IA, ONEZONE_IA, + INTELLIGENT_TIERING, GLACIER, DEEP_ARCHIVE, or + GLACIER_IR. For more information, see Understanding and managing Amazon S3 storage classes. +is_multipart +The object's upload type. If the object was uploaded as a multipart upload, this + value is True. Otherwise, it's False. For more information, + see Uploading and copying objects using multipart upload in Amazon S3. +encryption_status +The object's server-side encryption status, depending on what kind of encryption key is used: + server-side encryption with Amazon S3 managed keys (SSE-S3), server-side encryption with + AWS Key Management Service (AWS KMS) keys (SSE-KMS), dual-layer server-side encryption with + AWS KMS keys (DSSE-KMS), or server-side encryption with customer-provided keys + (SSE-C). If the object is unencrypted, this value is null. Possible values are + SSE-S3, SSE-KMS, DSSE-KMS, + SSE-C, or null. For more information, see Protecting data with encryption. +is_bucket_key_enabled +The object's S3 Bucket Key enablement status. If the object uses an S3 Bucket Key + for SSE-KMS, this value is True. Otherwise, it's False. For + more information, see Configuring an S3 Bucket Key at the object + level. +kms_key_arn +The Amazon Resource Name (ARN) for the KMS key with which the object is encrypted, + for rows where encryption_status is SSE-KMS or + DSSE-KMS. If the object isn't encrypted with SSE-KMS or DSSE-KMS, the + value is null. For more information, see Using server-side encryption with AWS KMS keys + (SSE-KMS) and + Using dual-layer server-side encryption with AWS KMS keys + (DSSE-KMS). +If a row represents an object version that no longer existed at the time that a + delete or overwrite event was processed, kms_key_arn contains a null + value, even if the encryption_status column value is + SSE-KMS or DSSE-KMS. +checksum_algorithm +The algorithm that’s used to create the checksum for the object, one of + CRC64-NVME, CRC32, CRC32C, SHA1, + or SHA256. If no checksum is present, this value is null. For more + information, see Using supported checksum algorithms. +object_tags +The object tags that are associated with the object. Object tags are stored as a + map of key-value pairs. If an object has no object tags, an empty map + ({}) is stored. For more information, see Categorizing your storage using tags +If the record_type value is DELETE, the + object_tags column contains a null value. If the + record_type value is CREATE or + UPDATE_METADATA, rows that represent object versions that no longer + existed at the time that a delete or overwrite event was processed will contain a + null value in the object_tags column. +user_metadata +The user metadata that's associated with the object. User metadata is stored as a + map of key-value pairs. If an object has no user metadata, an empty map + ({}) is stored. For more information, see User-defined object metadata. +If the record_type value is DELETE, the + user_metadata column contains a null value. If the + record_type value is CREATE or + UPDATE_METADATA, rows that represent object versions that no longer + existed at the time that a delete or overwrite event was processed will contain a + null value in the user_metadata column. +requester +The AWS account ID of the requester or the AWS service principal that made the + request. +source_ip_address +The source IP address of the request. For records that are generated by a user request, this + column contains the source IP address of the request. For actions taken by Amazon S3 or + another AWS service on behalf of the user, this column contains a null value. +request_id +The request ID that's associated with the request. + Javascript is disabled or is unavailable in your browser. +To use the Amazon Web Services Documentation, Javascript must be enabled. Please refer to your browser's Help pages for instructions. +Thanks for letting us know we're doing a good job! +If you've got a moment, please tell us what we did right so we can do more of it. + +Thanks for letting us know this page needs work. We're sorry we let you down. +If you've got a moment, please tell us how we can make the documentation better. + diff --git a/crawl/crawled_data/Setting up Storage Browser for S3 - Amazon Simple Storage Service.txt b/crawl/crawled_data/Setting up Storage Browser for S3 - Amazon Simple Storage Service.txt new file mode 100644 index 00000000..f86ee0e4 --- /dev/null +++ b/crawl/crawled_data/Setting up Storage Browser for S3 - Amazon Simple Storage Service.txt @@ -0,0 +1,346 @@ +Title: Setting up Storage Browser for S3 - Amazon Simple Storage Service + +To connect end users with Amazon S3 locations, you must first set up an authentication and + authorization method. There are three methods to set up an authentication and authorization + method with Storage Browser: +Method 1: Managing data access for your + customers and third party partners +Method 2: Managing data access for your IAM + principals for your AWS account +Method 3: Managing data access at + scale +With this method, you can use AWS Amplify + Auth to manage access control and security for files. This method is ideal when + you want to connect your customers or third party partners with data in S3. With this + option, your customers can authenticate using social or enterprise identity + providers. +You provide IAM credentials to your end users and third party partners using AWS Amplify + Auth with an S3 bucket that’s configured to use Amplify Storage. AWS Amplify Auth is built + on Amazon Cognito, a fully + managed customer identity and access management service where you can authenticate and + authorize users from a built-in user directory or enterprise directory, or from consumer + identity providers. The Amplify authorization model defines which prefixes the current + authenticated user can access. For more information about how to set up authorization for + AWS Amplify, see Set up + storage. +To initialize the component with the Amplify authentication and storage methods, add the + following code snippet to your web application: +If you want to manage access for your IAM principals or your AWS account directly, + you can create an IAM role that has permissions to invoke the GetDataAccess S3 API operation. To set this up, you must create an + S3 Access Grants instance to map out permissions for S3 buckets and prefixes to the specified IAM + identities. The Storage Browser component will then invoke the ListCallerAccessGrants S3 API operation to fetch the available grants to the + identity requester and populate the locations in the component. After you obtain the + s3:GetDataAccess permission, those credentials are then used by the Storage + Browser component to request data access to S3. +If you want to associate an S3 Access Grants instance to your IAM + Identity Center for a more scalable solution (such as providing data access to your whole + company), you can request data from Amazon S3 on behalf of the current authenticated user. For + example, you can grant user groups in your corporate directory access to your data in S3. + This approach allows you to centrally manage S3 Access Grants permissions for your users and groups, + including the ones hosted on external providers such as Microsoft Entra, Okta, and + others. +When using this method, the integration with the + IAM Identity Center allows you to use existing user directories. Another benefit + of an IAM Identity Center trusted identity propagation is that each AWS CloudTrail data event for Amazon S3 contains a direct reference to the end user identity + that accessed the S3 data. +If you have an application that supports OAuth 2.0 and your users need access from these + applications to AWS services, we recommend that you use trusted identity propagation. With + trusted identity propagation, a user can sign in to an application, and that application can + pass the user’s identity in any requests that access data in AWS services. This + application interacts with IAM Identity Center on behalf of any authenticated users. For + more information, see Using trusted identity propagation with customer managed applications. +To set up Storage Browser authentication in the AWS Management Console using S3 Access Grants + and IAM Identity + Center trusted identity propagation, your applications must request data from + Amazon S3 on behalf of the current authenticated user. With this approach, you can give users + or groups of users from your corporate directory direct access to your S3 buckets, + prefixes, or objects. This means that your application won’t need to map any users to an + IAM principal. +The following workflow outlines the steps for setting up Storage Browser for S3, using IAM + Identity Center and S3 Access Grants: +The trusted token issuer represents your external identity provider (IdP) + within IAM Identity Center, enabling it to recognize identity tokens for your + application’s authenticated users. +This application interacts with IAM Identity Center on behalf of + authenticated users. +This step connects your application to S3 Access Grants, so that it can make requests + to S3 Access Grants on behalf of authenticated users. +This step syncs users from AWS Identity and Access Management Identity Center with the System for + Cross-domain Identity Management (SCIM). SCIM keeps your IAM Identity Center + identities in sync with identities from your identity provider (IdP). +To enable IAM Identity Center for your AWS Organizations, perform the following steps: +Sign in to the AWS Management Console, using one of these methods: +New to AWS (root user) + – Sign in as the account owner by choosing Root user and + entering your AWS account email address. On the next page, enter your + password. +Already using AWS (IAM + credentials) – Sign in using your IAM + credentials with administrative permissions. +Open the IAM Identity + Center console. +Under Enable IAM Identity Center, choose + Enable. +IAM Identity Center requires the setup of AWS Organizations. If you haven't set up an + organization, you can choose to have AWS create one for you. Choose + Create AWS organization to complete this process. +Choose Enable with AWS Organizations. +Choose Continue. +(Optional) Add any tags that you want to associate with this organization + instance. +(Optional) Configure the delegated administration. +If you’re using a multi-account environment, we recommend that you configure + delegated administration. With delegated administration, you can limit the number of + people who require access to the management account in AWS Organizations. For more + information, see Delegated + administration. +Choose Save. +AWS Organizations automatically sends a verification email to the address that is associated + with your management account. There might be a delay before you receive the verification + email. Make sure to verify your email address within 24 hours, before your verification + email expires. +To use Storage Browser for S3 with corporate directory users, you must configure IAM + Identity Center to use an external identity provider (IdP). You can use the preferred + identity provider of your choice. However, be aware that each identity provider uses + different configuration settings. For tutorials on using different external identity + providers, see IAM Identity Center source + tutorials. +Make sure to record the issuer URL and the audience attributes of the identity + provider that you’ve configured because you will need to refer to it in later steps. If + you don’t have the required access or permissions to configure an IdP, you might need to + contact the administrator of the external IdP to obtain them. +The trusted token issuer represents your external identity provider in the AWS Identity and Access Management + Identity Center, and recognizes tokens for your application’s authenticated users. The + account owner of the IAM Identity Center instance in your AWS Organizations must perform these + steps. These steps can be done either in the IAM Identity Center console, or + programmatically. +To add a trusted token issuer in the AWS Identity and Access Management Identity Center console, perform the + following steps: +Open the IAM Identity + Center console. +Choose Settings. +Choose the Authentication tab. +Navigate to the Trusted token issuers section, and fill out + the following details: +Under Issuer URL, enter the URL of the external IdP that + serves as the trusted token issuer. You might need to contact the administrator of + the external IdP to obtain this information. For more information, see Using applications with a trusted token issuer. +Under Trusted token issuer name, enter a name for the + trusted token issuer. This name will appear in the list of trusted token issuers + that you can select in Step 8, when an application resource is configured for + identity propagation. +Update your Map attributes to your preferred application + attribute, where each identity provider attribute is mapped to an IAM Identity + Center attribute. For example, you might want to map the + application attribute +email to the IAM Identity Center user attribute email. To + see the list of allowed user attributes in IAM Identity Center, see the table in + Attribute mappings + for AWS Managed Microsoft AD directory. +(Optional) If you want to add a resource tag, enter the key and value pair. To add + multiple resource tags, choose Add new tag to generate a new + entry and enter the key and value pairs. +Choose Create trusted token issuer. +After you finish creating the trusted token issuer, contact the application + administrator to let them know the name of the trusted token issuer, so that they can + confirm that the trusted token issuer is visible in the applicable console. +Make sure the application administrator selects this trusted token issuer in the + applicable console to enable user access to the application from applications that are + configured for trusted identity propagation. +To ensure that the bootstrap application and identity bearer + users can properly work with each other, make sure to create two IAM + roles. One IAM role is required for the bootstrap application and + the other IAM role must be used for the identity bearer, or end users who are accessing + the web application that requests access through S3 Access Grants. The bootstrap + application receives the token issued by the identity provider and invokes the + CreateTokenWithIAM API, exchanging this token with the one + issued by the Identity Center. +Create an IAM role, such as bootstrap-role, with permissions such as + the following. The following example IAM policy gives permissions to the + bootstrap-role to perform the token exchange: +Then, create a second IAM role (such as identity-bearer-role), which + the identity broker uses to generate the IAM credentials. The IAM credentials returned + from the identity broker to the web application are used by the Storage Browser for S3 component + to allow access to S3 data: +This IAM role (identity-bearer-role) must use a trust policy with the + following statement: +Before you begin, make sure that you’ve created the required IAM roles from the + previous step. You’ll need to specify one of these IAM roles in this step. +To create and configure a customer managed application in AWS IAM Identity Center, + perform the following steps: +Open the IAM Identity + Center console. +Choose Applications. +Choose the Customer managed tab. +Choose Add application. +On the Select application type page, under Setup + preference, choose I have an application I want to set + up. +Under Application type, choose OAuth + 2.0. +Choose Next. The Specify application + page is displayed. +Under the Application name and descriptionsection, enter a + Display name for the application, such as + storage-browser-oauth. +Enter a Description. The application description appears in + the IAM Identity Center console and API requests, but not in the AWS access + portal. +Under User and group assignment method, choose Do + not require assignments. This option allows all authorized IAM Identity + Center users and groups access to this application. +Under AWS access portal, enter an application URL where + users can access the application. +(Optional) If you want to add a resource tag, enter the key and value pair. To add + multiple resource tags, choose Add new tag to generate a new + entry and enter the key and value pairs. +Choose Next. The Specify authentication + page displays. +Under Authentication with trusted token issuer, use the + checkbox to select the trusted token issuer that you previously created. +Under Configure selected trusted token issuers, enter the + aud claim. The aud claim identifies the audience of + the JSON Web Token (JWT), and it is the name by which the trusted token issuer + identifies this application. +You might need to contact the administrator of the external IdP to obtain this + information. +Choose Next. The Specify authentication + credentials page displays. +Under Configuration method, choose Enter one or + more IAM roles. +Under Enter IAM roles, add the IAM role or Amazon Resource Name + (ARN) for the identity bearer token. You must enter the IAM role that you created from + the previous step for the identity broker application (for example, + bootstrap-role). +Choose Next. +On the Review and configure page, review the details of your + application configuration. If you need to modify any of the settings, choose + Edit for the section that you want to edit and make your + changes to. +Choose Submit. The details page of the application that you + just added is displayed. +After you’ve set up your applications, your users can access your applications from + within their AWS access portal based on the permission sets that you’ve created and the user + access that you’ve assigned. +After you set up your customer managed application, you must specify S3 Access Grants for + identity propagation. S3 Access Grants vends credentials for users to access Amazon S3 data. When you sign in to your customer + managed application, S3 Access Grants will pass your user identity to the trusted application. + +Prerequisite: Make sure that you’ve set up S3 Access Grants (such + as creating an S3 Access Grants + instance and registering a + location) before following these steps. For more information, see Getting started with S3 Access Grants. +To add S3 Access Grants for identity propagation to your customer managed application, perform + the following steps: +Open the IAM Identity + Center console. +Choose Applications. +Choose the Customer managed tab. +In the Customer managed applications list, select the OAuth + 2.0 application for which you want to initiate access requests. This is the + application that your users will sign in to. +On the Details page, under Trusted applications for + identity propagation, choose Specify trusted + applications. +Under Setup type, select Individual applications and specify access, and then + choose Next. +On the Select service page, choose + S3 Access Grants. S3 Access Grants has applications that you can use to define your + own web application for trusted identity propagation. +Choose Next. You'll select your applications in the next + step. +On the Select applications page, choose Individual + applications, select the checkbox for each application that can receive + requests for access, and then choose Next. +On the Configure access page, under Configuration + method, choose either of the following: +Select access per application – Select + this option to configure different access levels for each application. Choose the + application for which you want to configure the access level, and then choose + Edit access. In Level of access to + apply, change the access levels as needed, and then choose + Save changes. +Apply same level of access to all + applications – Select this option if you don't + need to configure access levels on a per-application basis. +Choose Next. +On the Review configuration page, review the choices that you + made. +You’ll want to make sure the s3:access_grants:read_write permission + is granted for your users. This permission allows your users to retrieve credentials + to access Amazon S3. Make sure to use either the IAM policy you created previously, or + S3 Access Grants, to limit access to write operations. +To make changes, choose Edit for the configuration section + that you want to make changes to. Then, make the required changes and choose + Save changes. +Choose Trust applications to add the trusted application for + identity propagation. +In this step, you use IAM Identity Center to provision your users. You can use SCIM + for automated or manual + provisioning of users and groups. SCIM keeps your IAM Identity Center + identities in sync with identities from your identity provider (IdP). This includes any + provisioning, updates, and deprovisioning of users between your IdP and IAM Identity + Center. +This step is required because when S3 Access Grants is used with IAM Identity Center, local + IAM Identity Center users aren’t used. Instead, users must be synchronized from the + identity provider with IAM Identity Center. +To synchronize users from your identity provider with IAM Identity Center, perform + the following steps: +Enable automatic + provisioning. +Generate an access token. +For examples of how to set up the identity provider with IAM Identity Center for + your specific use case, see + IAM + Identity Center Identity source tutorials. +After you’ve set up your IAM Identity Center instance and created grants in S3 Access Grants, + open your React application. In the React application, use + createManagedAuthAdapter to set up the authorization rules. You must + provide a credentials provider to return the credentials you acquired from IAM Identity + Center. You can then call createStorageBrowser to initialize the + Storage Browser for S3 component: + +Then, create a mechanism to exchange the JSON web tokens (JWTs) from your web + application with the IAM credentials from IAM Identity Center. For more information + about how to exchange the JWT, see the following resources: +How to develop a user-facing data application with IAM Identity Center and + S3 Access Grants post in AWS Storage Blog +Scaling data access with S3 Access Grants post in AWS Storage + Blog +S3 Access Grants workshop on AWS workshop studio +S3 Access Grants workshop on GitHub +Then, set up an API endpoint to handle requests for fetching credentials. To validate + the JSON web token (JWT) exchange, perform the following steps: +Retrieve the JSON web token from the authorization header for incoming + requests. +Validate the token using the public keys from the specified JSON web key set + (JWKS) URL. +Verify the token's expiration, issuer, subject, and audience claims. +To exchange the identity provider’s JSON web token with AWS IAM credentials, + perform the following steps: +Make sure to avoid logging any sensitive information. We recommend that you use + error handling controls for missing authorization, expired tokens, and other exceptions. + For more information, see the Implementing AWS Lambda error handling patterns post in AWS Compute + Blog. +Verify that the required Permission and + Scope parameters are provided in the request. +Use the CreateTokenWithIAM API to exchange the JSON web token for an IAM Identity + Center token. +After the IdP JSON web token is used, it can’t be used again. A new token must + be used to exchange with IAM Identity Center. +Use the AssumeRole API operation + to assume a transient role using the IAM Identity Center token. Make sure to use the + identity bearer role, also known as the role that carries the identity context (for + example, identity-bearer-role) to request the + credentials. +Return the IAM credentials to the web application. +Make sure that you’ve set up a proper logging mechanism. Responses are returned + in a standardized JSON format with an appropriate HTTP status code. + Javascript is disabled or is unavailable in your browser. +To use the Amazon Web Services Documentation, Javascript must be enabled. Please refer to your browser's Help pages for instructions. +Thanks for letting us know we're doing a good job! +If you've got a moment, please tell us what we did right so we can do more of it. + +Thanks for letting us know this page needs work. We're sorry we let you down. +If you've got a moment, please tell us how we can make the documentation better. + diff --git a/crawl/crawled_data/Setting up permissions for configuring metadata tables - Amazon Simple Storage Service.txt b/crawl/crawled_data/Setting up permissions for configuring metadata tables - Amazon Simple Storage Service.txt new file mode 100644 index 00000000..a602bde5 --- /dev/null +++ b/crawl/crawled_data/Setting up permissions for configuring metadata tables - Amazon Simple Storage Service.txt @@ -0,0 +1,45 @@ +Title: Setting up permissions for configuring metadata tables - Amazon Simple Storage Service + +The S3 Metadata feature is in preview release for Amazon S3 and is subject to change. +To create a metadata table configuration, you must have the necessary AWS Identity and Access Management (IAM) + permissions to both create and manage your metadata table configuration and to create and + manage your metadata table and the table bucket where your metadata table is stored. +To create and manage your metadata table configuration, you must have these permissions: +s3:CreateBucketMetadataTableConfiguration – This permission allows + you to create a metadata table configuration for your general purpose bucket. +s3:GetBucketMetadataTableConfiguration – This permission allows + you to retrieve information about your metadata table configuration. +s3:DeleteBucketMetadataTableConfiguration – This permission allows + you to delete your metadata table configuration. +To create and work with tables and table buckets, you must have certain + s3tables permissions. At a minimum, to create a metadata table configuration, + you must have the following s3tables permissions: +s3tables:CreateNamespace – This permission allows you to create a + namespace in a table bucket. Metadata tables use the default aws_s3_metadata + namespace. +s3tables:GetTable – This permission allows you to retrieve + information about your metadata table. +s3tables:CreateTable – This permission allows you to create your + metadata table. +s3tables:PutTablePolicy – This permission allows you to add or + update your metadata table policy. +For detailed information about all table and table bucket permissions, see Access + management for S3 Tables. +If you also want to integrate your table bucket with AWS analytics services so that you + can query your metadata table, you need additional permissions. For more information, see + + Integrating Amazon S3 Tables with AWS analytics services. +To create and work with metadata tables and table buckets, you can use the following + example policy. In this policy, the general purpose bucket that you're applying the metadata + table configuration to is referred to as amzn-s3-demo-source-bucket. The table + bucket where you're storing your metadata table is referred to as + amzn-s3-demo-bucket. To use this policy, replace these bucket names and the + user input placeholders with your own information: + Javascript is disabled or is unavailable in your browser. +To use the Amazon Web Services Documentation, Javascript must be enabled. Please refer to your browser's Help pages for instructions. +Thanks for letting us know we're doing a good job! +If you've got a moment, please tell us what we did right so we can do more of it. + +Thanks for letting us know this page needs work. We're sorry we let you down. +If you've got a moment, please tell us how we can make the documentation better. + diff --git a/crawl/crawled_data/Top DevOps Challenges and its SolutionsTop Challenges in DevOps and How to Solve Them | BrowserStack.txt b/crawl/crawled_data/Top DevOps Challenges and its SolutionsTop Challenges in DevOps and How to Solve Them | BrowserStack.txt new file mode 100644 index 00000000..ae517c70 --- /dev/null +++ b/crawl/crawled_data/Top DevOps Challenges and its SolutionsTop Challenges in DevOps and How to Solve Them | BrowserStack.txt @@ -0,0 +1,84 @@ +Title: Top DevOps Challenges and its SolutionsTop Challenges in DevOps and How to Solve Them | BrowserStack + + Transform your testing process with: Real Device Cloud, Company-wide Licences & Accessibility Testing +App & Browser Testing Made Easy +Give your users a seamless experience by testing on 3000+ real devices and browsers. Don't compromise with emulators and simulators + January 25, 2023 +The primary goal of DevOps is to make the overall SDLC workflow more efficient. Since DevOps comprehensively refers to the overall culture surrounding Software Development it needs to optimise the pipelines by looking at the people, process, and technology that makes up the software ecosystem. Factors such as cultural obstacles, poor communication in a team, legacy infrastructure, process failures, etc all lead to roadblocks in DevOps. This article will dive into some of the top challenges in DevOps and how they can be solved.  +Must Read : What is a DevOps Pipeline? How to Build One +During DevOps the ownership of the codebase changes from one team to another; from development to testing, to deployment and production. During such transitions there is a general waste of time and effort as the environments used by these teams are separately configured and the codebase must be modified to work smoothly on all these environments. This sometimes causes teams to also waste time trying to investigate for problems in the code rather than in the environment on which it is run. +Solutions +Creating infrastructural blueprints for Continuous Delivery enablement and making sure that all environments are identical. This usually requires all teams to get together and plan the Continuous Delivery process to ensure smooth transition.  +A good solution to enable this is to adopt a cloud-based infrastructure for DevOps enablement. The different parts of the DevOps pipeline, viz. coding, builds, testing, deployment, and post-production monitoring require different tools to enable and are hosted on different environments.  +Hosting these pipelines on the cloud helps to create a centralised architecture that helps different teams to access the codebase and continue to develop the pipeline as the cloud environment manages the environment transition.  + +Pro Tip : BrowserStack is an industry leader in cloud-based infrastructure for DevOps with over 3000 real browsers, devices and OS available at the touch of a button. It offers integrations with popular CI/CD tools like Jenkins, Travis CI, TeamCity, Bamboo, etc and facilitates automated testing by leveraging the power of Appium, Espresso, XCUITest, EarlGrey, Cypress, Playwright,  Puppeteer. +Run DevOps Tests on Real Devices +The level of maturity and competence a software engineering team has with the Software Development Life Cycle is directly related to their ability to be able to adapt to the DevOps transformation of these processes.  +One of the key reasons behind the adoption of DevOps is the need to deliver quality software in shorter deadlines in a more reliable fashion. The DevOps process seeks to transform the conventional Software Development process by creating a continuous cycle of Code-Build-Test and to better integrate the development and operations processes to be able to achieve its goal of quality software delivered on time.  +Solutions +Organisations adopting DevOps need to adopt the correct tools and technologies and should be able to invest in the correct training and upskilling for their employees. A robust devops culture requires the following steps :  + +Organisations adopt DevOps looking to orchestrate a transformation in their SDLC processes, and a major reason for this is their struggle to modernise legacy processes and practices.  +Organisations historically have worked in silos, with specific teams dedicated to performing certain tasks such as development, testing, and operations. These teams usually work in silos and have minimal communication with each other. It does not help that these teams work with outdated tools that do not allow for greater flexibility of communication and pipeline efficiency. +Solutions +Though DevOps pipelines depend on automation, some parts still require human communication and cooperation. Having transparency and cooperation in teams helps smoothen the pipelines and improve overall efficiency. +Read More: 5 DevOps Tools Every Team Must Have +BrowserStack Live for Teams allows teams to collaborate effectively by allowing seamless connections with existing tools and helping to track bugs. Issues can be filed in an organised manner and assigned to the relevant members on the same dashboard thus creating a unified approach to handle the entire DevOps pipeline.  +Try BrowserStack Live for Teams +One of the most common problems with DevOps is the challenge in holistically monitoring the entire process. DevOps consists of several moving parts and each of these have different metrics to judge their effectiveness.  +For example a metric like number of code branches or deployment frequency might deal with the CI/CD process; whereas something like Defect Escape Rate is a part of the Continuous Testing pipeline.  +Often there is lack of clear visibility over the entire process and that often leads to finger-pointing and delays in production. Any manual processes made to envisage this leads to extensive labour and risks incorrect updates due to human error.  +Solutions +Continuous Monitoring tools like Nagios Core can allow the continuous overview of applications, metrics, services, network components, etc. It allows users to have a birds eye view of the infrastructure and detect critical issues. It also offers useful altering services to resolve these issues and provides a great degree of scalability and flexibility. It also provides up to date logs of any DevOps events, failures, code crashes, and system outages.  +This automated viewport allows teams to exactly know what has gone wrong and where and subsequently make the changes required to correct it.  +A suboptimal implementation of the CI/CD pipeline leads to recurring issues such as slow page loads for websites, delayed responses from servers, and poor memory optimization that hampers the overall performance of the application.  +A Deloitte study indicates that suboptimal website performance increases the bounce rate and decreases overall time on the site. A mere 1 second delay can result in  +Solutions +Automated testing principles can be extended by the QA team to check for performance using tools like Apache JMeter. +Must Read : JMeter vs Selenium: What is preferred by Testers? +Tools like BrowserStack SpeedLab allow real-time web performance testing and scores the website under test on key performance metrics for an overall enhanced UX. +BrowserStack Speed Lab +This free tool allows the running of speed tests for monitoring the events occurring during page load including CPU Processing Breakdown, Navigation Timing API, and Page Resource Summary. It also allows the comparison of mobile and desktop screenshots for comparing render time to provide insights and optimise performance. +Performance Measurement using BrowserStack Speed Lab +Try SpeedLab for Free +DevOps relies on version control for all components and processes to work on stable versions. However, the unexpected update or change in any process can cause the entire pipeline to break due to compatibility issues. This can commonly occur during automated system updates. +Suboptimal automated tests are the perfect storm for any Continuous Automation process in the DevOps pipeline. Since DevOps focuses on delivering production ready code in short sprints, it is imperative to have the correct automation testing tools and techniques in place to avoid flawed tests. +Read More: DevOps Testing Strategy +Solutions +One of the most common solutions to this is to put a forced stop to all auto-updates to make sure that no software updates happen without manual intervention and that unstable versions are not incorporated just for the sake of it. +It is of the utmost importance that the QA team chooses the correct testing tools which are compatible with the systems and the languages being used.  Also model-based testing helps in furthering this process by proactively identifying red flags even before they cause major issues to the overall system.  +Run Automation Tests on Real Devices +Security vulnerabilities in the DevOps pipeline make it susceptible to cyber-attacks and can lead to sensitive information being compromised.  +Solutions +Some potential solutions are: +One of the major challenges in Continuous Testing is to be able to scale the operations and the test capabilities at the same time to handle increased volumes of data, devices, etc +Solutions  +Choosing a Selenium based test infrastructure that flexibility handles version updates, increased device load, and manages data capacity in a stable manner.  + +Several test reports are too complex to understand by all the stakeholders and this delays bug fixes thereby affecting product release velocity. +Solutions +Pro Tip: BrowserStack’s Test Insights offers an interactive dashboard to help obtain actionable insights. This helps teams to recognize critical issues or bottlenecks and accelerate product release velocity. +BrowserStack Test Insights +Summary +Though there are a number of challenges in DevOps, with the right tools and training they can all be overcome with ease. Regardless of the DevOps challenges being faced by the organisation, having a robust testing methodology that allows for accurate results on real browsers and devices is a must. +BrowserStack provides more than 3000+ real browsers and devices on a Cloud Selenium Grid with 99% uptime and assists the product team in many ways:  +Learn more about DevOps configuration management, its importance, and why it is beneficial for enter... +DevOps work towards making Software Release faster with high quality. DevOps recommend Shift Left Pr... +Are you looking for a way to improve your DevOps team's efficiency and effectiveness? Cloud-based so... +Over 6 million developers and 50,000 teams test on BrowserStack. Join them. +PRODUCTS +WHY BROWSERSTACK +RESOURCES +COMPANY +Social +More Resources +Test on Devices +Tools +© 2024 BrowserStack. All rights reserved. + +Our team will get in touch with you within 12 hours +Please share some details regarding your query +We will respond back shortly to +In the meantime, here are some resources that might interest you: +Meanwhile, these links might interest you: diff --git a/crawl/crawled_data/Tracking a multipart upload with the AWS SDKs - Amazon Simple Storage Service.txt b/crawl/crawled_data/Tracking a multipart upload with the AWS SDKs - Amazon Simple Storage Service.txt new file mode 100644 index 00000000..e1a759b3 --- /dev/null +++ b/crawl/crawled_data/Tracking a multipart upload with the AWS SDKs - Amazon Simple Storage Service.txt @@ -0,0 +1,28 @@ +Title: Tracking a multipart upload with the AWS SDKs - Amazon Simple Storage Service + +You can track an object's upload progress to Amazon S3 with a listen interface. The high-level + multipart upload API provides such a listen interface, called ProgressListener. + Progress events occur periodically and notify the listener that bytes have been transferred. + For more general information about multipart uploads, see Uploading and copying objects using multipart upload in Amazon S3. +For an end-to-end procedure on uploading an object with multipart upload with an additional checksum, see + Tutorial: Upload an object through multipart upload and + verify its data integrity. +The following section show how to track a multipart upload with the AWS SDKs. +The following Java code uploads a file and uses the ProgressListener + to track the upload progress. For instructions on how to create and test a + working sample, see Getting + Started in the AWS SDK for Java Developer Guide. + +The following C# example uploads a file to an S3 bucket using the + TransferUtility class, and tracks the progress of the upload. For + information about setting up and running the code examples, see Getting Started + with the AWS SDK for .NET in the AWS SDK for .NET Developer + Guide. + Javascript is disabled or is unavailable in your browser. +To use the Amazon Web Services Documentation, Javascript must be enabled. Please refer to your browser's Help pages for instructions. +Thanks for letting us know we're doing a good job! +If you've got a moment, please tell us what we did right so we can do more of it. + +Thanks for letting us know this page needs work. We're sorry we let you down. +If you've got a moment, please tell us how we can make the documentation better. + diff --git a/crawl/crawled_data/Troubleshooting Storage Browser for S3 - Amazon Simple Storage Service.txt b/crawl/crawled_data/Troubleshooting Storage Browser for S3 - Amazon Simple Storage Service.txt new file mode 100644 index 00000000..e427bb81 --- /dev/null +++ b/crawl/crawled_data/Troubleshooting Storage Browser for S3 - Amazon Simple Storage Service.txt @@ -0,0 +1,22 @@ +Title: Troubleshooting Storage Browser for S3 - Amazon Simple Storage Service + +If you’re experiencing issues with Storage Browser for S3, make sure to review the following troubleshooting tips: +Avoid trying to use the same token (idToken or accessToken) for multiple requests. Tokens can't be reused. This will result in a request failure. +Make sure that the IAM credentials that you provide to the Storage Browser component includes permissions to invoke the s3:GetDataAccess operation. Otherwise, your end users won’t be able to access your data. +Alternatively, you can check the following resources: +Storage Browser for S3 is backed by AWS Support. If you need assistance, contact the + AWS Support + Center. +If you’re having trouble with Storage Browser for S3 or would like to submit feedback, visit + the Amplify GitHub page. + +If you discover a potential security issue in this project, you can notify AWS + Security through the AWS Vulnerability Reporting page. + Javascript is disabled or is unavailable in your browser. +To use the Amazon Web Services Documentation, Javascript must be enabled. Please refer to your browser's Help pages for instructions. +Thanks for letting us know we're doing a good job! +If you've got a moment, please tell us what we did right so we can do more of it. + +Thanks for letting us know this page needs work. We're sorry we let you down. +If you've got a moment, please tell us how we can make the documentation better. + diff --git a/crawl/crawled_data/Uploading a directory using the high-level .NET TransferUtility class - Amazon Simple Storage Service.txt b/crawl/crawled_data/Uploading a directory using the high-level .NET TransferUtility class - Amazon Simple Storage Service.txt new file mode 100644 index 00000000..e08f173b --- /dev/null +++ b/crawl/crawled_data/Uploading a directory using the high-level .NET TransferUtility class - Amazon Simple Storage Service.txt @@ -0,0 +1,27 @@ +Title: Uploading a directory using the high-level .NET TransferUtility class - Amazon Simple Storage Service + +You can use the TransferUtility class to upload an entire directory. By + default, the API uploads only the files at the root of the specified directory. You can, + however, specify recursively uploading files in all of the sub directories. +To select files in the specified directory based on filtering criteria, specify + filtering expressions. For example, to upload only the PDF files from a directory, + specify the "*.pdf" filter expression. +When uploading files from a directory, you don't specify the key names for the + resulting objects. Amazon S3 constructs the key names using the original file path. For + example, assume that you have a directory called c:\myfolder with the + following structure: +When you upload this directory, Amazon S3 uses the following key names: +The following C# example uploads a directory to an Amazon S3 bucket. It shows how to + use various TransferUtility.UploadDirectory overloads to upload the + directory. Each successive call to upload replaces the previous upload. For + information about setting up and running the code examples, see Getting Started + with the AWS SDK for .NET in the AWS SDK for .NET Developer + Guide. + Javascript is disabled or is unavailable in your browser. +To use the Amazon Web Services Documentation, Javascript must be enabled. Please refer to your browser's Help pages for instructions. +Thanks for letting us know we're doing a good job! +If you've got a moment, please tell us what we did right so we can do more of it. + +Thanks for letting us know this page needs work. We're sorry we let you down. +If you've got a moment, please tell us how we can make the documentation better. + diff --git a/crawl/crawled_data/Uploading and copying objects using multipart upload in Amazon S3 - Amazon Simple Storage Service.txt b/crawl/crawled_data/Uploading and copying objects using multipart upload in Amazon S3 - Amazon Simple Storage Service.txt new file mode 100644 index 00000000..06134949 --- /dev/null +++ b/crawl/crawled_data/Uploading and copying objects using multipart upload in Amazon S3 - Amazon Simple Storage Service.txt @@ -0,0 +1,344 @@ +Title: Uploading and copying objects using multipart upload in Amazon S3 - Amazon Simple Storage Service + +Multipart upload allows you to upload a single object to Amazon S3 as a set of parts. Each part is a + contiguous portion of the object's data. You can upload these object parts independently, and in + any order. For uploads, your updated AWS client automatically calculates a checksum of the + object and sends it to Amazon S3 along with the size of the object as a part of the request. If + transmission of any part fails, you can retransmit that part without affecting other parts. + After all parts of your object are uploaded, Amazon S3 assembles them to create the object. It's a + best practice to use multipart upload for objects that are 100 MB or larger instead of uploading them in a + single operation. +Using multipart upload provides the following advantages: +Improved throughput – You can upload parts in + parallel to improve throughput. +Quick recovery from any network issues – Smaller + part size minimizes the impact of restarting a failed upload due to a network error. +Pause and resume object uploads – You can upload + object parts over time. After you initiate a multipart upload, there is no expiry; you must + explicitly complete or stop the multipart upload. +Begin an upload before you know the final object size + – You can upload an object as you create it. +We recommend that you use multipart upload in the following ways: +If you upload large objects over a stable high-bandwidth network, use multipart upload + to maximize the use of your available bandwidth by uploading object parts in parallel for + multi-threaded performance. +If you upload over a spotty network, use multipart upload to increase resiliency against + network errors by avoiding upload restarts. When using multipart upload, you only need to + retry uploading the parts that are interrupted during the upload. You don't need to restart + uploading your object from the beginning. +For more information about using the Amazon S3 Express One Zone storage class with directory buckets, + see S3 Express One Zone and + Working with directory buckets. For more + information about using multipart upload with S3 Express One Zone and directory buckets, see + Using multipart uploads with + directory buckets. +Multipart upload is a three-step process: You initiate the upload, upload the object parts, + and—after you've uploaded all the parts—complete the multipart upload. Upon receiving the + complete multipart upload request, Amazon S3 constructs the object from the uploaded parts, and you can access + the object just as you would any other object in your bucket. +You can list all of your in-progress multipart uploads or get a list of the parts that you + have uploaded for a specific multipart upload. Each of these operations is explained in this + section. +When you send a request to initiate a multipart upload, make sure to specify a checksum type. Amazon S3 + will then return a response with an upload ID, which is a unique identifier for your multipart upload. + This upload ID is required when you upload parts, list parts, complete an upload, or stop an + upload. If you want to provide metadata describing the object being uploaded, you must + provide it in the request to initiate the multipart upload. +When uploading a part, you must specify a part number in addition to the upload ID. You + can choose any part number between 1 and 10,000. A part number uniquely identifies a part + and its position in the object you are uploading. The part number that you choose doesn’t + need to be in a consecutive sequence (for example, it can be 1, 5, and 14). Be aware that if + you upload a new part using the same part number as a previously uploaded part, the + previously uploaded part gets overwritten. +When you upload a part, Amazon S3 returns the checksum algorithm type with the checksum value + for each part as a header in the response. For each part upload, you must record the part + number and the ETag value. You must include these values in the subsequent request to complete + the multipart upload. Each part will have its own ETag at the time of upload. However, once the multipart upload is + complete and all parts are consolidated, all parts belong to one ETag as a checksum of + checksums. +After you initiate a multipart upload and upload one or more parts, you must either complete or + stop the multipart upload to stop incurring charges for storage of the uploaded parts. Only + after you complete or stop a multipart upload will Amazon S3 free up the parts + storage and stop billing you for the parts storage. +After stopping a multipart upload, you can't upload any part using that upload ID again. If part + uploads were in progress, they can still succeed or fail even after you stop the upload. To + make sure you free all storage consumed by all parts, you must stop a multipart upload only after all + part uploads have completed. +When you complete a multipart upload, Amazon S3 creates an object by concatenating the parts in + ascending order based on the part number. If any object metadata was provided in the + initiate multipart upload request, Amazon S3 associates that metadata with + the object. After a successful complete request, the parts no longer + exist. +Your complete multipart upload request must include the upload ID and a list of + part numbers and their corresponding ETag values. The Amazon S3 response includes an ETag that + uniquely identifies the combined object data. This ETag is not necessarily an MD5 hash of the + object data. +When you provide a full object checksum during a multipart upload, the AWS SDK passes the checksum to + Amazon S3, and S3 validates the object integrity server-side, comparing it to the received value. + Then, S3 stores the object if the values match. If the two values don’t match, Amazon S3 fails the + request with a BadDigest error. The checksum of your object is also stored in + object metadata that you'll later use to validate an object's data integrity. + For this example, assume that you're generating a multipart upload for a 100 GB file. + In this case, you would have the following API calls for the entire process. There would be + a total of 1,002 API calls. +A CreateMultipartUpload call to start the process. +1,000 individual UploadPart calls, each + uploading a part of 100 MB, for a total size of 100 GB. +A CompleteMultipartUpload call to complete the process. +You can list the parts of a specific multipart upload or all in-progress multipart uploads. The + list parts operation returns the parts information that you uploaded for a specific multipart upload. + For each list parts request, Amazon S3 returns the parts information for the specified multipart upload, up + to a maximum of 1,000 parts. If there are more than 1,000 parts in the multipart upload, you must send + a series of list part requests to retrieve all of the parts. Note that the returned list of + parts doesn't include parts that haven't finished uploading. Using the list + multipart uploads operation, you can obtain a list of multipart uploads that + are in progress. +An in-progress multipart upload is an upload that you have initiated, but have not yet completed or + stopped. Each request returns at most 1,000 multipart uploads. If there are more than 1,000 + multipart uploads in progress, you must send additional requests to retrieve the remaining + multipart uploads. Use the returned listing only for verification. +Do not use the result of + this listing when sending a complete multipart upload request. Instead, + maintain your own list of the part numbers that you specified when uploading parts and the + corresponding ETag values that Amazon S3 returns. +When you upload an object to Amazon S3, you can specify a checksum algorithm for Amazon S3 to use. + By default, the AWS SDK and S3 console use an algorithm for all object uploads, which you can + override. If you’re using an older SDK and your uploaded object doesn’t have a specified + checksum, Amazon S3 automatically uses the CRC-64NVME checksum algorithm. (This is + also the recommended option for efficient data integrity verification.) When using + CRC-64NVME, Amazon S3 calculates the checksum of the full object after the multipart + or single part upload is complete. The CRC-64NVME checksum algorithm is used to + calculate either a direct checksum of the entire object, or a checksum of the checksums, for + each individual part. +After you upload an object to S3 using multipart upload, Amazon S3 calculates the checksum valuefor each + part, or for the full object—and stores the values. You can use the S3 API or AWS SDK to + retrieve the checksum value in the following ways: +For individual parts, you can use GetObject or HeadObject. If you want to retrieve the checksum values for + individual parts of multipart uploads while they're still in process, you can use ListParts. +For the entire object, you can use PutObject. If you want + to perform a multipart upload with a full object checksum, use CreateMultipartUpload and CompleteMultipartUpload by specifying the full object checksum + type. To validate the checksum value of the entire object or to confirm which checksum + type is being used in the multipart upload, use ListParts. +If you're using a multipart upload with Checksums, the part numbers for each part upload + (in the multipart upload) must use consecutive part numbers. When using Checksums, + if you try to complete a multipart upload request with nonconsecutive part numbers, Amazon S3 generates an + HTTP 500 Internal Server error. + For more information about how checksums work with multipart upload objects, see Checking object integrity in Amazon S3. +For an end-to-end procedure that demonstrates how to upload an object using multipart upload with an + additional checksum, see Tutorial: Upload an object through multipart upload and + verify its data integrity. +In a distributed development environment, it is possible for your application to initiate + several updates on the same object at the same time. Your application might initiate several + multipart uploads using the same object key. For each of these uploads, your application can + then upload parts and send a complete upload request to Amazon S3 to create the object. When the + buckets have S3 Versioning enabled, completing a multipart upload always creates a new version. When you + initiate multiple multipart uploads that use the same object key in a versioning-enabled + bucket, the current version of the object is determined by which upload started most recently + (createdDate). +For example, you start a CreateMultipartUpload request for an object at 10:00 + AM. Then, you submit a second CreateMultipartUpload request for the same object + at 11:00 AM. Because the second request was submitted the most recently, the object uploaded + by the 11:00 AM request becomes the current version, even if the first upload is completed + after the second one. For buckets that don't have versioning enabled, it's possible that any + other request received between the time when the multipart upload is initiated and when it completes, the + other request might take precedence. +Another example of when a concurrent multipart upload request can take precedence is if another operation deletes a key after + you initiate a multipart upload with that key. Before you complete the operation, the complete multipart upload response + might indicate a successful object creation without you ever seeing the object. +You can check for the existence of an object in your bucket before creating it using a + conditional write on upload operations. This can prevent overwrites of existing data. Conditional writes + will validate that there is no existing object with the same key name already in your bucket + while uploading. +You can use conditional writes for PutObject or CompleteMultipartUpload requests. +For more information about conditional requests see, Add preconditions to S3 operations with conditional requests. +After you initiate a multipart upload, Amazon S3 retains all the parts until you either + complete or stop the upload. Throughout its lifetime, you are billed for all storage, + bandwidth, and requests for this multipart upload and its associated parts. +These parts are billed according to the storage class specified when the parts are + uploaded. However, you will not be billed for these parts if they're uploaded to + S3 Glacier Flexible Retrieval or S3 Glacier Deep Archive. In-progress multipart + parts + for a PUT request to the S3 Glacier Flexible Retrieval storage class are billed as + S3 Glacier Flexible Retrieval staging storage at S3 Standard storage rates until the upload + completes. In addition, both CreateMultipartUpload and UploadPart + are billed at S3 Standard rates. Only the CompleteMultipartUpload request is + billed at the S3 Glacier Flexible Retrieval rate. Similarly, in-progress multipart parts for a + PUT to the S3 Glacier Deep Archive storage class are billed as S3 Glacier Flexible Retrieval staging + storage at S3 Standard storage rates until the upload completes, with only the + CompleteMultipartUpload request charged at S3 Glacier Deep Archive rates. +If you stop the multipart upload, Amazon S3 deletes upload artifacts and all parts that you uploaded. You + will not be billed for those artifacts. There are no early delete charges for deleting + incomplete multipart uploads regardless of storage class specified. For more information about + pricing, see Amazon S3 pricing. +To minimize your storage costs, we recommend that you configure a lifecycle rule to + delete incomplete multipart uploads after a specified number of days by using the + AbortIncompleteMultipartUpload action. For more information about creating a + lifecycle rule to delete incomplete multipart uploads, see Configuring a bucket + lifecycle configuration to delete incomplete multipart uploads. +The following + sections in the Amazon Simple Storage Service API Reference describe the REST API for + multipart upload. +For a multipart upload walkthrough that uses AWS Lambda functions, see Uploading large objects to Amazon S3 using multipart upload and transfer acceleration. +Create Multipart + Upload +Upload Part +Upload Part (Copy) +Complete Multipart + Upload +Abort Multipart Upload +List Parts +List Multipart + Uploads +The following topics in the AWS Command Line Interface describe the operations for multipart upload. +Initiate Multipart + Upload +Upload Part +Upload Part + (Copy) +Complete Multipart + Upload +Abort Multipart + Upload +List Parts +List Multipart + Uploads + +You can use an AWS SDKs to upload an object in parts. For a list of AWS SDKs + supported by API action see: +Create Multipart + Upload +Upload Part +Upload Part (Copy) +Complete Multipart + Upload +Abort Multipart Upload +List Parts +List Multipart + Uploads +You must have the necessary permissions to use the multipart upload operations. You can use access + control lists (ACLs), the bucket policy, or the user policy to grant individuals permissions + to perform these operations. The following table lists the required permissions for various + multipart upload operations when using ACLs, a bucket policy, or a user policy. +Create Multipart Upload +You must be allowed to perform the s3:PutObject action on an object + to create a multipart upload request. +The bucket owner can allow other principals to perform the + s3:PutObject action. +Initiate Multipart Upload +You must be allowed to perform the s3:PutObject action on an object + to initiate a multipart upload. +The bucket owner can allow other principals to perform the + s3:PutObject action. +Container element that identifies who initiated the multipart upload. If the initiator + is an AWS account, this element provides the same information as the Owner + element. If the initiator is an IAM user, this element provides the user ARN and + display name. +You must be allowed to perform the s3:PutObject action on an + object to upload a part. +The bucket owner must allow the initiator to perform the + s3:PutObject action on an object in order for the initiator to upload + a part for that object. +You must be allowed to perform the s3:PutObject action on an + object to upload a part. Because you are uploading a part from an existing object, + you must be allowed s3:GetObject on the source object. +For the initiator to upload a part for an object, the owner of the bucket must + allow the initiator to perform the s3:PutObject action on the + object. +You must be allowed to perform the s3:PutObject action on an + object to complete a multipart upload. +The bucket owner must allow the initiator to perform the + s3:PutObject action on an object in order for the initiator to + complete a multipart upload for that object. +You must be allowed to perform the s3:AbortMultipartUpload + action to stop a multipart upload. +By default, the bucket owner and the initiator of the multipart upload are allowed to + perform this action as a part of IAM and S3 bucket polices. If the initiator is an + IAM user, that user's AWS account is also allowed to stop that multipart upload. With VPC + endpoint policies, the initiator of the multipart upload doesn't automatically gain the + permission to perform the s3:AbortMultipartUpload action. +In addition to these defaults, the bucket owner can allow other principals to + perform the s3:AbortMultipartUpload action on an object. The bucket + owner can deny any principal the ability to perform the + s3:AbortMultipartUpload action. +You must be allowed to perform the s3:ListMultipartUploadParts + action to list parts in a multipart upload. +By default, the bucket owner has permission to list parts for any multipart upload to the + bucket. The initiator of the multipart upload has the permission to list parts of the specific + multipart upload. If the multipart upload initiator is an IAM user, the AWS account controlling that + IAM user also has permission to list parts of that upload. + In addition to these defaults, the bucket owner can allow other principals to + perform the s3:ListMultipartUploadParts action on an object. The bucket + owner can also deny any principal the ability to perform the + s3:ListMultipartUploadParts action. +You must be allowed to perform the + s3:ListBucketMultipartUploads action on a bucket to list multipart + uploads in progress to that bucket. +In addition to the default, the bucket owner can allow other principals to + perform the s3:ListBucketMultipartUploads action on the + bucket. +To perform a multipart upload with encryption using an AWS Key Management Service (AWS KMS) KMS key, the + requester must have the following permissions: +The kms:Decrypt and kms:GenerateDataKey actions on + the key. +The kms:GenerateDataKey action for the CreateMultipartUpload API. +The kms:Decrypt action on the UploadPart and UploadPartCopy APIs. + These permissions are required because Amazon S3 must decrypt and read data from the + encrypted file parts before it completes the multipart upload. The kms:Decrypt + permission, and the server-side encryption with customer-provided encryption keys, are also required for you to obtain an object's + checksum value. If you don't have these required permissions when you use the CompleteMultipartUpload API, the object is created + without a checksum value. +If your IAM user or role is in the same AWS account as the KMS key, then + you must have these permissions on the key policy. If your IAM user or role + belongs to a different account than the KMS key, then you must have the + permissions on both the key policy and your IAM user or role. +When you use the CompleteMultipartUpload API, you must provide the SSE-C (server-side encryption with customer-provided encryption keys), or your object will be created without a checksum, and no checksum value is returned. +For information on the relationship between ACL permissions and permissions in access + policies, see Mapping of ACL permissions and access policy + permissions. For information about IAM users, + roles, and best practices, see IAM identities (users, user groups, and roles) in the IAM User Guide. +There are three Amazon S3 APIs that are used to perform the actual multipart upload: CreateMultipartUpload, UploadPart, and CompleteMultipartUpload. The following table indicates which + checksum headers and values must be provided for each of the APIs: +CRC-64NVME +x-amz-checsum-algorithm +Optional headers: +x-amz-checksum-CRC64nvme +Optional headers: +x-amz-checksum-algorithm +x-amz-crc64 +CRC-32 +CRC-32C +Required headers: +x-amz-checksum-algorithm +x-amz-checksum-type +Optional headers: +x-amz-checksum-crc64nvme +Optional headers: +x-amz-checksum-algorithm +x-amz-crc32 +x-amz-crc32c +CRC-32 +CRC-32C +SHA-1 +SHA-256 +Required headers: +x-amz-checksum-algorithm +Required headers: +x-amz-checksum-crc32 +x-amz-checksum-crc32c +x-amz-checksum-sha1 +x-amz-checksum-sha256 +Required headers: +All part-level checksums need to be included in the CompleteMultiPartUpload request. +Optional headers: +x-amz-crc32 +x-amz-crc32c +x-amz-sha1 +x-amz-sha256 + Javascript is disabled or is unavailable in your browser. +To use the Amazon Web Services Documentation, Javascript must be enabled. Please refer to your browser's Help pages for instructions. +Thanks for letting us know we're doing a good job! +If you've got a moment, please tell us what we did right so we can do more of it. + +Thanks for letting us know this page needs work. We're sorry we let you down. +If you've got a moment, please tell us how we can make the documentation better. + diff --git a/crawl/crawled_data/Using Storage Browser for S3 - Amazon Simple Storage Service.txt b/crawl/crawled_data/Using Storage Browser for S3 - Amazon Simple Storage Service.txt new file mode 100644 index 00000000..56228757 --- /dev/null +++ b/crawl/crawled_data/Using Storage Browser for S3 - Amazon Simple Storage Service.txt @@ -0,0 +1,35 @@ +Title: Using Storage Browser for S3 - Amazon Simple Storage Service + +In Storage Browser for S3, a location is an S3 bucket or prefix, that you + grant end users access to using S3 Access Grants, IAM policies, or your + own managed authorization service. After you've authorized your end users to access a specific + location, they can work with any data within that location. +The Storage Browser for S3 user interface has four main views: +Home page: The home page lists the S3 locations that + your users can access, as well as your permissions for each. This is the initial view for + users that shows the root level S3 resources that your end users have access to and the + permissions (READ/WRITE/READWRITE) for each S3 location. +Location details: This view allows users to browse + files and folders in S3, and upload or download files. (Note that in Storage Browser for S3, + objects are known as files, and prefixes and + buckets are known as folders.) +Location action: After a user chooses an action (such as + Upload) in Storage Browser, it opens up another view of the + file location. +Vertical ellipsis: The vertical ellipsis icon opens + the drop-down list of actions. +When using Storage Browser for S3, be aware of the following limitations: +Folders starting or ending with dots (.) aren’t supported. +S3 Access Grants with WRITE only permission isn't supported. +Storage Browser for S3 supports the PUT operation for files up to 160 GB in + size. +Storage Browser for S3 only supports the COPY operation for files smaller than 5 GB. If the file size + exceeds 5 GB, Storage Browser fails the request. + Javascript is disabled or is unavailable in your browser. +To use the Amazon Web Services Documentation, Javascript must be enabled. Please refer to your browser's Help pages for instructions. +Thanks for letting us know we're doing a good job! +If you've got a moment, please tell us what we did right so we can do more of it. + +Thanks for letting us know this page needs work. We're sorry we let you down. +If you've got a moment, please tell us how we can make the documentation better. + diff --git a/crawl/crawled_data/Virtual hosting of buckets - Amazon Simple Storage Service.txt b/crawl/crawled_data/Virtual hosting of buckets - Amazon Simple Storage Service.txt new file mode 100644 index 00000000..07469966 --- /dev/null +++ b/crawl/crawled_data/Virtual hosting of buckets - Amazon Simple Storage Service.txt @@ -0,0 +1,229 @@ +Title: Virtual hosting of buckets - Amazon Simple Storage Service + +Virtual hosting is the practice of serving multiple websites from a single web server. One + way to differentiate sites in your Amazon S3 REST API requests is by using the apparent hostname + of the Request-URI instead of just the path name part of the URI. An ordinary Amazon S3 REST + request specifies a bucket by using the first slash-delimited component of the Request-URI + path. Instead, you can use Amazon S3 virtual hosting to address a bucket in a REST API call by + using the HTTP Host header. In practice, Amazon S3 interprets Host as + meaning that most buckets are automatically accessible for limited types of requests at + https://bucket-name.s3.region-code.amazonaws.com. + For a complete list of Amazon S3 Regions and endpoints, see Amazon S3 endpoints and quotas in the Amazon Web Services General Reference. +Virtual hosting also has other benefits. By naming your bucket after your registered + domain name and by making that name a DNS alias for Amazon S3, you can completely customize the + URL of your Amazon S3 resources, for example, + http://my.bucket-name.com/. You can + also publish to the "root directory" of your bucket's virtual server. This ability can be + important because many existing applications search for files in this standard location. For + example, favicon.ico, robots.txt, and + crossdomain.xml are all expected to be found at the root. +When you're using virtual-hosted–style buckets with SSL, the SSL wildcard certificate + matches only buckets that do not contain dots (.). To work around this + limitation, use HTTP or write your own certificate-verification logic. For more + information, see Amazon S3 Path + Deprecation Plan on the AWS News + Blog. +Currently, Amazon S3 supports both virtual-hosted–style and path-style URL access in all AWS Regions. However, path-style + URLs will be discontinued in the future. For more information, see the following Important note. +In Amazon S3, path-style URLs use the following format: +For example, if you create a bucket named amzn-s3-demo-bucket1 in the US West (Oregon) Region, + and you want to access the puppy.jpg object in that bucket, you can use the + following path-style URL: +Update (September 23, 2020) – To make sure that customers have the time that they need to transition to virtual-hosted–style URLs, +we have decided to delay the deprecation of path-style URLs. For more information, +see Amazon S3 Path Deprecation Plan – The Rest of the Story in the AWS News Blog. +When hosting website content that will be accessed from a web browser, avoid using + path-style URLs, which might interfere with the browser same origin security model. + To host website content, we recommend that you use either S3 website endpoints or a + CloudFront distribution. For more information, see Website endpoints and Deploy a React-based single-page application to Amazon S3 and CloudFront in the + AWS Perspective Guidance Patterns. +In a virtual-hosted–style URI, the bucket name is part of the domain name in the + URL. +Amazon S3 virtual-hosted–style URLs use the following format: +In this example, amzn-s3-demo-bucket1 is the bucket name, US West (Oregon) is the Region, and puppy.png is the key name: +As long as your GET request does not use the SSL endpoint, you can + specify the bucket for the request by using the HTTP Host header. The + Host header in a REST request is interpreted as follows: +If the Host header is omitted or its value is + s3.region-code.amazonaws.com, the + bucket for the request will be the first slash-delimited component of the + Request-URI, and the key for the request will be the rest of the Request-URI. + This is the ordinary method, as illustrated by the first and second examples in + this section. Omitting the Host header is valid only for HTTP 1.0 + requests. +Otherwise, if the value of the Host header ends in + .s3.region-code.amazonaws.com, + the bucket name is the leading component of the Host header's value + up to .s3.region-code.amazonaws.com. The + key for the request is the Request-URI. This interpretation exposes buckets as + subdomains of + .s3.region-code.amazonaws.com, as + illustrated by the third and fourth examples in this section. +Otherwise, the bucket for the request is the lowercase value of the + Host header, and the key for the request is the Request-URI. + This interpretation is useful when you have registered the same DNS name as your + bucket name and have configured that name to be a canonical name (CNAME) alias + for Amazon S3. The procedure for registering domain names and configuring CNAME DNS + records is beyond the scope of this guide, but the result is illustrated by the + final example in this section. +This section provides example URLs and requests. +This example uses the following: +Bucket Name ‐ example.com +Region ‐ US East (N. Virginia) +Key Name ‐ homepage.html +The URL is as follows: +The request is as follows: +The request with HTTP 1.0 and omitting the Host header is as + follows: +For information about DNS-compatible names, see Limitations. For more information about + keys, see Keys. +This example uses the following: +Bucket name ‐ + amzn-s3-demo-bucket1 + +Region ‐ Europe (Ireland) +Key name ‐ + homepage.html +The URL is as follows: +The request is as follows: +To use this method, you must configure your DNS name as a CNAME alias for + bucket-name.s3.us-east-1.amazonaws.com. + For more information, see Customizing Amazon S3 URLs with CNAME + records. +This example uses the following: +Bucket Name ‐ example.com + +Key name ‐ + homepage.html +The URL is as follows: +The example is as follows: +Depending on your needs, you might not want + s3.region-code.amazonaws.com to appear on + your website or service. For example, if you're hosting website images on Amazon S3, you + might prefer http://images.example.com/ instead of + http://images.example.com.s3.us-east-1.amazonaws.com/. Any bucket + with a DNS-compatible name can be referenced as follows: + http://BucketName.s3.Region.amazonaws.com/[Filename], + for example, + http://images.example.com.s3.us-east-1.amazonaws.com/mydog.jpg. + By using CNAME, you can map images.example.com to an Amazon S3 + hostname so that the previous URL could become + http://images.example.com/mydog.jpg. +Your bucket name must be the same as the CNAME. For example, if you create a CNAME to + map images.example.com to + images.example.com.s3.us-east-1.amazonaws.com, both + http://images.example.com/filename and + http://images.example.com.s3.us-east-1.amazonaws.com/filename + will be the same. +The CNAME DNS record should alias your domain name to the appropriate virtual + hosted–style hostname. For example, if your bucket name and domain name are + images.example.com and your bucket is in the + US East (N. Virginia) Region, the CNAME record should alias to + images.example.com.s3.us-east-1.amazonaws.com. +Amazon S3 uses the hostname to determine the bucket name. So the CNAME and the bucket name + must be the same. For example, suppose that you have configured + www.example.com as a CNAME for + www.example.com.s3.us-east-1.amazonaws.com. When you access + http://www.example.com, Amazon S3 receives a request similar to + the following: +Amazon S3 sees only the original hostname www.example.com and is + unaware of the CNAME mapping used to resolve the request. +You can use any Amazon S3 endpoint in a CNAME alias. For example, + s3.ap-southeast-1.amazonaws.com can be used in CNAME aliases. + For more information about endpoints, see Request + Endpoints in the Amazon S3 API Reference. To create a static website by using a custom domain, see Tutorial: Configuring a static website using a + custom domain registered with Route 53. +When using custom URLs with CNAMEs, you will need to ensure a matching bucket + exists for any CNAME or alias record you configure. For example, if you create DNS + entries for www.example.com and + login.example.com to publish web content using S3, you + will need to create both buckets www.example.com and + login.example.com. +When a CNAME or alias records is configured pointing to an S3 endpoint without a + matching bucket, any AWS user can create that bucket and publish content under the + configured alias, even if ownership is not the same. +For the same reason, we recommend that you change or remove the corresponding + CNAME or alias when deleting a bucket. +Select a hostname that belongs to a domain that you control. +This example uses the images subdomain of the + example.com domain. +Create a bucket that matches the hostname. +In this example, the host and bucket names are + images.example.com. The bucket name must exactly match the hostname. +Create a CNAME DNS record that defines the hostname as an alias for the Amazon S3 + bucket. +For example: +images.example.com CNAME + images.example.com.s3.us-west-2.amazonaws.com +For request-routing reasons, the CNAME DNS record must be defined exactly + as shown in the preceding example. Otherwise, it might appear to operate + correctly, but it will eventually result in unpredictable behavior. +The procedure for configuring CNAME DNS records depends on your DNS server or + DNS provider. For specific information, see your server documentation or contact + your provider. + + SOAP support over HTTP is deprecated, but SOAP is still available over HTTPS. + New Amazon S3 features are not supported for SOAP. Instead of using SOAP, we recommend that you use + either the REST API or the AWS SDKs. + +The following sections cover various aspects of Amazon S3 backward compatibility that + relate to path-style and virtual-hosted–style URL requests. +Some Regions support legacy endpoints. You might see these endpoints in your + server access logs or AWS CloudTrail logs. For more information, review the following + information. For a complete list of Amazon S3 Regions and endpoints, see Amazon S3 endpoints and quotas in the Amazon Web Services General Reference. +Although you might see legacy endpoints in your logs, we recommend that you + always use the standard endpoint syntax to access your buckets. +Amazon S3 virtual-hosted–style URLs use the following format: +In Amazon S3, path-style URLs use the following format: +Some older Amazon S3 Regions support endpoints that contain a dash (-) + between s3 and the Region code (for example, + s3‐us-west-2), instead of a dot (for example, + s3.us-west-2). If your bucket is in one of these Regions, you + might see the following endpoint format in your server access logs or CloudTrail + logs: +In this example, the bucket name is amzn-s3-demo-bucket1 and the + Region is US West (Oregon): +For some Regions, you can use the legacy global endpoint to construct requests + that do not specify a Region-specific endpoint. The legacy global endpoint point + is as follows: +In your server access logs or CloudTrail logs, you might see requests that use the + legacy global endpoint. In this example, the bucket name is + amzn-s3-demo-bucket1 and the legacy global + endpoint is: +Requests made with the legacy global endpoint go to the US East (N. Virginia) + Region by default. Therefore, the legacy global endpoint is sometimes used + in place of the Regional endpoint for US East (N. Virginia). If you create a + bucket in US East (N. Virginia) and use the global endpoint, Amazon S3 routes your + request to this Region by default. +The legacy global endpoint is also used for virtual-hosted–style + requests in other supported Regions. If you create a bucket in a Region that + was launched before March 20, 2019, and use the legacy global endpoint, Amazon S3 + updates the DNS record to reroute the request to the correct location, which + might take time. In the meantime, the default rule applies, and your + virtual-hosted–style request goes to the US East (N. Virginia) Region. + Amazon S3 then redirects it with an HTTP 307 Temporary Redirect to the correct + Region. +For S3 buckets in Regions launched after March 20, 2019, the DNS server + doesn't route your request directly to the AWS Region where your bucket + resides. It returns an HTTP 400 Bad Request error instead. For more information, + see Making requests in the Amazon S3 API + Reference. +For the US East (N. Virginia) Region, you can use the legacy global endpoint + for path-style requests. +For all other Regions, the path-style syntax requires that you use the + Region-specific endpoint when attempting to access a bucket. If you try to + access a bucket with the legacy global endpoint or another endpoint that is + different than the one for the Region where the bucket resides, you receive an + HTTP response code 307 Temporary Redirect error and a message that indicates the + correct URI for your resource. For example, if you use + https://s3.amazonaws.com/bucket-name + for a bucket that was created in the US West (Oregon) Region, you will receive + an HTTP 307 Temporary Redirect error. + Javascript is disabled or is unavailable in your browser. +To use the Amazon Web Services Documentation, Javascript must be enabled. Please refer to your browser's Help pages for instructions. +Thanks for letting us know we're doing a good job! +If you've got a moment, please tell us what we did right so we can do more of it. + +Thanks for letting us know this page needs work. We're sorry we let you down. +If you've got a moment, please tell us how we can make the documentation better. + diff --git a/crawl/crawled_data/What is Amazon S3? - Amazon Simple Storage Service.txt b/crawl/crawled_data/What is Amazon S3? - Amazon Simple Storage Service.txt new file mode 100644 index 00000000..5229223e --- /dev/null +++ b/crawl/crawled_data/What is Amazon S3? - Amazon Simple Storage Service.txt @@ -0,0 +1,431 @@ +Title: What is Amazon S3? - Amazon Simple Storage Service + +Amazon Simple Storage Service (Amazon S3) is an object storage service that offers industry-leading scalability, + data availability, security, and performance. Customers of all sizes and industries can use + Amazon S3 to store and protect any amount of data for a range of use cases, such as data lakes, + websites, mobile applications, backup and restore, archive, enterprise applications, IoT + devices, and big data analytics. Amazon S3 provides management features so that you can optimize, + organize, and configure access to your data to meet your specific business, organizational, + and compliance requirements. +For more information about using the Amazon S3 Express One Zone storage class with directory buckets, see S3 Express One Zone and Working with directory buckets. +Amazon S3 offers a range of storage classes designed for different use cases. For + example, you can store mission-critical production data in S3 Standard or S3 Express One Zone for frequent + access, save costs by storing infrequently accessed data in S3 Standard-IA or + S3 One Zone-IA, and archive data at the lowest costs in S3 Glacier Instant Retrieval, + S3 Glacier Flexible Retrieval, and S3 Glacier Deep Archive. +Amazon S3 Express One Zone is a high-performance, single-zone Amazon S3 storage class that is purpose-built + to deliver consistent, single-digit millisecond data access for your most + latency-sensitive applications. S3 Express One Zone is the lowest latency cloud object + storage class available today, with data access + speeds + up to 10x faster and with request costs + 50 + percent lower than S3 Standard. S3 Express One Zone is the first S3 storage class where you can select a single Availability Zone with +the option to co-locate your object storage with your compute resources, which provides the highest possible access speed. + Additionally, to further increase access speed and support hundreds of thousands of + requests per second, data is stored in a new bucket type: an + Amazon S3 directory bucket. For more information, see S3 Express One Zone and Working with directory buckets. +You can store data with changing or unknown access patterns in + S3 Intelligent-Tiering, which optimizes storage costs by automatically moving your + data between four access tiers when your access patterns change. These four access + tiers include two low-latency access tiers optimized for frequent and infrequent + access, and two opt-in archive access tiers designed for asynchronous access for + rarely accessed data. +For more information, see Understanding and managing Amazon S3 storage classes. +Amazon S3 has storage management features that you can use to manage costs, meet + regulatory requirements, reduce latency, and save multiple distinct copies of your + data for compliance requirements. +S3 Lifecycle – Configure a lifecycle configuration to manage + your objects and store them cost effectively throughout their lifecycle. You + can transition objects to other S3 storage classes or expire objects that + reach the end of their lifetimes. +S3 Object Lock – Prevent Amazon S3 objects from being + deleted or overwritten for a fixed amount of time or indefinitely. You can + use Object Lock to help meet regulatory requirements that require write-once-read-many +(WORM) storage or to simply add another + layer of protection against object changes and deletions. +S3 Replication + – Replicate objects and their respective metadata and object tags to + one or more destination buckets in the same or different AWS Regions for + reduced latency, compliance, security, and other use cases. +S3 Batch Operations – Manage billions of objects at scale + with a single S3 API request or a few clicks in the Amazon S3 console. You can + use Batch Operations to perform operations such as Copy, Invoke AWS Lambda + function, and Restore on + millions or billions of objects. +Amazon S3 provides features for auditing and managing access to your buckets and + objects. By default, S3 buckets and the objects in them are private. You have access + only to the S3 resources that you create. To grant granular resource permissions + that support your specific use case or to audit the permissions of your Amazon S3 + resources, you can use the following features. +S3 Block Public Access – Block public access to S3 + buckets and objects. By default, Block Public Access settings are turned on + at the bucket level. We recommend that you keep all Block Public Access + settings enabled unless you know that you need to turn off one or more of + them for your specific use case. For more information, see Configuring block public access + settings for your S3 buckets. +AWS Identity and Access Management (IAM) – IAM is a web service that helps + you securely control access to AWS resources, including your Amazon S3 + resources. With IAM, you can centrally manage permissions that control + which AWS resources users can access. You use IAM to control who is + authenticated (signed in) and authorized (has permissions) to use + resources. +Bucket + policies – Use IAM-based policy language to configure + resource-based permissions for your S3 buckets and the objects in + them. + +Amazon S3 access points + – Configure named network endpoints with dedicated access policies to + manage data access at scale for shared datasets in Amazon S3. +Access control + lists (ACLs) – Grant read and write permissions for + individual buckets and objects to authorized users. As a general rule, we + recommend using S3 resource-based policies (bucket policies and access point + policies) or IAM user policies for access control instead of ACLs. + Policies are a simplified and more flexible access control option. With + bucket policies and access point policies, you can define rules that apply + broadly across all requests to your Amazon S3 resources. For more information + about the specific cases when you'd use ACLs instead of resource-based + policies or IAM user policies, see Managing access with ACLs. +S3 Object Ownership – Take ownership of every object + in your bucket, simplifying access management for data stored in Amazon S3. + S3 Object Ownership is an Amazon S3 bucket-level setting that you can use to + disable or enable ACLs. By default, ACLs are disabled. With ACLs disabled, + the bucket owner owns all the objects in the bucket and manages access to + data exclusively by using access-management policies. +IAM Access Analyzer for S3 + – Evaluate and monitor your S3 bucket access policies, ensuring that + the policies provide only the intended access to your S3 resources. +To transform data and trigger workflows to automate a variety of other processing + activities at scale, you can use the following features. +S3 Object Lambda + – Add your own code to S3 GET, HEAD, and LIST requests to modify and process data as + it is returned to an application. Filter rows, dynamically resize images, + redact confidential data, and much more. +Event + notifications – Trigger workflows that use Amazon Simple Notification Service + (Amazon SNS), Amazon Simple Queue Service (Amazon SQS), and AWS Lambda when a change is made to your S3 + resources. +Amazon S3 provides logging and monitoring tools that you can use to monitor and control + how your Amazon S3 resources are being used. For more information, see Monitoring + tools. +Amazon CloudWatch + metrics for Amazon S3 – Track the operational health of your + S3 resources and configure billing alerts when estimated charges reach a + user-defined threshold. +AWS CloudTrail + – Record actions taken by a user, a role, or an AWS service in + Amazon S3. CloudTrail logs provide you with detailed API tracking for S3 bucket-level + and object-level operations. +Server access + logging – Get detailed records for the requests that are + made to a bucket. You can use server access logs for many use cases, such as + conducting security and access audits, learning about your customer base, + and understanding your Amazon S3 bill. +AWS Trusted + Advisor – Evaluate your account by using AWS best + practice checks to identify ways to optimize your AWS infrastructure, + improve security and performance, reduce costs, and monitor service quotas. + You can then follow the recommendations to optimize your services and + resources. +Amazon S3 offers features to help you gain visibility into your storage usage, which + empowers you to better understand, analyze, and optimize your storage at + scale. +Amazon S3 Storage Lens + – Understand, analyze, and optimize your storage. S3 Storage Lens provides + 60+ usage and activity metrics and interactive dashboards to aggregate data + for your entire organization, specific accounts, AWS Regions, buckets, or + prefixes. +Storage + Class Analysis – Analyze storage access patterns to + decide when it's time to move data to a more cost-effective storage class. + +S3 Inventory with + Inventory reports – Audit and report on objects and their + corresponding metadata and configure other Amazon S3 features to take action in + Inventory reports. For example, you can report on the replication and + encryption status of your objects. For a list of all the metadata available + for each object in Inventory reports, see Amazon S3 Inventory list. +Amazon S3 provides strong read-after-write consistency for PUT and DELETE requests of + objects in your Amazon S3 bucket in all AWS Regions. This behavior applies to both + writes of new objects as well as PUT requests that overwrite existing objects and + DELETE requests. In addition, read operations on Amazon S3 Select, Amazon S3 access control + lists (ACLs), Amazon S3 Object Tags, and object metadata (for example, the HEAD object) + are strongly consistent. For more information, see Amazon S3 data consistency model. +Amazon S3 is an object storage service that stores data as objects within buckets. An + object is a file and any metadata that describes + the file. A bucket is a container for objects. +To store your data in Amazon S3, you first create a bucket and specify a bucket name and + AWS Region. Then, you upload your data to that bucket as objects in Amazon S3. Each object + has a key (or key + name), which is the unique identifier for the object within the + bucket. +S3 provides features that you can configure to support your specific use case. For + example, you can use S3 Versioning to keep multiple versions of an object in the same + bucket, which allows you to restore objects that are accidentally deleted or + overwritten. +Buckets and the objects in them are private and can be accessed only if you explicitly + grant access permissions. You can use bucket policies, AWS Identity and Access Management (IAM) policies, + access control lists (ACLs), and S3 Access Points to manage access. +A general purpose bucket is a container for objects stored in Amazon S3. You can store any number of + objects in a bucket and all accounts have a default bucket quota of 10,000 general purpose buckets. To see your bucket utilization, bucket quota, or request an + increase to this quota, visit the Service Quotas + console. +Every object is contained in a bucket. For example, if the object named + photos/puppy.jpg is stored in the + amzn-s3-demo-bucket bucket in the US West (Oregon) + Region, then it is addressable by using the URL + https://amzn-s3-demo-bucket.s3.us-west-2.amazonaws.com/photos/puppy.jpg. + For more information, see Accessing a + Bucket. +When you create a bucket, you enter a bucket name and choose the AWS Region + where the bucket will reside. After you create a bucket, you cannot change the name + of the bucket or its Region. Bucket names must follow the bucket naming rules. You can also configure a bucket to use S3 Versioning or other storage management + features. + Buckets also: +Organize the Amazon S3 namespace at the highest level. +Identify the account responsible for storage and data transfer + charges. +Provide access control options, such as bucket policies, access control + lists (ACLs), and S3 Access Points, that you can use to manage access to + your Amazon S3 resources. +Serve as the unit of aggregation for usage reporting. + For more information about buckets, see Buckets overview. +Objects are the fundamental entities stored in Amazon S3. Objects consist of object + data and metadata. The metadata is a set of name-value pairs that describe the + object. These pairs include some default metadata, such as the date last modified, + and standard HTTP metadata, such as Content-Type. You can also specify + custom metadata at the time that the object is stored. +An object is uniquely identified within a bucket by a key (name) and a version ID (if + S3 Versioning is enabled on the bucket). For more information about objects, see + Amazon S3 objects overview. +An object key (or key + name) is the unique identifier for an object within a bucket. Every + object in a bucket has exactly one key. The combination of a bucket, object key, and + optionally, version ID (if S3 Versioning is enabled for the bucket) uniquely identify + each object. So you can think of Amazon S3 as a basic data map between "bucket + key + + version" and the object itself. +Every object in Amazon S3 can be uniquely addressed through the combination of the web service + endpoint, bucket name, key, and optionally, a version. For example, in the URL + https://amzn-s3-demo-bucket.s3.us-west-2.amazonaws.com/photos/puppy.jpg, + amzn-s3-demo-bucket is the name of the bucket + and photos/puppy.jpg is the key. +For more information about object keys, see Naming Amazon S3 objects. +You can use S3 Versioning to keep multiple variants of an object in the same + bucket. With S3 Versioning, you can preserve, retrieve, and restore every version of + every object stored in your buckets. You can easily recover from both unintended + user actions and application failures. +For more information, see Retaining multiple versions of objects with S3 Versioning. +When you enable S3 Versioning in a bucket, Amazon S3 generates a unique version ID for + each object added to the bucket. Objects that already existed in the bucket at the + time that you enable versioning have a version ID of null. If you + modify these (or any other) objects with other operations, such as CopyObject and PutObject, the new objects + get a unique version ID. +For more information, see Retaining multiple versions of objects with S3 Versioning. +A bucket policy is a resource-based AWS Identity and Access Management (IAM) policy that you can use to + grant access permissions to your bucket and the objects in it. Only the bucket owner + can associate a policy with a bucket. The permissions attached to the bucket apply + to all of the objects in the bucket that are owned by the bucket owner. Bucket + policies are limited to 20 KB in size. +Bucket policies use JSON-based access policy language that is standard across + AWS. You can use bucket policies to add or deny permissions for the objects in a + bucket. Bucket policies allow or deny requests based on the elements in the policy, + including the requester, S3 actions, resources, and aspects or conditions of the + request (for example, the IP address used to make the request). For example, you can + create a bucket policy that grants cross-account permissions to upload objects to an + S3 bucket while ensuring that the bucket owner has full control of the uploaded + objects. For more information, see Examples of Amazon S3 bucket policies. +In your bucket policy, you can use wildcard characters on Amazon Resource Names + (ARNs) and other values to grant permissions to a subset of objects. For example, + you can control access to groups of objects that begin with a common prefix or end with a given extension, such as + .html. +Amazon S3 Access Points are named network endpoints with dedicated access policies that + describe how data can be accessed using that endpoint. Access Points are attached to + buckets that you can use to perform S3 object operations, such as GetObject and + PutObject. Access Points simplify managing data access at scale for shared datasets + in Amazon S3. +Each access point has its own access point policy. You can configure Block Public Access settings + for each access point. To restrict Amazon S3 data access to a private network, you can + also configure any access point to accept requests only from a virtual private cloud + (VPC). +For more information, see Managing access to shared datasets with access points. +You can use ACLs to grant read and write permissions to authorized users for + individual buckets and objects. Each bucket and object has an ACL attached to it as + a subresource. The ACL defines which AWS accounts or groups are granted access and + the type of access. ACLs are an access control mechanism that predates IAM. For + more information about ACLs, see Access control list (ACL) overview. +S3 Object Ownership is an Amazon S3 bucket-level setting that you can use to both control ownership of the objects that are + uploaded to your bucket and to disable or enable ACLs. By default, Object Ownership is set to the Bucket owner enforced setting, + and all ACLs are disabled. When ACLs are disabled, the bucket owner owns all the objects in the bucket and manages access to them + exclusively by using access-management policies. + A majority of modern use cases in Amazon S3 no longer require the use of ACLs. We recommend that you keep ACLs disabled, except + in unusual circumstances where you need to control access for each object individually. With ACLs disabled, you can use policies + to control access to all objects in your bucket, regardless of who uploaded the objects to your bucket. + For more information, see Controlling ownership of objects and disabling ACLs + for your bucket. +You can choose the geographical AWS Region where Amazon S3 stores the buckets that + you create. You might choose a Region to optimize latency, minimize costs, or + address regulatory requirements. Objects stored in an AWS Region never leave the + Region unless you explicitly transfer or replicate them to another Region. For example, objects stored in the Europe (Ireland) Region + never leave it. + +You can access Amazon S3 and its features only in the AWS Regions that are + enabled for your account. For more information about enabling a Region to create + and manage AWS resources, see Managing AWS Regions in + the AWS General Reference. +For a list of Amazon S3 Regions and endpoints, see Regions and endpoints in the + AWS General Reference. +Amazon S3 provides strong read-after-write consistency for PUT and DELETE requests of + objects in your Amazon S3 bucket in all AWS Regions. This behavior applies to both writes + to new objects as well as PUT requests that overwrite existing objects and DELETE + requests. In addition, read operations on Amazon S3 Select, Amazon S3 access controls lists + (ACLs), Amazon S3 Object Tags, and object metadata (for example, the HEAD object) are + strongly consistent. +Updates to a single key are atomic. For example, if you make a PUT request to an + existing key from one thread and perform a GET request on the same key from a second + thread concurrently, you will get either the old data or the new data, but never partial + or corrupt data. +Amazon S3 achieves high availability by replicating data across multiple servers within + AWS data centers. If a PUT request is successful, your data is safely stored. Any read + (GET or LIST request) that is initiated following the receipt of a successful PUT + response will return the data written by the PUT request. Here are examples of this + behavior: +A process writes a new object to Amazon S3 and immediately lists keys within its + bucket. The new object appears in the list. +A process replaces an existing object and immediately tries to read it. Amazon S3 + returns the new data. +A process deletes an existing object and immediately tries to read it. Amazon S3 + does not return any data because the object has been deleted. +A process deletes an existing object and immediately lists keys within its + bucket. The object does not appear in the listing. +Amazon S3 does not support object locking for concurrent writers. If two PUT + requests are simultaneously made to the same key, the request with the + latest timestamp wins. If this is an issue, you must build an object-locking + mechanism into your application. +Updates are key-based. There is no way to make atomic updates across keys. + For example, you cannot make the update of one key dependent on the update + of another key unless you design this functionality into your + application. +Bucket configurations have an eventual consistency model. Specifically, this means + that: +If you delete a bucket and immediately list all buckets, the deleted bucket + might still appear in the list. +If you enable versioning on a bucket for the first time, it might take a short + amount of time for the change to be fully propagated. We recommend that you wait + for 15 minutes after enabling versioning before issuing write operations (PUT or + DELETE requests) on objects in the bucket. +This section provides examples of behavior to be expected from Amazon S3 when multiple + clients are writing to the same items. +In this example, both W1 (write 1) and W2 (write 2) finish before the start of R1 + (read 1) and R2 (read 2). Because S3 is strongly consistent, R1 and R2 both return + color = ruby. +In the next example, W2 does not finish before the start of R1. Therefore, R1 + might return color = ruby or color = garnet. However, + because W1 and W2 finish before the start of R2, R2 returns color = + garnet. +In the last example, W2 begins before W1 has received an acknowledgment. + Therefore, these writes are considered concurrent. Amazon S3 internally uses + last-writer-wins semantics to determine which write takes precedence. However, the + order in which Amazon S3 receives the requests and the order in which applications + receive acknowledgments cannot be predicted because of various factors, such as + network latency. For example, W2 might be initiated by an Amazon EC2 instance in the same + Region, while W1 might be initiated by a host that is farther away. The best way to + determine the final value is to perform a read after both writes have been + acknowledged. +After you load your data into Amazon S3, you can use it with other AWS services. The + following are the services that you might use most frequently: +Amazon Elastic Compute Cloud + (Amazon EC2) – Provides secure and scalable computing + capacity in the AWS Cloud. Using Amazon EC2 eliminates your need to invest in + hardware upfront, so you can develop and deploy applications faster. You can + use Amazon EC2 to launch as many or as few virtual servers as you need, configure + security and networking, and manage storage. +Amazon EMR – Helps businesses, researchers, data + analysts, and developers easily and cost-effectively process vast amounts of + data. Amazon EMR uses a hosted Hadoop framework running on the web-scale + infrastructure of Amazon EC2 and Amazon S3. +AWS Snow + Family – Helps customers that need to run + operations in austere, non-data center environments, and in locations where + there's a lack of consistent network connectivity. You can use AWS Snow Family + devices to locally and cost-effectively access the storage and compute power of + the AWS Cloud in places where an internet connection might not be an option. + +AWS Transfer Family – Provides fully managed support for + file transfers directly into and out of Amazon S3 or Amazon Elastic File System (Amazon EFS) using Secure + Shell (SSH) File Transfer Protocol (SFTP), File Transfer Protocol over SSL + (FTPS), and File Transfer Protocol (FTP). +You can work with Amazon S3 in any of the following ways: +The console is a web-based user interface for managing Amazon S3 and AWS resources. + If you've signed up for an AWS account, you can access the Amazon S3 console by signing + into the AWS Management Console and choosing S3 from the AWS Management Console home + page. +You can use the AWS command line tools to issue commands or build scripts at + your system's command line to perform AWS (including S3) tasks. +The AWS Command Line Interface (AWS CLI) provides commands + for a broad set of AWS services. The AWS CLI is supported on Windows, macOS, and + Linux. To get started, see the AWS Command Line Interface User Guide. For more information about the commands for + Amazon S3, see s3api and s3control in the AWS CLI Command Reference. +AWS provides SDKs (software development kits) that consist of libraries and sample code + for various programming languages and platforms (Java, Python, Ruby, .NET, iOS, + Android, and so on). The AWS SDKs provide a convenient way to create programmatic + access to S3 and AWS. Amazon S3 is a REST service. You can send requests to Amazon S3 using + the AWS SDK libraries, which wrap the underlying Amazon S3 REST API and simplify your + programming tasks. For example, the SDKs take care of tasks such as calculating + signatures, cryptographically signing requests, managing errors, and retrying + requests automatically. For information about the AWS SDKs, including how to + download and install them, see Tools for + AWS. +Every interaction with Amazon S3 is either authenticated or anonymous. If you are using + the AWS SDKs, the libraries compute the signature for authentication from the keys + that you provide. For more information about how to make requests to Amazon S3, see Making requests + . +The architecture of Amazon S3 is designed to be programming language-neutral, using + AWS-supported interfaces to store and retrieve objects. You can access S3 and + AWS programmatically by using the Amazon S3 REST API. The REST API is an HTTP interface + to Amazon S3. With the REST API, you use standard HTTP requests to create, fetch, and + delete buckets and objects. +To use the REST API, you can use any toolkit that supports HTTP. You can even use + a browser to fetch objects, as long as they are anonymously readable. +The REST API uses standard HTTP headers and status codes, so that standard + browsers and toolkits work as expected. In some areas, we have added functionality + to HTTP (for example, we added headers to support access control). In these cases, + we have done our best to add the new functionality in a way that matches the style + of standard HTTP usage. +If you make direct REST API calls in your application, you must write the code to + compute the signature and add it to the request. For more information about how to + make requests to Amazon S3, see Making requests + in the Amazon S3 API Reference. +SOAP API support over HTTP is deprecated, but it is still available over + HTTPS. Newer Amazon S3 features are not supported for SOAP. We recommend that you use + either the REST API or the AWS SDKs. +Pricing for Amazon S3 is designed so that you don't have to plan for the storage + requirements of your application. Most storage providers require you to purchase a + predetermined amount of storage and network transfer capacity. In this scenario, if you + exceed that capacity, your service is shut off or you are charged high overage fees. If + you do not exceed that capacity, you pay as though you used it all. +Amazon S3 charges you only for what you actually use, with no hidden fees and no overage + charges. This model gives you a variable-cost service that can grow with your business + while giving you the cost advantages of the AWS infrastructure. For more information, + see Amazon S3 Pricing. +When you sign up for AWS, your AWS account is automatically signed up for all + services in AWS, including Amazon S3. However, you are charged only for the services that + you use. If you are a new Amazon S3 customer, you can get started with Amazon S3 for free. For + more information, see AWS free tier. +To see your bill, go to the Billing and Cost Management Dashboard in the AWS Billing and Cost Management console. To learn more about AWS account billing, see the AWS Billing User Guide. If you have + questions concerning AWS billing and AWS accounts, contact AWS Support. +Amazon S3 supports the processing, storage, and transmission +of credit card data by a merchant or service provider, and has been +validated as being compliant with Payment Card Industry (PCI) Data Security Standard (DSS). +For more information about PCI DSS, including how to request a copy of the AWS PCI Compliance Package, +see PCI DSS Level 1. + + Javascript is disabled or is unavailable in your browser. +To use the Amazon Web Services Documentation, Javascript must be enabled. Please refer to your browser's Help pages for instructions. +Thanks for letting us know we're doing a good job! +If you've got a moment, please tell us what we did right so we can do more of it. + +Thanks for letting us know this page needs work. We're sorry we let you down. +If you've got a moment, please tell us how we can make the documentation better. + diff --git a/crawl/main.py b/crawl/main.py index 3a4621e3..19c72eb4 100644 --- a/crawl/main.py +++ b/crawl/main.py @@ -4,8 +4,55 @@ # List of URLs to crawl urls = [ - "https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/concepts.html", - "https://docs.aws.amazon.com/ec2/latest/instancetypes/instance-types.html#current-gen-instances" + "https://www.browserstack.com/guide/devops-challenges-and-its-solutions", + + "https://sematext.com/blog/devops-challenges/", + + "https://docs.aws.amazon.com/AmazonS3/latest/userguide/Welcome.html", + + "https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingBucket.html", + + "https://docs.aws.amazon.com/AmazonS3/latest/userguide/common-bucket-patterns.html", + + "https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html", + + "https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html", + + "https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html", + + "https://docs.aws.amazon.com/AmazonS3/latest/userguide/mountpoint-installation.html", + + "https://docs.aws.amazon.com/AmazonS3/latest/userguide/mountpoint-usage.html", + + "https://docs.aws.amazon.com/AmazonS3/latest/userguide/using-storagebrowser.html", + + "https://docs.aws.amazon.com/AmazonS3/latest/userguide/installing-storagebrowser.html", + + "https://docs.aws.amazon.com/AmazonS3/latest/userguide/setup-storagebrowser.html", + + "https://docs.aws.amazon.com/AmazonS3/latest/userguide/s3config-storagebrowser.html", + + "https://docs.aws.amazon.com/AmazonS3/latest/userguide/troubleshooting-storagebrowser.html", + + "https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html", + + "https://docs.aws.amazon.com/AmazonS3/latest/userguide/metadata-tables-restrictions.html", + + "https://docs.aws.amazon.com/AmazonS3/latest/userguide/metadata-tables-schema.html", + + "https://docs.aws.amazon.com/AmazonS3/latest/userguide/metadata-tables-permissions.html", + + "https://docs.aws.amazon.com/AmazonS3/latest/userguide/metadata-tables-optimizing-query-performance.html", + + "https://docs.aws.amazon.com/AmazonS3/latest/userguide/metadata-tables-example-queries.html", + + "https://docs.aws.amazon.com/AmazonS3/latest/userguide/metadata-tables-join-custom-metadata.html", + + "https://docs.aws.amazon.com/AmazonS3/latest/userguide/mpuoverview.html", + + "https://docs.aws.amazon.com/AmazonS3/latest/userguide/HLuploadDirDotNet.html", + + "https://docs.aws.amazon.com/AmazonS3/latest/userguide/track-mpu.html" ] # Directory to save the files From 4ea75c6c3bf43e1533c7e1392c27a815dde09b39 Mon Sep 17 00:00:00 2001 From: abolfazl1381 Date: Sun, 15 Dec 2024 11:17:20 +0330 Subject: [PATCH 20/22] fix(jenkins install): add None to jenkins installation model --- app/media/MyAnsible/group_vars/all | 1 + app/models/jenkins.py | 2 +- crawl/main.py | 50 +----------------------------- 3 files changed, 3 insertions(+), 50 deletions(-) diff --git a/app/media/MyAnsible/group_vars/all b/app/media/MyAnsible/group_vars/all index eb422e93..3be1741d 100644 --- a/app/media/MyAnsible/group_vars/all +++ b/app/media/MyAnsible/group_vars/all @@ -1,3 +1,4 @@ + # General install_ansible_modules: "true" disable_transparent_huge_pages: "true" diff --git a/app/models/jenkins.py b/app/models/jenkins.py index e84ca17e..116bcce9 100644 --- a/app/models/jenkins.py +++ b/app/models/jenkins.py @@ -5,7 +5,7 @@ class JenkinsInstallation(BaseModel): - os: str = 'Ubuntu' + os: str | None = 'Ubuntu' environment:str = 'Linux' diff --git a/crawl/main.py b/crawl/main.py index 19c72eb4..ce378269 100644 --- a/crawl/main.py +++ b/crawl/main.py @@ -4,55 +4,7 @@ # List of URLs to crawl urls = [ - "https://www.browserstack.com/guide/devops-challenges-and-its-solutions", - - "https://sematext.com/blog/devops-challenges/", - - "https://docs.aws.amazon.com/AmazonS3/latest/userguide/Welcome.html", - - "https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingBucket.html", - - "https://docs.aws.amazon.com/AmazonS3/latest/userguide/common-bucket-patterns.html", - - "https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html", - - "https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html", - - "https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html", - - "https://docs.aws.amazon.com/AmazonS3/latest/userguide/mountpoint-installation.html", - - "https://docs.aws.amazon.com/AmazonS3/latest/userguide/mountpoint-usage.html", - - "https://docs.aws.amazon.com/AmazonS3/latest/userguide/using-storagebrowser.html", - - "https://docs.aws.amazon.com/AmazonS3/latest/userguide/installing-storagebrowser.html", - - "https://docs.aws.amazon.com/AmazonS3/latest/userguide/setup-storagebrowser.html", - - "https://docs.aws.amazon.com/AmazonS3/latest/userguide/s3config-storagebrowser.html", - - "https://docs.aws.amazon.com/AmazonS3/latest/userguide/troubleshooting-storagebrowser.html", - - "https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html", - - "https://docs.aws.amazon.com/AmazonS3/latest/userguide/metadata-tables-restrictions.html", - - "https://docs.aws.amazon.com/AmazonS3/latest/userguide/metadata-tables-schema.html", - - "https://docs.aws.amazon.com/AmazonS3/latest/userguide/metadata-tables-permissions.html", - - "https://docs.aws.amazon.com/AmazonS3/latest/userguide/metadata-tables-optimizing-query-performance.html", - - "https://docs.aws.amazon.com/AmazonS3/latest/userguide/metadata-tables-example-queries.html", - - "https://docs.aws.amazon.com/AmazonS3/latest/userguide/metadata-tables-join-custom-metadata.html", - - "https://docs.aws.amazon.com/AmazonS3/latest/userguide/mpuoverview.html", - - "https://docs.aws.amazon.com/AmazonS3/latest/userguide/HLuploadDirDotNet.html", - - "https://docs.aws.amazon.com/AmazonS3/latest/userguide/track-mpu.html" + ] # Directory to save the files From 33713f8d499dae1064424ecfa6b6fbb9edf24400 Mon Sep 17 00:00:00 2001 From: abolfazl1381 Date: Mon, 16 Dec 2024 20:10:29 +0330 Subject: [PATCH 21/22] remove crawler form project --- ... Issues and Their Solutions - Sematext.txt | 63 --- ...Amazon EC2 instance types - Amazon EC2.txt | 79 ---- ...erview - Amazon Simple Storage Service.txt | 225 --------- ...zon S3 - Amazon Simple Storage Service.txt | 65 --- ...for S3 - Amazon Simple Storage Service.txt | 15 - ...tpoint - Amazon Simple Storage Service.txt | 117 ----- ...bucket - Amazon Simple Storage Service.txt | 226 --------- ...ueries - Amazon Simple Storage Service.txt | 40 -- ... rules - Amazon Simple Storage Service.txt | 115 ----- ...tpoint - Amazon Simple Storage Service.txt | 94 ---- ...for S3 - Amazon Simple Storage Service.txt | 15 - ...tables - Amazon Simple Storage Service.txt | 21 - ...ctions - Amazon Simple Storage Service.txt | 60 --- ...bjects - Amazon Simple Storage Service.txt | 123 ----- ...rmance - Amazon Simple Storage Service.txt | 20 - ...schema - Amazon Simple Storage Service.txt | 146 ------ ...for S3 - Amazon Simple Storage Service.txt | 346 -------------- ...tables - Amazon Simple Storage Service.txt | 45 -- ...s and How to Solve Them | BrowserStack.txt | 84 ---- ...S SDKs - Amazon Simple Storage Service.txt | 28 -- ...for S3 - Amazon Simple Storage Service.txt | 22 - ... class - Amazon Simple Storage Service.txt | 27 -- ...zon S3 - Amazon Simple Storage Service.txt | 344 -------------- ...for S3 - Amazon Simple Storage Service.txt | 35 -- ...uckets - Amazon Simple Storage Service.txt | 229 ---------- ...on EC2? - Amazon Elastic Compute Cloud.txt | 151 ------ ...on S3? - Amazon Simple Storage Service.txt | 431 ------------------ crawl/main.py | 42 -- 28 files changed, 3208 deletions(-) delete mode 100644 crawl/crawled_data/10 Biggest DevOps Challenges & Issues and Their Solutions - Sematext.txt delete mode 100644 crawl/crawled_data/Amazon EC2 instance types - Amazon EC2.txt delete mode 100644 crawl/crawled_data/Buckets overview - Amazon Simple Storage Service.txt delete mode 100644 crawl/crawled_data/Common bucket patterns for building applications on Amazon S3 - Amazon Simple Storage Service.txt delete mode 100644 crawl/crawled_data/Configuring Storage Browser for S3 - Amazon Simple Storage Service.txt delete mode 100644 crawl/crawled_data/Configuring and using Mountpoint - Amazon Simple Storage Service.txt delete mode 100644 crawl/crawled_data/Creating a bucket - Amazon Simple Storage Service.txt delete mode 100644 crawl/crawled_data/Example metadata table queries - Amazon Simple Storage Service.txt delete mode 100644 crawl/crawled_data/General purpose bucket naming rules - Amazon Simple Storage Service.txt delete mode 100644 crawl/crawled_data/Installing Mountpoint - Amazon Simple Storage Service.txt delete mode 100644 crawl/crawled_data/Installing Storage Browser for S3 - Amazon Simple Storage Service.txt delete mode 100644 crawl/crawled_data/Joining custom metadata with S3 metadata tables - Amazon Simple Storage Service.txt delete mode 100644 crawl/crawled_data/Metadata table limitations and restrictions - Amazon Simple Storage Service.txt delete mode 100644 crawl/crawled_data/Naming Amazon S3 objects - Amazon Simple Storage Service.txt delete mode 100644 crawl/crawled_data/Optimizing metadata table query performance - Amazon Simple Storage Service.txt delete mode 100644 crawl/crawled_data/S3 Metadata tables schema - Amazon Simple Storage Service.txt delete mode 100644 crawl/crawled_data/Setting up Storage Browser for S3 - Amazon Simple Storage Service.txt delete mode 100644 crawl/crawled_data/Setting up permissions for configuring metadata tables - Amazon Simple Storage Service.txt delete mode 100644 crawl/crawled_data/Top DevOps Challenges and its SolutionsTop Challenges in DevOps and How to Solve Them | BrowserStack.txt delete mode 100644 crawl/crawled_data/Tracking a multipart upload with the AWS SDKs - Amazon Simple Storage Service.txt delete mode 100644 crawl/crawled_data/Troubleshooting Storage Browser for S3 - Amazon Simple Storage Service.txt delete mode 100644 crawl/crawled_data/Uploading a directory using the high-level .NET TransferUtility class - Amazon Simple Storage Service.txt delete mode 100644 crawl/crawled_data/Uploading and copying objects using multipart upload in Amazon S3 - Amazon Simple Storage Service.txt delete mode 100644 crawl/crawled_data/Using Storage Browser for S3 - Amazon Simple Storage Service.txt delete mode 100644 crawl/crawled_data/Virtual hosting of buckets - Amazon Simple Storage Service.txt delete mode 100644 crawl/crawled_data/What is Amazon EC2? - Amazon Elastic Compute Cloud.txt delete mode 100644 crawl/crawled_data/What is Amazon S3? - Amazon Simple Storage Service.txt delete mode 100644 crawl/main.py diff --git a/crawl/crawled_data/10 Biggest DevOps Challenges & Issues and Their Solutions - Sematext.txt b/crawl/crawled_data/10 Biggest DevOps Challenges & Issues and Their Solutions - Sematext.txt deleted file mode 100644 index 691dc388..00000000 --- a/crawl/crawled_data/10 Biggest DevOps Challenges & Issues and Their Solutions - Sematext.txt +++ /dev/null @@ -1,63 +0,0 @@ -Title: 10 Biggest DevOps Challenges & Issues and Their Solutions - Sematext - -Registration is open - Live, Instructor-led Online Classes - Elasticsearch in March - Solr in April - OpenSearch in May. See all classes -Solr / Elasticsearch Experts – Search & Big Data Analytics -Close -Upcoming Solr, Elasticsearch, and OpenSearch Online Training Classes See Online Classes -Check the latest product updates -Compare Sematext with other solutions according to reviews on G2.com. -Explore all DevOps terms and definitions in our Glossary -See all Videos -See All → -Table of Contents -To enhance their continuous delivery initiative, companies have long started to switch to a DevOps culture. By adapting best practices such as CI/CD pipeline, continuous testing or continuous security, teams get faster and more creative in scaling their operations. -On the other hand, the companies that are yet to take the plunge are under pressure to comply if they want to get or maintain a competitive edge. -If you are planning to implement DevOps practices, know that even though you’ll benefit greatly from it, the adoption process is not without its fair share of issues. -We asked engineers what were the biggest DevOps challenges they encountered when implementing the model and their solutions. Here’s what they shared with us. -If you are planning to adopt DevOps practices, here are the top DevOps challenges that you must address in the process and how to overcome them: -Determining the metrics that are the most relevant and useful for a particular organization is one of the top challenges in transitioning to the DevOps model. That is because a wide range of metrics is available to measure the effectiveness and efficiency of a DevOps process, such as deployment frequency, lead time for changes, mean time to recover from failures, and defect escape rate. However, not all of these metrics may be applicable or valuable for a given organization, and identifying the metrics important to your organization can be challenging. -You could adopt a data-driven approach to identify and track DevOps metrics. This can involve using tools such as analytics platforms or dashboards to collect and visualize data on various aspects of the DevOps process. This can help organizations find the right metrics, identify the patterns and trends in their data, and focus on areas where they may be able to improve their processes or where they may be experiencing bottlenecks or other issues. -Security in DevOps ensures that security needs are adequately addressed throughout the development process, as traditional approaches to security can be a barrier to rapid development and deployment. It can be tough in organizations that have adopted a continuous delivery model, where code changes are made and deployed frequently. -You could use the DevSecOps approach to resolve this challenge. DevSecOps integrates security into the DevOps process from the outset. It involves the collaboration of development, security, and operations teams to build and deploy secure systems continuously and automatedly. -To put it in a better way, Radu Tizu, CTO of Multicode Plus, said, ‘Dev(Sec)Ops comes as a requirement in the current ecosystem of unlimited auto-scalable number of worker nodes – for consistency, trackability (and auditing) but also fast response times. ‘Data center is on fire?’ run this command and change the target region, and you get a brand new infrastructure deployment; ‘moving to a new region with strict data retention policies?’ do the same and replicate the infrastructure, ‘Black Friday is this Friday?’ DevSecOps can triple your capacity in no time.’ -To implement a DevSecOps approach, organizations can adopt tools and processes that enable developers to identify and fix security issues early in the development cycle, such as static analysis tools and automated testing frameworks. -Using microservices in a DevOps environment increases the complexity of managing a large number of independently deployable service components. This can make it more difficult to track dependencies between services, troubleshoot issues, and ensure that the overall system functions correctly. -To tackle this DevOps challenge, tools and practices that enable organizations to manage and monitor their microservices environment effectively can be adopted. This can involve using tools such as service meshes or orchestration platforms to manage communication between services and implementing monitoring and logging systems to track the health and performance of individual services. -Managing changes in a DevOps environment calls for balancing rapid deployment with the need to ensure the stability and reliability of the system. It can be difficult because changing a system introduces new risks and vulnerabilities, and the more frequent changes are made, the greater the potential for issues. -One solution to this challenge is to adopt a culture of continuous testing and monitoring, in which changes are thoroughly tested and validated before being deployed to production. It usually means implementing automated testing frameworks and using tools such as monitoring and logging platforms to track the performance and behavior of the system after changes are made. -However, there is an even bigger challenge to implementing testing frameworks and tools for monitoring. Neil Douek, Director of DevOps / Agility at Dops Digital says – “In my experience you can’t implement a culture, but rather nurture it into existence over a period of time. I always look towards fundamentals, such as branching strategy and repository hygiene, and then develop sustainable standards that can support CI/CD.” So, you need to ensure that your team members understand what to test (or log) in their code and how to make tests (or generate logs). Only after you and your team are satisfied with the fundamentals, you should introduce automated testing and log management rather than directly leapfrogging into it. -Just like the metrics, the DevOps market is filled with several DevOps tools. But, there are no standard tools that apply to all organizations because each organization may have different requirements. So, selecting and integrating the right tools into the development and deployment process is critical to achieving DevOps success within an organization. -One way to overcome this challenge is to adopt a DevOps toolchain, a set of integrated tools that work together to support the development and deployment process. A DevOps toolchain includes tools for version control, continuous integration, testing, deployment, and monitoring. You should also follow these 4 best practices for choosing DevOps tools and simplify your suite. -Cross-functional teams in a DevOps environment ensure that team members have the necessary skills and knowledge to collaborate effectively and work together. But establishing a team with the necessary skills can be challenging because DevOps requires diverse skills and expertise, including development, operations, security, and testing. -However, organizations can provide training and resources to help DevOps teams build the necessary skills and knowledge. It involves providing training on relevant tools and technologies, as well as on practices such as agile methodologies and continuous delivery. -Another solution to this issue is to adopt a culture of collaboration and ownership within the organization. According to Lewis Stevens, Senior Cloud DevOps and Infrastructure Engineer of DigiTickets – lk message, ‘The main challenge I have found trying to implement a DevOps culture is that the Developers or the Operations team struggle to work on each other’s repositories. To overcome this, I will typically focus on creating a new small project where a developer and operations engineer is assigned solely to the project and can take ownership of it.’ -While these solutions can help in creating a cross functional team, Rafal Krol, a Cloud Site Reliability Engineer at Chaos Gears also warns that change can be overwhelming for members of an organization, so the organization should introduce change gradually. He says – “If I were to name one challenge of embracing DevOps culture, I’d doubt the lack of a general agreement on the importance of learning. A cultural change takes time to ripe…” -Floor Nooitgedagt, Principal Recruitment Consultant for DevOps, Cloud & Data at Apollo Solutions argues – “…If your CIO or CTO doesn’t truly advocate DevOps, it’ll be very difficult to successfully implement a strong DevOps culture across the business. And it all starts with senior management and a clear vision.” -Organizations with clear vision can implement a center of excellence (COE) model in a DevOps environment to ensure that DevOps practices are successfully implanted. COE effectively supports and facilitates the development and deployment process across the organization. But as the COE must coordinate with multiple teams and functions and may need to juggle competing priorities and demands, this can become challenging. -One way to solve this DevOps challenge is to adopt a collaborative and consultative approach to the COE model, in which the COE works closely with teams across the organization to understand their needs and goals and provides guidance and support as needed. This ensures that the COE can effectively support the development and deployment process while respecting the autonomy and independence of individual teams. -Strict governance strives for the system’s quality and stability by imposing clear policies and procedures for development and deployment. But on the other hand, overly strict governance can become a barrier to rapid development and deployment, as it may require teams to follow rigid processes and procedures that can slow down the development and deployment process. So finding the right balance between control and necessary oversight for flexibility and agility can become a challenge of the DevOps adoption process. -One solution to this challenge is to adopt a flexible and agile approach to governance, in which policies and procedures are designed to support and enable the development and deployment process rather than hinder it. -A DevOps environment has to manage multiple environments. But, ensuring the different environments are kept in sync, and that code changes and deployments are properly tested and validated before being promoted to production can be difficult. Different environments may have different configurations and dependencies. -One way to conquer this challenge is to adopt a continuous integration and delivery (CI/CD) approach, in which code changes are automatically built, tested, and deployed through a series of automated DevOps pipeline stages. -Organizations need to ensure that DevOps environments have the budget allocation aligned with the needs and goals of the organization. However, DevOps involves a wide range of activities and tools, and it can be challenging to determine which areas are most important and should be prioritized. -To solve this DevOps challenge, you could involve stakeholders from across the organization in the budgeting process and establish clear policies and procedures for allocating resources. This ensures that budget allocation is based on the needs and goals of the organization and that resources are being used effectively to support the development and deployment process. -In the rapidly evolving landscape of microservices, the challenges of ensuring their continuous availability and optimal performance are paramount. Manual testing and deploying new functionalities across an ever-growing number of microservices can be complex and time-consuming. DevOps teams face the pressing need to replicate microservices in test environments while maintaining their stability. This is where Sematext Cloud comes into play with its array of advanced features designed to improve your DevOps processes and address potential challenges. -By seamlessly integrating Sematext agents, microservices can be monitored as they dynamically come and go, even on-the-fly. This proactive approach helps swiftly identify instability or performance issues, ensuring the reliability and efficiency of the microservices ecosystem. -Furthermore, the embrace of an active CI/CD pipeline helps both developers and operations. This collaboration facilitates rapid product patching and the introduction of new functionalities, sometimes on a daily basis. However, fully harnessing the power of CI/CD takes time and effort as both teams learn its intricacies, enhance it through plugins, and grasp its limitations. -Providing real-time monitoring and analytics capabilities, Sematext makes it easier to achieve operational performance.From monitoring microservices in real-time to refining the CI/CD pipeline, Sematext equips teams with the tools needed to enhance efficiency, minimize downtime, and deliver superior products. -Watch the video below to learn more about Sematext Cloud or start your 14-day free trial to experience the power of enhanced monitoring. - -Getting Started with Sematext Cloud -DevOps is a powerful approach to software development and deployment that can help organizations deliver value to customers faster and more reliably. However, implementing DevOps also brings with it several challenges and issues that teams must address to succeed. -Challenges like selecting and integrating the right tools, managing security risks, and coordinating across teams and functions can pressure companies trying to adopt DevOps practices. However, by understanding these challenges and adopting the right solutions, organizations can overcome these hurdles and build a successful DevOps practice that drives business value. -Start Free Trial -See all jobs -Write for us -Get tips, how-tos, and news about Elastic / ELK Stack, Observability, Solr, and Sematext Cloud news and updates. - -© Sematext Group. All rights reserved -Terms Of Service · Privacy Policy - -Apache Lucene, Apache Solr and their respective logos are trademarks of the Apache Software Foundation. -Elasticsearch, Kibana, Logstash, and Beats are trademarks of Elasticsearch BV, registered in the U.S. -and in other countries. Sematext Group, Inc. is not affiliated with Elasticsearch BV. diff --git a/crawl/crawled_data/Amazon EC2 instance types - Amazon EC2.txt b/crawl/crawled_data/Amazon EC2 instance types - Amazon EC2.txt deleted file mode 100644 index 861c4cda..00000000 --- a/crawl/crawled_data/Amazon EC2 instance types - Amazon EC2.txt +++ /dev/null @@ -1,79 +0,0 @@ -Title: Amazon EC2 instance types - Amazon EC2 - -When you launch an EC2 instance, the instance type that you specify - determines the hardware of the host computer used for your instance. Each instance type - offers different compute, memory, and storage capabilities, and is grouped in an instance - family based on these capabilities. Select an instance type based on the requirements of the - application or software that you plan to run on your instance. -Amazon EC2 dedicates some resources of the host computer, such as CPU, memory, and instance - storage, to a particular instance. Amazon EC2 shares other resources of the host computer, such - as the network and the disk subsystem, among instances. If each instance on a host computer - tries to use as much of one of these shared resources as possible, each receives an equal - share of that resource. However, when a resource is underused, an instance can consume a - higher share of that resource while it's available. -Each instance type provides higher or lower minimum performance from a shared resource. - For example, instance types with high I/O performance have a larger allocation of shared resources. - Allocating a larger share of shared resources also reduces the variance of I/O performance. - For most applications, moderate I/O performance is more than enough. However, for - applications that require greater or more consistent I/O performance, consider - an instance type with higher I/O performance. -Current generation instances -Previous generation instances -Amazon EC2 instance type naming conventions -Amazon EC2 instance type specifications -Instances built on the AWS Nitro System -Amazon EC2 instance type quotas -For the best performance, we recommend that you use the following instance types - when you launch new instances. For more information, see Amazon EC2 Instance Types. -General purpose: M5 | M5a | M5ad | M5d | M5dn | M5n | M5zn | M6a | M6g | M6gd | M6i | M6id | M6idn | M6in | M7a | M7g | M7gd | M7i | M7i-flex | M8g | Mac1 | Mac2 | Mac2-m1ultra | Mac2-m2 | Mac2-m2pro | T2 | T3 | T3a | T4g -Compute optimized: C5 | C5a | C5ad | C5d | C5n | C6a | C6g | C6gd | C6gn | C6i | C6id | C6in | C7a | C7g | C7gd | C7gn | C7i | C7i-flex | C8g -Memory optimized: R5 | R5a | R5ad | R5b | R5d | R5dn | R5n | R6a | R6g | R6gd | R6i | R6idn | R6in | R6id | R7a | R7g | R7gd | R7i | R7iz | R8g | U-3tb1 | U-6tb1 | U-9tb1 | U-12tb1 | U-18tb1 | U-24tb1 | U7i-6tb | U7i-8tb | U7i-12tb | U7in-16tb | U7in-24tb | U7in-32tb | X1 | X1e | X2gd | X2idn | X2iedn | X2iezn | X8g | z1d -Storage optimized: D2 | D3 | D3en | H1 | I3 | I3en | I4g | I4i | I7ie | I8g | Im4gn | Is4gen -Accelerated computing: DL1 | DL2q | F1 | G4ad | G4dn | G5 | G5g | G6 | G6e | Gr6 | Inf1 | Inf2 | P2 | P3 | P3dn | P4d | P4de | P5 | P5e | P5en | Trn1 | Trn1n | Trn2 | Trn2u | VT1 -High-performance computing: Hpc6a | Hpc6id | Hpc7a | Hpc7g -Amazon Web Services offers previous generation instance types for users who have optimized their - applications around them and have yet to upgrade. We encourage you to use current generation - instance types to get the best performance, but we continue to support the following previous - generation instance types. For more information about which current - generation instance type would be a suitable upgrade, see - Previous Generation Instances. -General purpose: A1 | M1 | M2 | M3 | M4 | T1 -Compute optimized: C1 | C3 | C4 -Memory optimized: R3 | R4 -Storage optimized: I2 -Accelerated computing: G3 -Fixed performance instances provide fixed CPU resources. These instances can - deliver and sustain full CPU performance at any time, and for as long as a workload - needs it. If you need consistently high CPU performance for applications such as - video encoding, high volume websites, or HPC applications, we recommend that you use - fixed performance instances. -Burstable performance (T) instances provide a baseline level of CPU - performance with the ability to burst above the baseline. The baseline CPU is - designed to meet the needs of the majority of general purpose workloads, such as - large-scale micro-services, web servers, small and medium databases, data logging, - code repositories, virtual desktops, and development and test environments. -The baseline utilization and ability to burst are governed by CPU credits. Each - burstable performance instance continuously earns credits when it stays below the CPU - baseline, and continuously spends credits when it bursts above the baseline. For more - information, see Burstable - performance instances in the Amazon EC2 User Guide. -M7i-flex and C7i-flex instances offer a balance of compute, memory, and network - resources, and they provide the most cost-effective way to run a broad spectrum of - general purpose applications. These instances provide reliable CPU resources to - deliver a baseline CPU performance of 40 percent, which is designed to meet the - compute requirements for a majority of general purpose workloads. When more - performance is needed, these instances provide the ability to exceed the baseline - CPU performance and deliver up to 100 percent CPU performance for 95 percent of the - time over a 24-hour window. -M7i-flex and C7i-flex instances running at a high CPU utilization that is consistently - above the baseline for long periods of time might see a gradual reduction in the maximum - burst CPU throughput. For more information, see M7i-flex instances and C7i-flex instances. -For pricing information, see Amazon EC2 Pricing. - Javascript is disabled or is unavailable in your browser. -To use the Amazon Web Services Documentation, Javascript must be enabled. Please refer to your browser's Help pages for instructions. -Thanks for letting us know we're doing a good job! -If you've got a moment, please tell us what we did right so we can do more of it. - -Thanks for letting us know this page needs work. We're sorry we let you down. -If you've got a moment, please tell us how we can make the documentation better. - diff --git a/crawl/crawled_data/Buckets overview - Amazon Simple Storage Service.txt b/crawl/crawled_data/Buckets overview - Amazon Simple Storage Service.txt deleted file mode 100644 index 1c41cddf..00000000 --- a/crawl/crawled_data/Buckets overview - Amazon Simple Storage Service.txt +++ /dev/null @@ -1,225 +0,0 @@ -Title: Buckets overview - Amazon Simple Storage Service - -To upload your data (photos, videos, documents, etc.) to Amazon S3, you must first create an S3 - bucket in one of the AWS Regions. -There are two types of Amazon S3 buckets, general purpose buckets and directory buckets. Choose - the bucket type that best fits your application and performance requirements: -General purpose buckets are the original S3 - bucket type and are recommended for most use cases and access patterns. General - purpose buckets also allow objects that are stored across all storage classes, - except S3 Express One Zone. -Directory buckets use the S3 Express One Zone storage - class, which is recommended if your application is performance sensitive and - benefits from single-digit millisecond PUT and GET - latencies, see S3 Express One Zone and Working with directory buckets. -The following sections provide more information about general purpose buckets, including bucket - naming rules, quotas, and bucket configuration details. For a list of restriction and - limitations related to Amazon S3 buckets see, Bucket quotas, limitations, and restrictions. -A general purpose bucket is a container for objects stored in Amazon S3. You can store any number of - objects in a bucket and all accounts have a default bucket quota of 10,000 general purpose buckets. To see your bucket utilization, bucket quota, or request an - increase to this quota, visit the Service Quotas - console. -General purpose bucket quotas for commercial Regions can only be viewed and managed - from US East (N. Virginia). -General purpose bucket quotas for AWS GovCloud (US) can only be viewed and managed from - AWS GovCloud (US-West). -Every object is contained in a bucket. For example, if the object named - photos/puppy.jpg is stored in the - amzn-s3-demo-bucket bucket in the US West (Oregon) - Region, then it is addressable by using the URL - https://amzn-s3-demo-bucket.s3.us-west-2.amazonaws.com/photos/puppy.jpg. - For more information, see Accessing a - Bucket. -In terms of implementation, buckets and objects are AWS resources, and Amazon S3 provides - APIs for you to manage them. For example, you can create a bucket and upload objects using - the Amazon S3 API. You can also use the Amazon S3 console to perform these operations. The console - uses the Amazon S3 APIs to send requests to Amazon S3. -This section describes how to work with buckets. For information about working with - objects, see Amazon S3 objects overview. - Amazon S3 supports global buckets, which means that each bucket name must be unique across all - AWS accounts in all the AWS Regions within a partition. A partition is a grouping of - Regions. AWS currently has three partitions: aws (Standard Regions), - aws-cn (China Regions), and aws-us-gov (AWS GovCloud (US)). -After a bucket is created, the name of that bucket cannot be used by another AWS account - in the same partition until the bucket is deleted. You should not depend on specific bucket - naming conventions for availability or security verification purposes. For bucket naming - guidelines, see General purpose bucket naming rules. -Amazon S3 creates buckets in a Region that you specify. To reduce latency, minimize costs, or - address regulatory requirements, choose any AWS Region that is geographically close to - you. For example, if you reside in Europe, you might find it advantageous to create buckets - in the Europe (Ireland) or Europe (Frankfurt) Regions. For a list of Amazon S3 Regions, see - Regions and - Endpoints in the AWS General Reference. -Objects that belong to a bucket that you create in a specific AWS Region never leave - that Region, unless you explicitly transfer them to another Region. For example, objects - that are stored in the Europe (Ireland) Region never leave it. -When you build applications on Amazon S3, you can use unique general purpose buckets to separate - different datasets or workloads. Depending on your use case, there are different design - patterns and best practices for using buckets. For more information, see Common bucket patterns for building applications on - Amazon S3. -You can use your AWS account root user credentials to create a bucket and perform any other Amazon S3 - operation. However, we recommend that you do not use the root user credentials of your - AWS account to make requests, such as to create a bucket. Instead, create an AWS Identity and Access Management - (IAM) user, and grant that user full access (users by default have no permissions). -These users are referred to as administrators. You - can use the administrator user credentials, instead of the root user credentials of your - account, to interact with AWS and perform tasks, such as create a bucket, create - users, and grant them permissions. -For more information, see AWS account root user - credentials and IAM user credentials in the AWS - General Reference and Security best practices in IAM in the - IAM User Guide. -The AWS account that creates a resource owns that resource. For example, if you - create an IAM user in your AWS account and grant the user permission to create a - bucket, the user can create a bucket. But the user does not own the bucket; the - AWS account that the user belongs to owns the bucket. The user needs additional - permission from the resource owner to perform any other bucket operations. For more - information about managing permissions for your Amazon S3 resources, see Identity and Access Management for Amazon S3. -Public access is granted to buckets and objects through bucket policies, access - control lists (ACLs), or both. To help you manage public access to Amazon S3 resources, Amazon S3 - provides settings to block public access. Amazon S3 Block Public Access settings can override - ACLs and bucket policies so that you can enforce uniform limits on public access to - these resources. You can apply Block Public Access settings to individual buckets or to - all buckets in your account. -To ensure that all of your Amazon S3 buckets and objects have their public access blocked, - all four settings for Block Public Access are enabled by default when you create a new - bucket. We recommend that you turn on all four settings for Block Public Access for your - account too. These settings block all public access for all current and future - buckets. -Before applying these settings, verify that your applications will work correctly - without public access. If you require some level of public access to your buckets or - objects—for example, to host a static website, as described at Hosting a static website using Amazon S3—you can customize - the individual settings to suit your storage use cases. For more information, see Blocking public access to your Amazon S3 - storage. - However, we highly recommend keeping Block Public Access enabled. If you want to keep - all four Block Public Access settings enabled and host a static website, you can use - Amazon CloudFront origin access control (OAC). Amazon CloudFront provides the capabilities required to set - up a secure static website. Amazon S3 static websites support only HTTP endpoints. Amazon CloudFront - uses the durable storage of Amazon S3 while providing additional security headers, such as - HTTPS. HTTPS adds security by encrypting a normal HTTP request and protecting against - common cyberattacks. -For more information, see Getting started with a secure static website in the Amazon CloudFront Developer Guide. -If you see an Error when you list your buckets and their public - access settings, you might not have the required permissions. Make sure that you - have the following permissions added to your user or role policy: -In some rare cases, requests can also fail because of an AWS Region - outage. -Amazon S3 supports various options for you to configure your bucket. For example, you can - configure your bucket for website hosting, add a configuration to manage the lifecycle - of objects in the bucket, and configure the bucket to log all access to the bucket. Amazon S3 - supports subresources for you to store and manage the bucket configuration information. - You can use the Amazon S3 API to create and manage these subresources. However, you can also - use the console or the AWS SDKs. -There are also object-level configurations. For example, you can configure - object-level permissions by configuring an access control list (ACL) specific to - that object. -These are referred to as subresources because they exist in the context of a specific - bucket or object. The following table lists subresources that enable you to manage - bucket-specific configurations. - -cors (cross-origin resource sharing) - You can configure your bucket to allow cross-origin - requests. -For more information, see Using cross-origin resource sharing (CORS). - -event notification - -You can enable your bucket to send you notifications of specified - bucket events. -For more information, see Amazon S3 Event Notifications. -You can define lifecycle rules for objects in your bucket that - have a well-defined lifecycle. For example, you can define a rule to - archive objects one year after creation, or delete an object 10 - years after creation. -For more information, see Managing the lifecycle of objects. - -location - - When you create a bucket, you specify the AWS Region where you - want Amazon S3 to create the bucket. Amazon S3 stores this information in the - location subresource and provides an API for you to retrieve this - information. - -logging - -Logging enables you to track requests for access to your bucket. - Each access log record provides details about a single access - request, such as the requester, bucket name, request time, request - action, response status, and error code, if any. Access log - information can be useful in security and access audits. It can also - help you learn about your customer base and understand your Amazon S3 - bill.   -For more information, see Logging requests with server access logging. - -object locking - -To use S3 Object Lock, you must enable it for a bucket. You can - also optionally configure a default retention mode and period that - applies to new objects that are placed in the bucket. -For more information, see Locking objects with Object Lock. - -policy and ACL (access - control list) -All your resources (such as buckets and objects) are private by - default. Amazon S3 supports both bucket policy and access control list - (ACL) options for you to grant and manage bucket-level permissions. - Amazon S3 stores the permission information in the - policy and acl - subresources. -For more information, see Identity and Access Management for Amazon S3. - -replication - -Replication is the automatic, asynchronous copying of objects - across buckets in different or the same AWS Regions. For more - information, see Replicating objects within and across Regions. - -requestPayment - -By default, the AWS account that creates the bucket (the bucket - owner) pays for downloads from the bucket. Using this subresource, - the bucket owner can specify that the person requesting the download - will be charged for the download. Amazon S3 provides an API for you to - manage this subresource. -For more information, see Using Requester Pays buckets for storage - transfers and usage. - -tagging - -You can add cost allocation tags to your bucket to categorize and - track your AWS costs. Amazon S3 provides the - tagging subresource to store and manage - tags on a bucket. Using tags you apply to your bucket, AWS - generates a cost allocation report with usage and costs aggregated - by your tags. -For more information, see Billing and usage reporting for Amazon S3. - -transfer acceleration - -Transfer Acceleration enables fast, easy, and secure transfers of files - over long distances between your client and an S3 bucket. - Transfer Acceleration takes advantage of the globally distributed edge - locations of Amazon CloudFront. -For more information, see Configuring fast, secure file transfers using - Amazon S3 Transfer Acceleration. -Versioning helps you recover accidental overwrites and deletes. -We recommend versioning as a best practice to recover objects from - being deleted or overwritten by mistake. -For more information, see Retaining multiple versions of objects with S3 Versioning. -You can configure your bucket for static website hosting. Amazon S3 - stores this configuration by creating a website - subresource. -For more information, see Hosting a static website using Amazon S3. -The high availability engineering of Amazon S3 is focused on get, put, list, and delete operations. Because - bucket operations work against a centralized, global resource space, we recommend that you - don't create, delete, or configure buckets on the high availability code path - of your application. It's better to create, delete, or configure buckets in a separate - initialization or setup routine that you run less often. - Javascript is disabled or is unavailable in your browser. -To use the Amazon Web Services Documentation, Javascript must be enabled. Please refer to your browser's Help pages for instructions. -Thanks for letting us know we're doing a good job! -If you've got a moment, please tell us what we did right so we can do more of it. - -Thanks for letting us know this page needs work. We're sorry we let you down. -If you've got a moment, please tell us how we can make the documentation better. - diff --git a/crawl/crawled_data/Common bucket patterns for building applications on Amazon S3 - Amazon Simple Storage Service.txt b/crawl/crawled_data/Common bucket patterns for building applications on Amazon S3 - Amazon Simple Storage Service.txt deleted file mode 100644 index 1de5b92a..00000000 --- a/crawl/crawled_data/Common bucket patterns for building applications on Amazon S3 - Amazon Simple Storage Service.txt +++ /dev/null @@ -1,65 +0,0 @@ -Title: Common bucket patterns for building applications on Amazon S3 - Amazon Simple Storage Service - -When you build applications on Amazon S3, you can use unique general - purpose buckets to separate different datasets or workloads. When you build applications - that serve end users or different user groups, use our best practices design - patterns to build applications that can best take advantage of Amazon S3 features and - scalability. - -We recommend that you create bucket names that are not predictable. Do not write code - assuming your chosen bucket name is available unless you have already created the - bucket. One method for creating bucket names that are not predictable is to append a - Globally Unique Identifier (GUID) to your bucket name, for example, - amzn-s3-demo-bucket-a1b2c3d4-5678-90ab-cdef-EXAMPLE11111. - For more information about general purpose bucket naming rules, see General purpose bucket naming rules. -With multi-tenant buckets, you create a single bucket for a team or workload. You use - unique S3 prefixes to organize the objects - that you store in the bucket. A prefix is a string of characters at the beginning of the - object key name. A prefix can be any length, subject to the maximum length of the object - key name (1,024 bytes). You can think of prefixes as a way to organize your data in a - similar way to directories. However, prefixes are not directories. -For example, to store information about cities, you might organize it by continent, - then by country, then by province or state. Because these names don't usually contain - punctuation, you might use slash (/) as the delimiter. The following examples shows - prefixes being used to organize city names by continent, country, and then province or - state, using a slash (/) delimiter. -Europe/France/NouvelleA-Aquitaine/Bordeaux -North America/Canada/Quebec/Montreal -North America/USA/Washington/Bellevue -North America/USA/Washington/Seattle -This pattern scales well when you have hundreds of unique datasets within a bucket. - With prefixes, you can easily organize and group these datasets. -However, one potential drawback to the multi-tenant bucket pattern is that many S3 - bucket-level features like default bucket - encryption, S3 Versioning, and - S3 Requester Pays are set at the - bucket-level and not the prefix-level. If the different datasets within the multi-tenant - bucket have unique requirements, the fact that you can't configure many S3 bucket-level - features at the prefix-level can make it difficult for you to specify the correct - settings for each dataset. Additionally, in a multi-tenant bucket, cost allocation can become complex as you work to - understand the storage, requests, and data transfer associated with specific prefixes. -With the bucket-per-use pattern, you create a bucket for each distinct dataset, end - user, or team. Because you can configure S3 bucket-level features for each of these - buckets, you can use this pattern to configure unique bucket-level settings. For - example, you can configure features like default - bucket encryption, S3 Versioning, - and S3 Requester Pays in a way that is - customized to the dataset in each bucket. Using one bucket for each distinct dataset, - end user, or team can also help you simplify both your access management and cost - allocation strategies. -A potential drawback to this strategy is that you will need to manage potentially - thousands of buckets. All AWS accounts have a default bucket quota of 10,000 general - purpose buckets. You can increase the bucket quota for an account by submitting a quota - increase request. To request an increase for general purpose buckets, visit the Service Quotas console. -To manage your bucket-per-use pattern and simplify your infrastructure management, you - can use AWS CloudFormation. You can create a custom AWS CloudFormation template for your pattern that - already defines all of your desired settings for your S3 buckets so that you can easily - deploy and track any changes to your infrastructure. For more information, see AWS::S3::Bucket in the AWS CloudFormation User Guide. - Javascript is disabled or is unavailable in your browser. -To use the Amazon Web Services Documentation, Javascript must be enabled. Please refer to your browser's Help pages for instructions. -Thanks for letting us know we're doing a good job! -If you've got a moment, please tell us what we did right so we can do more of it. - -Thanks for letting us know this page needs work. We're sorry we let you down. -If you've got a moment, please tell us how we can make the documentation better. - diff --git a/crawl/crawled_data/Configuring Storage Browser for S3 - Amazon Simple Storage Service.txt b/crawl/crawled_data/Configuring Storage Browser for S3 - Amazon Simple Storage Service.txt deleted file mode 100644 index 68b5ff98..00000000 --- a/crawl/crawled_data/Configuring Storage Browser for S3 - Amazon Simple Storage Service.txt +++ /dev/null @@ -1,15 +0,0 @@ -Title: Configuring Storage Browser for S3 - Amazon Simple Storage Service - -To allow Storage Browser for S3 access to S3 buckets, the Storage Browser component - makes the REST API calls to Amazon S3. By default, cross-origin resource sharing (CORS) isn’t - enabled on S3 buckets. As a result, you must enable CORS for each S3 bucket that Storage Browser is accessing data from. -For example, to enable CORS on your S3 bucket, you can update your CORS policy like - this: - Javascript is disabled or is unavailable in your browser. -To use the Amazon Web Services Documentation, Javascript must be enabled. Please refer to your browser's Help pages for instructions. -Thanks for letting us know we're doing a good job! -If you've got a moment, please tell us what we did right so we can do more of it. - -Thanks for letting us know this page needs work. We're sorry we let you down. -If you've got a moment, please tell us how we can make the documentation better. - diff --git a/crawl/crawled_data/Configuring and using Mountpoint - Amazon Simple Storage Service.txt b/crawl/crawled_data/Configuring and using Mountpoint - Amazon Simple Storage Service.txt deleted file mode 100644 index 3d07c048..00000000 --- a/crawl/crawled_data/Configuring and using Mountpoint - Amazon Simple Storage Service.txt +++ /dev/null @@ -1,117 +0,0 @@ -Title: Configuring and using Mountpoint - Amazon Simple Storage Service - -To use Mountpoint for Amazon S3, your host needs valid AWS credentials with access to the bucket or - buckets that you would like to mount. For different ways to authenticate, see - Mountpoint AWS Credentials on GitHub. -For example, you can create a new AWS Identity and Access Management (IAM) user and role for this purpose. Make - sure that this role has access to the bucket or buckets that you would like to mount. You can - pass the - IAM role to your Amazon EC2 instance with an instance profile. -Use Mountpoint for Amazon S3 to do the following: -Mount buckets with the mount-s3 command. -In the following example, replace - amzn-s3-demo-bucket with the name of your S3 - bucket, and replace ~/mnt with the directory on - your host where you want your S3 bucket to be mounted. -Because the Mountpoint client runs in the background by default, the - ~/mnt directory now gives you access to the - objects in your S3 bucket. -Access the objects in your bucket through Mountpoint. -After you mount your bucket locally, you can use common Linux - commands, such as cat or ls, to work with your S3 objects. - Mountpoint for Amazon S3 interprets keys in your S3 bucket as file system paths by splitting them on - the forward slash (/) character. For example, if you have the object key - Data/2023-01-01.csv in your bucket, you will have a directory named - Data in your Mountpoint file system, with a file named - 2023-01-01.csv inside it. -Mountpoint for Amazon S3 intentionally does not implement the full POSIX standard specification for - file systems. Mountpoint is optimized for workloads that need high-throughput - read and write access to data stored in Amazon S3 through a file system interface, but that - otherwise do not rely on file system features. For more information, see Mountpoint for Amazon S3 - file - system behavior on GitHub. Customers that need richer file - system semantics should consider other AWS file services, such as Amazon Elastic File System (Amazon EFS) or Amazon FSx. - -Unmount your bucket by using the umount command. This command unmounts - your S3 bucket and exits Mountpoint. -To use the following example command, replace - ~/mnt with the directory on your host where - your S3 bucket is mounted. -To get a list of options for this command, run umount --help. -For additional Mountpoint configuration details, see S3 bucket configuration, and file system configuration on GitHub. -Mountpoint for Amazon S3 supports different types of data caching. To accelerate repeated read - requests, you can opt in to the following: -Local cache – You can use a local cache in - your Amazon EC2 instance storage or an Amazon Elastic Block Store volume. If you repeatedly read the same data - from the same compute instance and if you have unused space in your local instance - storage for the repeatedly read dataset, you should opt in to a local cache. -Shared cache – You can use a shared cache on - S3 Express One Zone. If you repeatedly read small objects from multiple compute instances or if - you do not know the size of your repeatedly read dataset and want to benefit from - elasticity of cache size, you should opt in to the shared cache. Once you opt in, - Mountpoint retains objects with sizes up to one megabyte in a directory bucket - that uses S3 Express One Zone. -Combined local and shared cache – If you - have unused space in your local cache but also want a shared cache across multiple - instances, you can opt in to both a local cache and shared cache. -Caching in Mountpoint is ideal for use cases where you repeatedly read the same - data that doesn’t change during the multiple reads. For example, you can use caching with - machine learning training jobs that need to read a training dataset multiple times to - improve model accuracy. -For more information about how to configure caching in Mountpoint, see the - following examples. -You can opt in to a local cache with the --cache - CACHE_PATH flag. In the following example, replace - CACHE_PATH with the filepath to the directory - that you want to cache your data in. Replace - amzn-s3-demo-bucket with the name of your S3 - bucket, and replace ~/mnt with the directory on - your host where you want your S3 bucket to be mounted. -When you opt in to local caching while mounting an S3 bucket, Mountpoint creates an - empty sub-directory at the configured cache location, if that sub-directory doesn’t - already exist. When you first mount a bucket and when you unmount, Mountpoint deletes the - contents of the local cache. -If you enable local caching, Mountpoint will persist unencrypted object - content from your mounted S3 bucket at the local cache location provided at mount. In - order to protect your data, you should restrict access to the data cache location by - using file system access control mechanisms. -If you repeatedly read small objects (up to 1 MB) from multiple compute instances or - the size of the dataset that you repeatedly read often exceeds the size of your local - cache, you should use a shared cache in S3 Express One Zone. When you read the same - data repeatedly from multiple instances, this improves latency by avoiding redundant - requests to your mounted S3 bucket. -Once you opt in to the shared cache, you pay for the data cached in your - directory bucket in S3 Express One Zone. You also pay for requests made against your data in the - directory bucket in S3 Express One Zone. For more information, see Amazon S3 pricing. Mountpoint never deletes cached objects - from directory buckets. To manage your storage costs, you should set up a Lifecycle - policy on your directory bucket so that Amazon S3 expires the cached data in - S3 Express One Zone after a period of time that you specify. For more information, see Mountpoint for Amazon S3 caching configuration on GitHub. -To opt in to caching in S3 Express One Zone when you mount a general purpose bucket to your compute - instance, use the --cache-xz flag and specify a directory bucket as your - cache location. In the following example, replace the user input - placeholders. -If you have unused space on your instance but you also want to use a shared cache - across multiple instances, you can opt in to both a local cache and shared cache. With - this caching configuration, you can avoid redundant read requests from the same instance - to the shared cache in directory bucket when the required data is cached in local - storage. This can reduce request costs and improve performance. - To opt in to both a local cache and shared cache when you mount an S3 bucket, you - specify both cache locations by using the --cache and --cache-xz - flags. To use the following example to opt into both a local and shared cache, replace the - user input placeholders. -For more information, Mountpoint for Amazon S3 caching configuration on GitHub. -If you enable shared caching, Mountpoint will copy object content from your - mounted S3 bucket into the S3 directory bucket that you provide as your shared cache - location, making it accessible to any caller with access to the S3 directory bucket. To - protect your cached data, you should follow the Security best practices for Amazon S3 to ensure that your buckets use the correct - policies and are not publicly accessible. You should use a directory bucket dedicated - to Mountpoint shared caching and grant access only to Mountpoint - clients. - Javascript is disabled or is unavailable in your browser. -To use the Amazon Web Services Documentation, Javascript must be enabled. Please refer to your browser's Help pages for instructions. -Thanks for letting us know we're doing a good job! -If you've got a moment, please tell us what we did right so we can do more of it. - -Thanks for letting us know this page needs work. We're sorry we let you down. -If you've got a moment, please tell us how we can make the documentation better. - diff --git a/crawl/crawled_data/Creating a bucket - Amazon Simple Storage Service.txt b/crawl/crawled_data/Creating a bucket - Amazon Simple Storage Service.txt deleted file mode 100644 index 752878be..00000000 --- a/crawl/crawled_data/Creating a bucket - Amazon Simple Storage Service.txt +++ /dev/null @@ -1,226 +0,0 @@ -Title: Creating a bucket - Amazon Simple Storage Service - -To upload your data to Amazon S3, you must first create an Amazon S3 bucket in one of the - AWS Regions. The AWS account that creates the bucket owns it. When you create a bucket, you must choose a bucket name and Region. You can - optionally choose other storage management options for the bucket. After you create a - bucket, you cannot change the bucket name or Region. For information about naming buckets, - see General purpose bucket naming rules. - -By default, you can create up to 10,000 general purpose buckets - per AWS account. To request a quota increase for general purpose buckets, - visit the Service Quotas - console. -You can store any number of objects in a bucket. For a list of restriction and limitations - related to Amazon S3 buckets see, Bucket quotas, limitations, and restrictions. - S3 Object Ownership is an Amazon S3 bucket-level setting that you can use both to control - ownership of objects that are uploaded to your bucket and to disable or enable access control lists (ACLs). By default, - Object Ownership is set to the Bucket owner enforced setting, and all ACLs are disabled. With ACLs disabled, the bucket owner - owns every object in the bucket and manages access to data exclusively by using policies. - For more information, see Controlling ownership of objects and disabling ACLs - for your bucket. -Server-side encryption with Amazon S3 managed keys (SSE-S3) - is the base level of encryption configuration for every bucket in Amazon S3. All new objects uploaded to an S3 bucket -are automatically encrypted with SSE-S3 as the base level of encryption setting. If you want to use a different type of default encryption, you can also specify server-side encryption with AWS Key Management Service - (AWS KMS) keys (SSE-KMS) or customer-provided keys (SSE-C) to encrypt your data. -For more information, see Setting default server-side encryption behavior for Amazon S3 - buckets. -You can use the Amazon S3 console, Amazon S3 APIs, AWS CLI, or AWS SDKs to create a bucket. For more - information about the permissions required to create a bucket, see CreateBucket in the Amazon Simple Storage Service API Reference. -Sign in to the AWS Management Console and open the Amazon S3 console at - https://console.aws.amazon.com/s3/. -In the navigation bar on the top of the page, choose the name of the currently displayed AWS Region. Next, choose the Region in which you want to create a bucket. - -To minimize latency and costs and address regulatory requirements, choose a Region - close to you. Objects stored in a Region never leave that Region unless you explicitly - transfer them to another Region. For a list of Amazon S3 AWS Regions, see AWS service endpoints in the - Amazon Web Services General Reference. -In the left navigation pane, choose Buckets. -Choose Create bucket. -The Create bucket page opens. -Under General configuration, view the AWS Region where your bucket will be created. - Under Bucket type, choose General purpose. -For Bucket name, enter a name for your bucket. -The bucket name must: -Be unique within a partition. A partition is a grouping of Regions. AWS - currently has three partitions: aws (Standard Regions), - aws-cn (China Regions), and aws-us-gov - (AWS GovCloud (US) Regions). -Be between 3 and 63 characters long. -Consist only of lowercase letters, numbers, dots (.), and hyphens (-). For - best compatibility, we recommend that you avoid using dots (.) in bucket names, - except for buckets that are used only for static website hosting. -Begin and end with a letter or number. -After you create the bucket, you cannot change its name. The AWS account that creates the bucket owns it. For more information about - naming buckets, see General purpose bucket naming rules. -Avoid including sensitive information, such as account numbers, in the bucket - name. The bucket name is visible in the URLs that point to the objects in the - bucket. -AWS Management Console allows you to copy an existing bucket's settings to your new bucket. If you do not want to copy the settings of an existing bucket, skip to the next step. -This option: -Is not available in the AWS CLI and is only available in console -Is not available for directory buckets -Does not copy the bucket policy from the existing bucket to the new bucket - To copy an existing bucket's settings, under Copy settings from existing bucket, select Choose bucket. The Choose bucket window opens. Find the bucket with the settings that you would like to copy, and select Choose bucket. The Choose bucket window closes, and the Create bucket window re-opens. -Under Copy settings from existing bucket, you will now see the name of the bucket you selected. You will also see a Restore defaults option that you can use to remove the copied bucket settings. Review the remaining bucket settings, on the Create bucket page. You will see that they now match the settings of the bucket that you selected. You can skip to the final step. -Under Object Ownership, to disable or enable ACLs and control - ownership of objects uploaded in your bucket, choose one of the following - settings: - Bucket owner enforced (default) – - ACLs are disabled, and the bucket owner automatically owns and has full control over every object in the bucket. ACLs - no longer affect access permissions to data in the S3 bucket. The bucket uses policies exclusively to define access control. -By default, ACLs are disabled. A majority of modern use cases in Amazon S3 no - longer require the use of ACLs. We recommend that you keep ACLs disabled, except - in unusual circumstances where you must control access for each object - individually. For more information, see Controlling ownership of objects and disabling ACLs - for your bucket. -Bucket owner preferred – The bucket owner owns and - has full control over new objects that other accounts write to the bucket with - the bucket-owner-full-control canned ACL. -If you apply the Bucket owner preferred setting, to - require all Amazon S3 uploads to include the bucket-owner-full-control - canned ACL, you can add a - bucket policy that allows only object uploads that use this - ACL. -Object writer – The AWS account that uploads an - object owns the object, has full control over it, and can grant other users - access to it through ACLs. -The default setting is Bucket owner enforced. To apply the - default setting and keep ACLs disabled, only the s3:CreateBucket - permission is needed. To enable ACLs, you must have the - s3:PutBucketOwnershipControls permission. -Under Block Public Access settings for this bucket, choose the - Block Public Access settings that you want to apply to the bucket. -By default, all four Block Public Access settings are enabled. We recommend that you - keep all settings enabled, unless you know that you need to turn off one or more of them - for your specific use case. For more information about blocking public access, see Blocking public access to your Amazon S3 - storage. -To enable all Block Public Access settings, only the s3:CreateBucket permission - is required. To turn off any Block Public Access settings, you must have the - s3:PutBucketPublicAccessBlock permission. -(Optional) Under Bucket Versioning, you can choose if you wish to keep - variants of objects in your bucket. For more information about versioning, - see Retaining multiple versions of objects with S3 Versioning. -To disable or enable versioning on your - bucket, choose either Disable or Enable. -(Optional) Under Tags, you can choose to add tags to your bucket. - Tags are key-value pairs used to categorize storage. -To add a bucket tag, enter a Key and optionally a - Value and choose Add Tag. -Under Default encryption, choose - Edit. -To configure default encryption, under Encryption type, - choose one of the following: -Amazon S3 managed key (SSE-S3) -AWS Key Management Service key (SSE-KMS) -If you use the SSE-KMS option for your default encryption configuration, you are - subject to the requests per second (RPS) quota of AWS KMS. For more information - about AWS KMS quotas and how to request a quota increase, see Quotas in the AWS Key Management Service Developer Guide. -Buckets and new objects are encrypted with server-side encryption with an Amazon S3 managed key as the base level of encryption configuration. For more - information about default encryption, see Setting default server-side encryption behavior for Amazon S3 - buckets. -For more information about using Amazon S3 server-side encryption to encrypt your data, - see Using server-side encryption with Amazon S3 managed keys - (SSE-S3). -If you chose AWS Key Management Service key (SSE-KMS), do the following: -Under AWS KMS key, specify your KMS key in one of the following ways: -To choose from a list of available KMS keys, choose Choose from - your AWS KMS keys, and choose your - KMS key from the list of available keys. -Both the AWS managed key (aws/s3) and your customer managed keys appear in this - list. For more information about customer managed keys, see Customer keys and - AWS keys in the AWS Key Management Service Developer Guide. -To enter the KMS key ARN, choose Enter AWS KMS key - ARN, and enter your KMS key ARN in the field that appears. - -To create a new customer managed key in the AWS KMS console, choose Create a - KMS key. -For more information about creating an AWS KMS key, see Creating keys in the AWS Key Management Service Developer Guide. -You can use only KMS keys that are available in the same AWS Region as the - bucket. The Amazon S3 console lists only the first 100 KMS keys in the same Region as the bucket. - To use a KMS key that is not listed, you must enter your KMS key ARN. If you want to use - a KMS key that is owned by a different account, you must first have permission to use the - key and then you must enter the KMS key ARN. For more information on cross account permissions for KMS keys, see Creating KMS keys that other accounts can use in the AWS Key Management Service Developer Guide. For more information on SSE-KMS, see Specifying server-side encryption with AWS KMS - (SSE-KMS). -When you use an AWS KMS key for server-side encryption in Amazon S3, you must - choose a symmetric encryption KMS key. Amazon S3 supports only symmetric encryption KMS keys and not - asymmetric KMS keys. For more information, see Identifying symmetric and - asymmetric KMS keys in the AWS Key Management Service Developer Guide. -For more information about creating an AWS KMS key, see Creating keys in the - AWS Key Management Service Developer Guide. For more information about using AWS KMS with - Amazon S3, see Using server-side encryption with AWS KMS keys - (SSE-KMS). -When you configure your bucket to use default encryption with SSE-KMS, you can also - enable S3 Bucket Keys. S3 Bucket Keys lower the cost of encryption by decreasing request traffic from - Amazon S3 to AWS KMS. For more information, see Reducing the cost of SSE-KMS with Amazon S3 Bucket Keys. -To use S3 Bucket Keys, under Bucket Key, choose - Enable. -(Optional) If you want to enable S3 Object Lock, do the - following: -Choose Advanced settings. -Enabling Object Lock also enables versioning for - the bucket. After enabling you must configure the Object Lock default - retention and legal hold settings to protect new objects from being deleted - or overwritten. -If you want to enable Object Lock, choose - Enable, read the warning that appears, and acknowledge it. -For more information, see Locking objects with Object Lock. -To create an Object Lock enabled bucket, you must have the following permissions: s3:CreateBucket, s3:PutBucketVersioning and s3:PutBucketObjectLockConfiguration. -Choose Create bucket. -When you use the AWS SDKs to create a bucket, you must create a client and then - use the client to send a request to create a bucket. As a best practice, you should - create your client and bucket in the same AWS Region. If you don't specify a - Region when you create a client or a bucket, Amazon S3 uses the default Region, - US East (N. Virginia). If you want to constrain the bucket creation to a specific - AWS Region, use the LocationConstraint condition key. -To create a client to access a dual-stack endpoint, you must specify an - AWS Region. For more information, see Using Amazon S3 dual-stack endpoints - in the Amazon S3 API Reference - . For a list of - available AWS Regions, see Regions and endpoints in the - AWS General Reference. -When you create a client, the Region maps to the Region-specific endpoint. The - client uses this endpoint to communicate with Amazon S3: - s3.region.amazonaws.com. If your - Region launched after March 20, 2019, your client and bucket must be in the same - Region. However, you can use a client in the US East (N. Virginia) Region to create a bucket in - any Region that launched before March 20, 2019. For more information, see Legacy endpoints. -These AWS SDK code examples perform the following tasks: -Create a client by explicitly specifying an - AWS Region – In the example, the client uses the - s3.us-west-2.amazonaws.com endpoint to communicate with - Amazon S3. You can specify any AWS Region. For a list of AWS Regions, see - Regions and - endpoints in the AWS General Reference. - -Send a create bucket request by specifying only a - bucket name – The client sends a request to Amazon S3 to - create the bucket in the Region where you created a client. -Retrieve information about the location of the - bucket – Amazon S3 stores bucket location information in - the location subresource that is associated with the - bucket. -The following example shows you how to create a bucket with a GUID at the end - of the bucket name in US East (N. Virginia) Region (us-east-1;) by using the - AWS SDK for Java. For information about other AWS SDKs, see Tools to Build on AWS. -This example shows you how to create an Amazon S3 bucket using the AWS SDK for Java. - For instructions on creating and testing a working sample, see Getting - Started in the AWS SDK for Java Developer Guide. -For information about how to create and test a working sample, see - AWS - SDK for .NET Version 3 API Reference. -For information about how to create and test a working sample, see - AWS SDK for - Ruby - Version 3. -The following AWS CLI example creates a bucket in the US West (N. California) Region - (us-west-1) Region with an example bucket name that uses a globally - unique identifier (GUID). -For more - information, see create-bucket in the AWS CLI Command Reference. - Javascript is disabled or is unavailable in your browser. -To use the Amazon Web Services Documentation, Javascript must be enabled. Please refer to your browser's Help pages for instructions. -Thanks for letting us know we're doing a good job! -If you've got a moment, please tell us what we did right so we can do more of it. - -Thanks for letting us know this page needs work. We're sorry we let you down. -If you've got a moment, please tell us how we can make the documentation better. - diff --git a/crawl/crawled_data/Example metadata table queries - Amazon Simple Storage Service.txt b/crawl/crawled_data/Example metadata table queries - Amazon Simple Storage Service.txt deleted file mode 100644 index 0c13f2fd..00000000 --- a/crawl/crawled_data/Example metadata table queries - Amazon Simple Storage Service.txt +++ /dev/null @@ -1,40 +0,0 @@ -Title: Example metadata table queries - Amazon Simple Storage Service - -The S3 Metadata feature is in preview release for Amazon S3 and is subject to change. -The following examples show how you can get different types information from your S3 - Metadata tables by using standard SQL queries. -Remember when using these examples: -The examples are written to work with Amazon Athena. You might have to modify the examples - to work with a different query engine. -Make sure that you understand how to optimize your - queries. -Replace amzn-s3-demo-bucket with the name of the S3 table bucket that's - storing your metadata table. -Replace my_metadata_table with the name of the - metadata table that you're querying. -For a full list of supported columns, see the S3 Metadata tables schema. -The following query returns objects with a specific file extension (.jpg in - this case). -The following query returns object deletion events, including the AWS account ID or - AWS service principal that made the request. -The following query returns the ARNs of the AWS Key Management Service (AWS KMS) keys encrypting your - objects. -The following query returns objects that aren't encrypted with AWS KMS keys. -Some AWS services (such as Amazon Bedrock), - upload objects to Amazon S3. You can query the object metadata provided by these services. For - example, the following query includes the user_metadata column to determine if - there are objects uploaded by Amazon Bedrock to a general purpose bucket. -If Amazon Bedrock uploaded an object to your bucket, the user_metadata column will - display the following metadata associated with the object in the query result: -The following query can help you determine the current state of your objects. The query - identifies the most recent version of each object, filters out deleted objects, and marks - the latest version of each object based on sequence numbers. Results are ordered by the - bucket, key, and sequence_number columns. - Javascript is disabled or is unavailable in your browser. -To use the Amazon Web Services Documentation, Javascript must be enabled. Please refer to your browser's Help pages for instructions. -Thanks for letting us know we're doing a good job! -If you've got a moment, please tell us what we did right so we can do more of it. - -Thanks for letting us know this page needs work. We're sorry we let you down. -If you've got a moment, please tell us how we can make the documentation better. - diff --git a/crawl/crawled_data/General purpose bucket naming rules - Amazon Simple Storage Service.txt b/crawl/crawled_data/General purpose bucket naming rules - Amazon Simple Storage Service.txt deleted file mode 100644 index 735132ff..00000000 --- a/crawl/crawled_data/General purpose bucket naming rules - Amazon Simple Storage Service.txt +++ /dev/null @@ -1,115 +0,0 @@ -Title: General purpose bucket naming rules - Amazon Simple Storage Service - -When you create a general purpose bucket, you choose its name and the AWS Region to create it in. After - you create a general purpose bucket, you can't change its name or Region. The following sections provide information - about general purpose bucket naming, including naming rules, best practices, and an example for creating a general purpose bucket - with a name that includes a globally unique identifier (GUID). -For directory bucket naming rules, see Directory bucket naming rules. - For information on object key names, see Creating object key names. -The following naming rules apply for general purpose buckets. -Bucket names must be between 3 (min) and 63 (max) characters long. -Bucket names can consist only of lowercase letters, numbers, dots (.), and - hyphens (-). -Bucket names must begin and end with a letter or number. -Bucket names must not contain two adjacent periods. -Bucket names must not be formatted as an IP address (for example, - 192.168.5.4). -Bucket names must not start with the prefix xn--. -Bucket names must not start with the prefix sthree-. -Bucket names must not start with the prefix - sthree-configurator. -Bucket names must not start with the prefix - amzn-s3-demo-. -Bucket names must not end with the suffix -s3alias. This suffix - is reserved for access point alias names. For more information, see Using a bucket-style alias for your S3 bucket - access point. -Bucket names must not end with the suffix --ol-s3. This suffix is - reserved for Object Lambda Access Point alias names. For more information, see How to use a bucket-style alias for your S3 bucket - Object Lambda Access Point. -Bucket names must not end with the suffix .mrap. This suffix is - reserved for Multi-Region Access Point names. For more information, see Rules for naming Amazon S3 Multi-Region Access Points. -Bucket names must not end with the suffix --x-s3. This suffix is - reserved for directory buckets. For more information, see Directory bucket naming rules. -Bucket names must be unique across all AWS accounts in all the AWS Regions - within a partition. A partition is a grouping of Regions. AWS currently has - three partitions: aws (Standard Regions), aws-cn - (China Regions), and aws-us-gov (AWS GovCloud (US)). -A bucket name cannot be used by another AWS account in the same partition - until the bucket is deleted. -Buckets used with Amazon S3 Transfer Acceleration can't have dots (.) in their names. For - more information about Transfer Acceleration, see Configuring fast, secure file transfers using - Amazon S3 Transfer Acceleration. -Bucket names must be unique across all AWS accounts in all the - AWS Regions within a partition. A partition is a grouping of Regions. - AWS currently has three partitions: aws (Standard Regions), - aws-cn (China Regions), and aws-us-gov - (AWS GovCloud (US)). -A bucket name cannot be used by another AWS account in the same - partition until the bucket is deleted. After you delete a bucket, - be aware that another AWS account in the same partition can use the - same bucket name. -Before March 1, 2018, buckets created in the US East (N. Virginia) Region could have - names that were up to 255 characters long and included uppercase letters and - underscores. Beginning March 1, 2018, new buckets in US East (N. Virginia) must conform - to the same rules applied in all other Regions. -The following example bucket names are valid and follow the recommended naming - guidelines for general purpose buckets: -docexamplebucket-1a1b2c3d4-5678-90ab-cdef-EXAMPLEaaaaa -amzn-s3-demo-bucket1-a1b2c3d4-5678-90ab-cdef-EXAMPLE11111 -amzn-s3-demo-bucket-a1b2c3d4-5678-90ab-cdef-EXAMPLE22222 -amzn-s3-demo-bucket2 -The following example bucket names are valid but not recommended for uses other than - static website hosting: -example.com -www.example.com -my.example.s3.bucket -The following example bucket names are not - valid: -amzn_s3_demo_bucket (contains underscores) -AmznS3DemoBucket (contains uppercase letters) -amzn-s3-demo-bucket- (ends with a hyphen) -When naming your buckets, consider the following bucket naming best practices. -If your application automatically creates buckets, choose a bucket naming scheme - that is unlikely to cause naming conflicts. Ensure that your application logic will - choose a different bucket name if a bucket name is already taken. -We recommend that you create bucket names that are not predictable. Do not write - code assuming your chosen bucket name is available unless you have already created - the bucket. One method for creating bucket names that are not predictable is to - append a Globally Unique Identifier (GUID) to your bucket name, for example, - amzn-s3-demo-bucket-a1b2c3d4-5678-90ab-cdef-EXAMPLE11111. -For best compatibility, we recommend that you avoid using dots (.) in bucket - names, except for buckets that are used only for static website hosting. If you - include dots in a bucket's name, you can't use virtual-host-style addressing over - HTTPS, unless you perform your own certificate validation. This is because the - security certificates used for virtual hosting of buckets don't work for buckets - with dots in their names. -This limitation doesn't affect buckets used for static website hosting, because static - website hosting is only available over HTTP. For more information about - virtual-host-style addressing, see Virtual hosting of buckets. For more information about static website hosting, - see Hosting a static website using Amazon S3. -When you name a bucket, we recommend that you - choose a name that is relevant to you or your business. Avoid using names associated with others. For - example, you should avoid using AWS or Amazon in your bucket name. -If a bucket is empty, you can delete it. After a bucket is deleted, the name becomes - available for reuse. However, after you delete the bucket, you might not be able to - reuse the name for various reasons. -For example, when you delete the bucket and the name becomes available for reuse, - another AWS account might create a bucket with that name. In addition, some time might - pass before you can reuse the name of a deleted bucket. If you want to use the same - bucket name, we recommend that you don't delete the bucket. -The following examples show you how to create a general purpose bucket that uses a GUID at - the end of the bucket name. -The following AWS CLI example creates a bucket in the US West (N. California) Region - (us-west-1) Region with an example bucket name that uses a globally - unique identifier (GUID). -The following example shows you how to create a bucket with a GUID at the end - of the bucket name in US East (N. Virginia) Region (us-east-1;) by using the - AWS SDK for Java. For information about other AWS SDKs, see Tools to Build on AWS. - Javascript is disabled or is unavailable in your browser. -To use the Amazon Web Services Documentation, Javascript must be enabled. Please refer to your browser's Help pages for instructions. -Thanks for letting us know we're doing a good job! -If you've got a moment, please tell us what we did right so we can do more of it. - -Thanks for letting us know this page needs work. We're sorry we let you down. -If you've got a moment, please tell us how we can make the documentation better. - diff --git a/crawl/crawled_data/Installing Mountpoint - Amazon Simple Storage Service.txt b/crawl/crawled_data/Installing Mountpoint - Amazon Simple Storage Service.txt deleted file mode 100644 index 4a32a17b..00000000 --- a/crawl/crawled_data/Installing Mountpoint - Amazon Simple Storage Service.txt +++ /dev/null @@ -1,94 +0,0 @@ -Title: Installing Mountpoint - Amazon Simple Storage Service - -You can download and install prebuilt packages of Mountpoint for Amazon S3 by using the command line. - The instructions for downloading and installing Mountpoint vary, depending on which - Linux operating system that you're using. -Copy the following download URL for your architecture. -x86_64: -ARM64 (Graviton): -Download the Mountpoint for Amazon S3 package. Replace - download-link with the appropriate download - URL from the preceding step. -(Optional) Verify the authenticity and integrity of the downloaded file. First, copy - the appropriate signature URL for your architecture. -x86_64: -ARM64 (Graviton): -Next, see Verifying the signature of the - Mountpoint for Amazon S3 package. -Install the package by using the following command: -Verify that Mountpoint is successfully installed by entering the following - command: -You should see output similar to the following: -Copy the download URL for your architecture. -x86_64: -ARM64 (Graviton): -Download the Mountpoint for Amazon S3 package. Replace - download-link with the appropriate download - URL from the preceding step. -(Optional) Verify the authenticity and integrity of the downloaded file. First, copy - the signature URL for your architecture. -x86_64: -ARM64 (Graviton): -Next, see Verifying the signature of the - Mountpoint for Amazon S3 package. -Install the package by using the following command: -Verify that Mountpoint for Amazon S3 is successfully installed by running the following - command: -You should see output similar to the following: -Consult your operating system documentation to install the FUSE and - libfuse2 packages, which are required. -Copy the download URL for your architecture. -x86_64: -ARM64 - (Graviton): -Download the Mountpoint for Amazon S3 package. Replace - download-link with the appropriate download - URL from the preceding step. -(Optional) Verify the authenticity and integrity of the downloaded file. First, copy - the signature URL for your architecture. -x86_64: -ARM64 (Graviton): -Next, see Verifying the signature of the - Mountpoint for Amazon S3 package. -Install the package by using the following command: -Add the mount-s3 binary to your PATH environment variable. - In your $HOME/.profile file, append the following line: -Save the .profile file, and run the following command: -Verify that Mountpoint for Amazon S3 is successfully installed by running the following - command: -You should see output similar to the following: -Install GnuPG (the gpg command). It is required to - verify the authenticity and integrity of a downloaded Mountpoint for Amazon S3 package. - GnuPG is installed by default on Amazon Linux Amazon Machine Images (AMIs). - After you installGnuPG, proceed to step 2. -Download the Mountpoint public key by running the following command: -Import the Mountpoint public key into your keyring by running the following - command: -Verify the fingerprint of the Mountpoint public key by running the following - command: -Confirm that the displayed fingerprint string matches the following: -If the fingerprint string doesn't match, do not finish installing - Mountpoint, and contact AWS Support. -Download the package signature file. Replace - signature-link with the appropriate - signature link from the preceding sections. -Verify the signature of the downloaded package by running the following command. - Replace signature-filename with the file name - from the previous step. -For example, on RPM-based distributions, including Amazon Linux, enter the following - command: -The output should include the phrase Good signature. If the output - includes the phrase BAD signature, redownload the Mountpoint - package file and repeat these steps. If the issue persists, do not finish installing - Mountpoint, and contact AWS Support. -The output may include a warning about a trusted signature. This does not indicate a - problem. It only means that you have not independently verified the Mountpoint - public key. - Javascript is disabled or is unavailable in your browser. -To use the Amazon Web Services Documentation, Javascript must be enabled. Please refer to your browser's Help pages for instructions. -Thanks for letting us know we're doing a good job! -If you've got a moment, please tell us what we did right so we can do more of it. - -Thanks for letting us know this page needs work. We're sorry we let you down. -If you've got a moment, please tell us how we can make the documentation better. - diff --git a/crawl/crawled_data/Installing Storage Browser for S3 - Amazon Simple Storage Service.txt b/crawl/crawled_data/Installing Storage Browser for S3 - Amazon Simple Storage Service.txt deleted file mode 100644 index 8eaa8871..00000000 --- a/crawl/crawled_data/Installing Storage Browser for S3 - Amazon Simple Storage Service.txt +++ /dev/null @@ -1,15 +0,0 @@ -Title: Installing Storage Browser for S3 - Amazon Simple Storage Service - -You can install Storage Browser for S3 from the latest version of - aws-amplify/ui-react-storage and aws-amplify packages in the - aws-amplify GitHub repository. When installing Storage Browser for S3, make - sure to add the following dependencies to your package.json file: -Alternatively, you can add the dependencies using Node Package Manager (NPM): - Javascript is disabled or is unavailable in your browser. -To use the Amazon Web Services Documentation, Javascript must be enabled. Please refer to your browser's Help pages for instructions. -Thanks for letting us know we're doing a good job! -If you've got a moment, please tell us what we did right so we can do more of it. - -Thanks for letting us know this page needs work. We're sorry we let you down. -If you've got a moment, please tell us how we can make the documentation better. - diff --git a/crawl/crawled_data/Joining custom metadata with S3 metadata tables - Amazon Simple Storage Service.txt b/crawl/crawled_data/Joining custom metadata with S3 metadata tables - Amazon Simple Storage Service.txt deleted file mode 100644 index 7612f810..00000000 --- a/crawl/crawled_data/Joining custom metadata with S3 metadata tables - Amazon Simple Storage Service.txt +++ /dev/null @@ -1,21 +0,0 @@ -Title: Joining custom metadata with S3 metadata tables - Amazon Simple Storage Service - -The S3 Metadata feature is in preview release for Amazon S3 and is subject to change. -You can analyze data across your S3 managed metadata tables and customer (self-managed) - metadata tables. By using a standard SQL JOIN operator, you can query data from - these - multiple - sources. -The following example SQL query finds matching records between an S3 managed metadata table - (my_s3_metadata_table) and a self-managed metadata - table (my_self_managed_metadata_table). The query also - filters informations based on CREATE events, which indicate that a new object (or a - new version of the object) was written to the bucket. (For more information, see the S3 Metadata tables schema.) - Javascript is disabled or is unavailable in your browser. -To use the Amazon Web Services Documentation, Javascript must be enabled. Please refer to your browser's Help pages for instructions. -Thanks for letting us know we're doing a good job! -If you've got a moment, please tell us what we did right so we can do more of it. - -Thanks for letting us know this page needs work. We're sorry we let you down. -If you've got a moment, please tell us how we can make the documentation better. - diff --git a/crawl/crawled_data/Metadata table limitations and restrictions - Amazon Simple Storage Service.txt b/crawl/crawled_data/Metadata table limitations and restrictions - Amazon Simple Storage Service.txt deleted file mode 100644 index 726aebd2..00000000 --- a/crawl/crawled_data/Metadata table limitations and restrictions - Amazon Simple Storage Service.txt +++ /dev/null @@ -1,60 +0,0 @@ -Title: Metadata table limitations and restrictions - Amazon Simple Storage Service - -The S3 Metadata feature is in preview release for Amazon S3 and is subject to change. -Before creating a metadata table configuration, be aware of the following limitations and - restrictions: -S3 Metadata is currently available only in the US East (N. Virginia), US East (Ohio), - and US West (Oregon) Regions. -S3 Metadata supports all storage classes, except for the following: -The S3 Express One Zone storage class -The S3 One Zone-Infrequent Access (S3 One Zone-IA; Z-IA) storage class in directory buckets in Local Zones -For the S3 Intelligent-Tiering storage class, the specific tier isn't shown in the - metadata table. -To create a metadata table configuration, you must create or specify an S3 table - bucket to store your metadata table in. This table bucket must be in the same AWS Region - and AWS account as your general purpose bucket. -S3 Metadata isn't supported for directory buckets or table buckets. You can create - metadata table configurations only for general purpose buckets. -S3 Metadata doesn't apply to any objects that already existed in your general purpose - bucket before you created your metadata table configuration. In other words, S3 Metadata - only captures metadata for change events (such as uploads, updates, and deletes) that - happen after you have created your metadata table configuration. -S3 Metadata is designed to continuously append to the metadata table as you make - changes to your general purpose bucket. Each update creates a snapshot—a new version of the metadata table. Because of the - read-only nature of the metadata table, you can't delete records in the metadata table. - You also can't use the snapshot expiration capability of S3 Tables to expire old snapshots - of your metadata table. -To help minimize your costs, you can periodically delete your metadata table - configuration and your metadata tables, and then recreate them. For more information, see - Deleting metadata table - configurations and Deleting metadata tables. -When you're creating or updating table bucket or table policies, make sure that you - don't restrict Amazon S3 from writing to your table bucket or your metadata table. If Amazon S3 is - unable to write to your table bucket or your metadata table, you must create a new - metadata table by deleting your metadata table configuration and your metadata table, and - then creating a new configuration. -Before you can delete a metadata table, you must first delete the associated metadata - table configuration on your general purpose bucket. -You can create a metadata table configuration only for an entire general purpose - bucket. You can't apply a metadata table configuration at the prefix level. -You can't pause and resume updates to a metadata table. Instead, you can stop a - metadata table from updating by deleting its associated metadata table configuration. To - start receiving updates again, you must create a new metadata table configuration, which - creates a new metadata table. -Metadata tables don't contain all the same metadata as is available through S3 - Inventory or through the Amazon S3 REST API. For example, the following information isn't - available in metadata tables: -S3 Lifecycle expiration or transition status -Object Lock retention period or governance mode -Object access control list (ACL) information -Replication status -You can't adjust the partitioning or sorting for metadata tables. As a result, some - queries might require table scans and therefore might be less efficient. - Javascript is disabled or is unavailable in your browser. -To use the Amazon Web Services Documentation, Javascript must be enabled. Please refer to your browser's Help pages for instructions. -Thanks for letting us know we're doing a good job! -If you've got a moment, please tell us what we did right so we can do more of it. - -Thanks for letting us know this page needs work. We're sorry we let you down. -If you've got a moment, please tell us how we can make the documentation better. - diff --git a/crawl/crawled_data/Naming Amazon S3 objects - Amazon Simple Storage Service.txt b/crawl/crawled_data/Naming Amazon S3 objects - Amazon Simple Storage Service.txt deleted file mode 100644 index 7470c8bc..00000000 --- a/crawl/crawled_data/Naming Amazon S3 objects - Amazon Simple Storage Service.txt +++ /dev/null @@ -1,123 +0,0 @@ -Title: Naming Amazon S3 objects - Amazon Simple Storage Service - -The object key (or key name) uniquely identifies the object in an - Amazon S3 bucket. When you create an object, you specify the key name. For example, on the Amazon S3 - console, when you select a bucket, a list of objects in your bucket appears. - These names are the object keys. -The object key name is a sequence of Unicode characters with UTF-8 encoding of up to 1,024 - bytes long. Object key names are case sensitive. The following section will provide - limitations on object key names and guidance on choosing key names. -Object key names with the value "soap" aren't supported for virtual-hosted-style requests. For object key name values where "soap" is used, a - path-style - URL must be used instead. -The Amazon S3 data model is a flat structure: You create a bucket, and the bucket stores - objects. There is no hierarchy of subbuckets or subfolders. However, you can infer logical - hierarchy using key name prefixes and delimiters as the Amazon S3 console does. The Amazon S3 console - supports a concept of folders. For more information about how to edit metadata from the Amazon S3 - console, see Editing object metadata in the Amazon S3 console. -Suppose that your bucket (admin-created) has four objects with the - following object keys: -Development/Projects.xls -Finance/statement1.pdf -Private/taxdocument.pdf -s3-dg.pdf -The console uses the key name prefixes (Development/, - Finance/, and Private/) and delimiter ('/') to - present a folder structure. The s3-dg.pdf key does not have a prefix, so its object appears - directly at the root level of the bucket. If you open the Development/ - folder, you see the Projects.xlsx object in it. -Amazon S3 supports buckets and objects, and there is no hierarchy. However, by using - prefixes and delimiters in an object key name, the Amazon S3 console and the AWS SDKs can infer - hierarchy and introduce the concept of folders. -The Amazon S3 console implements folder object creation by creating a zero-byte object with the folder - prefix and delimiter value as the key. These folder objects don't appear in the console. - Otherwise they behave like any other objects and can be viewed and manipulated through the REST API, AWS CLI, and AWS SDKs. -You can use any UTF-8 character in an object key name. However, using certain - characters in key names can cause problems with some applications and protocols. The - following guidelines help you maximize compliance with DNS, web-safe characters, XML - parsers, and other APIs. -The following character sets are generally safe for use in key names. -0-9 -a-z -A-Z -Exclamation point (!) -Hyphen (-) -Underscore (_) -Period (.) -Asterisk (*) -Single quote (') -Open parenthesis (() -Close parenthesis ()) -The following are examples of valid object key names: -4my-organization -my.great_photos-2014/jan/myvacation.jpg -videos/2014/birthday/video1.wmv -Objects with key names ending with period(s) "." downloaded - using the Amazon S3 console will have the period(s) "." removed from the - key name of the downloaded object. To download an object with the key - name ending in period(s) "." retained in the downloaded object, - you must use the AWS Command Line Interface (AWS CLI), AWS SDKs, or REST API. -In addition, be aware of the following prefix limitations: -Objects with a prefix of "./" must be uploaded or downloaded - with the AWS Command Line Interface (AWS CLI), AWS SDKs, or REST API. You cannot - use the Amazon S3 console. -Objects with a prefix of "../" cannot be uploaded using the - AWS Command Line Interface (AWS CLI) or Amazon S3 console. -The following characters in a key name might require additional code handling and - likely need to be URL encoded or referenced as HEX. Some of these are non-printable - characters that your browser might not handle, which also requires special - handling: -Ampersand ("&") -Dollar ("$") -ASCII character ranges 00–1F hex (0–31 decimal) and 7F (127 decimal) - -'At' symbol ("@") -Equals ("=") -Semicolon (";") -Forward slash ("/") -Colon (":") -Plus ("+") -Space – Significant sequences of spaces might be lost in some uses - (especially multiple spaces) -Comma (",") -Question mark ("?") -We recommend that you don't use the following characters in a key name because of significant special character handling, which isn't consistent across all applications. -Backslash ("\") -Left curly brace ("{") -Non-printable ASCII characters (128–255 decimal characters) -Caret ("^") -Right curly brace ("}") -Percent character ("%") -Grave accent / back tick ("`") -Right square bracket ("]") -Quotation marks -'Greater Than' symbol (">") -Left square bracket ("[") -Tilde ("~") -'Less Than' symbol ("<") -'Pound' character ("#") -Vertical bar / pipe ("|") -As specified by the XML standard on end-of-line handling, - all XML text is normalized such that single carriage returns (ASCII code 13) and carriage returns immediately followed by a - line feed (ASCII code 10) are replaced by a single line feed character. To ensure the correct parsing of object keys in XML requests, - carriage returns and other special characters must be replaced with their - equivalent XML entity code when they are inserted within XML tags. The following is a list of such special - characters and their equivalent entity codes: -' as ' -” as " -& as & -< as < -> as > -\r as or -\n as or -The following example illustrates the use of an XML entity code as a substitution for a carriage return. - This DeleteObjects request deletes an object with the key parameter: - /some/prefix/objectwith\rcarriagereturn (where the \r is the carriage return). - Javascript is disabled or is unavailable in your browser. -To use the Amazon Web Services Documentation, Javascript must be enabled. Please refer to your browser's Help pages for instructions. -Thanks for letting us know we're doing a good job! -If you've got a moment, please tell us what we did right so we can do more of it. - -Thanks for letting us know this page needs work. We're sorry we let you down. -If you've got a moment, please tell us how we can make the documentation better. - diff --git a/crawl/crawled_data/Optimizing metadata table query performance - Amazon Simple Storage Service.txt b/crawl/crawled_data/Optimizing metadata table query performance - Amazon Simple Storage Service.txt deleted file mode 100644 index b940df62..00000000 --- a/crawl/crawled_data/Optimizing metadata table query performance - Amazon Simple Storage Service.txt +++ /dev/null @@ -1,20 +0,0 @@ -Title: Optimizing metadata table query performance - Amazon Simple Storage Service - -The S3 Metadata feature is in preview release for Amazon S3 and is subject to change. -Since S3 Metadata is based on the Apache Iceberg table format, you can optimize the - performance and cost of - your metadata table queries by using specific time ranges. -For example, the following SQL query provides the sensitivity level of new objects in an - S3 general purpose bucket: -This query scans the entire metadata table, which might take a long time to run. To - improve performance, you can include the record_timestamp column to focus on a - specific time range. Here's an updated version of the previous query that looks at new objects - from the past month: - Javascript is disabled or is unavailable in your browser. -To use the Amazon Web Services Documentation, Javascript must be enabled. Please refer to your browser's Help pages for instructions. -Thanks for letting us know we're doing a good job! -If you've got a moment, please tell us what we did right so we can do more of it. - -Thanks for letting us know this page needs work. We're sorry we let you down. -If you've got a moment, please tell us how we can make the documentation better. - diff --git a/crawl/crawled_data/S3 Metadata tables schema - Amazon Simple Storage Service.txt b/crawl/crawled_data/S3 Metadata tables schema - Amazon Simple Storage Service.txt deleted file mode 100644 index 83d87e76..00000000 --- a/crawl/crawled_data/S3 Metadata tables schema - Amazon Simple Storage Service.txt +++ /dev/null @@ -1,146 +0,0 @@ -Title: S3 Metadata tables schema - Amazon Simple Storage Service - -The S3 Metadata feature is in preview release for Amazon S3 and is subject to change. -Amazon S3 metadata tables contain rows and columns. Each row represents a mutation event that has - created, updated, or deleted an object in your general purpose bucket. Most of these events are - the result of various user actions, but some of these events are the result of actions taken by - Amazon S3 on your behalf, such as S3 Lifecycle expirations or storage class transitions. -S3 Metadata is an event-processing pipeline that is designed to keep the metadata table - eventually consistent with what changes have occurred in your general purpose bucket. Be aware - that by the time that S3 Metadata is notified that an object is created or updated, that object - might already have been overwritten or deleted in the bucket. To create a row for an event, the - object must still exist in the bucket at the time that S3 Metadata is notified of the event. -The following is an example of a metadata table for a general purpose bucket named - amzn-s3-demo-bucket: -Metadata tables have the following schema: -bucket -key -sequence_number -The sequence number, which is an ordinal that's included in the records for a given - object. To order records of the same bucket and key, you can sort on - sequence_number. For a given bucket and key, a lexicographically larger - sequence_number value implies that the record was introduced to the - bucket more recently. -record_type -The type of this record, one of CREATE, UPDATE_METADATA, - or DELETE. -CREATE records indicate that a new object (or a new version of the - object) was written to the bucket. -UPDATE_METADATA records capture changes to mutable metadata for an - existing object, such as the storage class or tags. -DELETE records indicate that this object (or this version of the - object) has been deleted. When versioning is enabled, DELETE records - represent either a delete marker or a permanent delete. Delete markers have a - record_type value of DELETE and an - is_delete_marker value of True. Permanent delete records - have null values in all other columns except bucket, key, - sequence_number, record_type, - record_timestamp, and version_id. For more information, - see Deleting object versions from a - versioning-enabled bucket. -record_timestamp -The timestamp that's associated with this record. -version_id -The object's version ID. When you enable versioning on a bucket, Amazon S3 assigns a - version number to objects that are added to the bucket. For more information, see - Retaining multiple versions of objects with S3 Versioning. -Objects that are stored in your bucket before you set the versioning state have a - version ID of null. -is_delete_marker -The object's delete marker status. If the object is a delete marker, this value is - True. Otherwise, it's False. For more information, see - Working with delete markers. -Rows that are added for delete markers have a record_type value of - DELETE, not UPDATE_METADATA. If the delete marker is - created as the result of an S3 Lifecycle expiration, the requester - value is s3.amazonaws.com. -size -The object size in bytes, not including the size of incomplete multipart uploads or - object metadata. If is_delete_marker is True, the size is - 0. For more information, see System-defined object metadata. -last_modified_date -The object creation date or the last modified date, whichever is the latest. For - multipart uploads, the object creation date is the date when the multipart upload is - initiated. For more information, see System-defined object metadata. -e_tag -The entity tag (ETag), which is a hash of the object. The ETag reflects changes - only to the contents of an object, not to its metadata. The ETag can be an MD5 digest of - the object data. Whether the ETag is an MD5 digest depends on how the object was created - and how it's encrypted. For more information, see Object in the - Amazon S3 API Reference. -storage_class -The storage class that’s used for storing the object. One of STANDARD, - REDUCED_REDUNDANCY, STANDARD_IA, ONEZONE_IA, - INTELLIGENT_TIERING, GLACIER, DEEP_ARCHIVE, or - GLACIER_IR. For more information, see Understanding and managing Amazon S3 storage classes. -is_multipart -The object's upload type. If the object was uploaded as a multipart upload, this - value is True. Otherwise, it's False. For more information, - see Uploading and copying objects using multipart upload in Amazon S3. -encryption_status -The object's server-side encryption status, depending on what kind of encryption key is used: - server-side encryption with Amazon S3 managed keys (SSE-S3), server-side encryption with - AWS Key Management Service (AWS KMS) keys (SSE-KMS), dual-layer server-side encryption with - AWS KMS keys (DSSE-KMS), or server-side encryption with customer-provided keys - (SSE-C). If the object is unencrypted, this value is null. Possible values are - SSE-S3, SSE-KMS, DSSE-KMS, - SSE-C, or null. For more information, see Protecting data with encryption. -is_bucket_key_enabled -The object's S3 Bucket Key enablement status. If the object uses an S3 Bucket Key - for SSE-KMS, this value is True. Otherwise, it's False. For - more information, see Configuring an S3 Bucket Key at the object - level. -kms_key_arn -The Amazon Resource Name (ARN) for the KMS key with which the object is encrypted, - for rows where encryption_status is SSE-KMS or - DSSE-KMS. If the object isn't encrypted with SSE-KMS or DSSE-KMS, the - value is null. For more information, see Using server-side encryption with AWS KMS keys - (SSE-KMS) and - Using dual-layer server-side encryption with AWS KMS keys - (DSSE-KMS). -If a row represents an object version that no longer existed at the time that a - delete or overwrite event was processed, kms_key_arn contains a null - value, even if the encryption_status column value is - SSE-KMS or DSSE-KMS. -checksum_algorithm -The algorithm that’s used to create the checksum for the object, one of - CRC64-NVME, CRC32, CRC32C, SHA1, - or SHA256. If no checksum is present, this value is null. For more - information, see Using supported checksum algorithms. -object_tags -The object tags that are associated with the object. Object tags are stored as a - map of key-value pairs. If an object has no object tags, an empty map - ({}) is stored. For more information, see Categorizing your storage using tags -If the record_type value is DELETE, the - object_tags column contains a null value. If the - record_type value is CREATE or - UPDATE_METADATA, rows that represent object versions that no longer - existed at the time that a delete or overwrite event was processed will contain a - null value in the object_tags column. -user_metadata -The user metadata that's associated with the object. User metadata is stored as a - map of key-value pairs. If an object has no user metadata, an empty map - ({}) is stored. For more information, see User-defined object metadata. -If the record_type value is DELETE, the - user_metadata column contains a null value. If the - record_type value is CREATE or - UPDATE_METADATA, rows that represent object versions that no longer - existed at the time that a delete or overwrite event was processed will contain a - null value in the user_metadata column. -requester -The AWS account ID of the requester or the AWS service principal that made the - request. -source_ip_address -The source IP address of the request. For records that are generated by a user request, this - column contains the source IP address of the request. For actions taken by Amazon S3 or - another AWS service on behalf of the user, this column contains a null value. -request_id -The request ID that's associated with the request. - Javascript is disabled or is unavailable in your browser. -To use the Amazon Web Services Documentation, Javascript must be enabled. Please refer to your browser's Help pages for instructions. -Thanks for letting us know we're doing a good job! -If you've got a moment, please tell us what we did right so we can do more of it. - -Thanks for letting us know this page needs work. We're sorry we let you down. -If you've got a moment, please tell us how we can make the documentation better. - diff --git a/crawl/crawled_data/Setting up Storage Browser for S3 - Amazon Simple Storage Service.txt b/crawl/crawled_data/Setting up Storage Browser for S3 - Amazon Simple Storage Service.txt deleted file mode 100644 index f86ee0e4..00000000 --- a/crawl/crawled_data/Setting up Storage Browser for S3 - Amazon Simple Storage Service.txt +++ /dev/null @@ -1,346 +0,0 @@ -Title: Setting up Storage Browser for S3 - Amazon Simple Storage Service - -To connect end users with Amazon S3 locations, you must first set up an authentication and - authorization method. There are three methods to set up an authentication and authorization - method with Storage Browser: -Method 1: Managing data access for your - customers and third party partners -Method 2: Managing data access for your IAM - principals for your AWS account -Method 3: Managing data access at - scale -With this method, you can use AWS Amplify - Auth to manage access control and security for files. This method is ideal when - you want to connect your customers or third party partners with data in S3. With this - option, your customers can authenticate using social or enterprise identity - providers. -You provide IAM credentials to your end users and third party partners using AWS Amplify - Auth with an S3 bucket that’s configured to use Amplify Storage. AWS Amplify Auth is built - on Amazon Cognito, a fully - managed customer identity and access management service where you can authenticate and - authorize users from a built-in user directory or enterprise directory, or from consumer - identity providers. The Amplify authorization model defines which prefixes the current - authenticated user can access. For more information about how to set up authorization for - AWS Amplify, see Set up - storage. -To initialize the component with the Amplify authentication and storage methods, add the - following code snippet to your web application: -If you want to manage access for your IAM principals or your AWS account directly, - you can create an IAM role that has permissions to invoke the GetDataAccess S3 API operation. To set this up, you must create an - S3 Access Grants instance to map out permissions for S3 buckets and prefixes to the specified IAM - identities. The Storage Browser component will then invoke the ListCallerAccessGrants S3 API operation to fetch the available grants to the - identity requester and populate the locations in the component. After you obtain the - s3:GetDataAccess permission, those credentials are then used by the Storage - Browser component to request data access to S3. -If you want to associate an S3 Access Grants instance to your IAM - Identity Center for a more scalable solution (such as providing data access to your whole - company), you can request data from Amazon S3 on behalf of the current authenticated user. For - example, you can grant user groups in your corporate directory access to your data in S3. - This approach allows you to centrally manage S3 Access Grants permissions for your users and groups, - including the ones hosted on external providers such as Microsoft Entra, Okta, and - others. -When using this method, the integration with the - IAM Identity Center allows you to use existing user directories. Another benefit - of an IAM Identity Center trusted identity propagation is that each AWS CloudTrail data event for Amazon S3 contains a direct reference to the end user identity - that accessed the S3 data. -If you have an application that supports OAuth 2.0 and your users need access from these - applications to AWS services, we recommend that you use trusted identity propagation. With - trusted identity propagation, a user can sign in to an application, and that application can - pass the user’s identity in any requests that access data in AWS services. This - application interacts with IAM Identity Center on behalf of any authenticated users. For - more information, see Using trusted identity propagation with customer managed applications. -To set up Storage Browser authentication in the AWS Management Console using S3 Access Grants - and IAM Identity - Center trusted identity propagation, your applications must request data from - Amazon S3 on behalf of the current authenticated user. With this approach, you can give users - or groups of users from your corporate directory direct access to your S3 buckets, - prefixes, or objects. This means that your application won’t need to map any users to an - IAM principal. -The following workflow outlines the steps for setting up Storage Browser for S3, using IAM - Identity Center and S3 Access Grants: -The trusted token issuer represents your external identity provider (IdP) - within IAM Identity Center, enabling it to recognize identity tokens for your - application’s authenticated users. -This application interacts with IAM Identity Center on behalf of - authenticated users. -This step connects your application to S3 Access Grants, so that it can make requests - to S3 Access Grants on behalf of authenticated users. -This step syncs users from AWS Identity and Access Management Identity Center with the System for - Cross-domain Identity Management (SCIM). SCIM keeps your IAM Identity Center - identities in sync with identities from your identity provider (IdP). -To enable IAM Identity Center for your AWS Organizations, perform the following steps: -Sign in to the AWS Management Console, using one of these methods: -New to AWS (root user) - – Sign in as the account owner by choosing Root user and - entering your AWS account email address. On the next page, enter your - password. -Already using AWS (IAM - credentials) – Sign in using your IAM - credentials with administrative permissions. -Open the IAM Identity - Center console. -Under Enable IAM Identity Center, choose - Enable. -IAM Identity Center requires the setup of AWS Organizations. If you haven't set up an - organization, you can choose to have AWS create one for you. Choose - Create AWS organization to complete this process. -Choose Enable with AWS Organizations. -Choose Continue. -(Optional) Add any tags that you want to associate with this organization - instance. -(Optional) Configure the delegated administration. -If you’re using a multi-account environment, we recommend that you configure - delegated administration. With delegated administration, you can limit the number of - people who require access to the management account in AWS Organizations. For more - information, see Delegated - administration. -Choose Save. -AWS Organizations automatically sends a verification email to the address that is associated - with your management account. There might be a delay before you receive the verification - email. Make sure to verify your email address within 24 hours, before your verification - email expires. -To use Storage Browser for S3 with corporate directory users, you must configure IAM - Identity Center to use an external identity provider (IdP). You can use the preferred - identity provider of your choice. However, be aware that each identity provider uses - different configuration settings. For tutorials on using different external identity - providers, see IAM Identity Center source - tutorials. -Make sure to record the issuer URL and the audience attributes of the identity - provider that you’ve configured because you will need to refer to it in later steps. If - you don’t have the required access or permissions to configure an IdP, you might need to - contact the administrator of the external IdP to obtain them. -The trusted token issuer represents your external identity provider in the AWS Identity and Access Management - Identity Center, and recognizes tokens for your application’s authenticated users. The - account owner of the IAM Identity Center instance in your AWS Organizations must perform these - steps. These steps can be done either in the IAM Identity Center console, or - programmatically. -To add a trusted token issuer in the AWS Identity and Access Management Identity Center console, perform the - following steps: -Open the IAM Identity - Center console. -Choose Settings. -Choose the Authentication tab. -Navigate to the Trusted token issuers section, and fill out - the following details: -Under Issuer URL, enter the URL of the external IdP that - serves as the trusted token issuer. You might need to contact the administrator of - the external IdP to obtain this information. For more information, see Using applications with a trusted token issuer. -Under Trusted token issuer name, enter a name for the - trusted token issuer. This name will appear in the list of trusted token issuers - that you can select in Step 8, when an application resource is configured for - identity propagation. -Update your Map attributes to your preferred application - attribute, where each identity provider attribute is mapped to an IAM Identity - Center attribute. For example, you might want to map the - application attribute -email to the IAM Identity Center user attribute email. To - see the list of allowed user attributes in IAM Identity Center, see the table in - Attribute mappings - for AWS Managed Microsoft AD directory. -(Optional) If you want to add a resource tag, enter the key and value pair. To add - multiple resource tags, choose Add new tag to generate a new - entry and enter the key and value pairs. -Choose Create trusted token issuer. -After you finish creating the trusted token issuer, contact the application - administrator to let them know the name of the trusted token issuer, so that they can - confirm that the trusted token issuer is visible in the applicable console. -Make sure the application administrator selects this trusted token issuer in the - applicable console to enable user access to the application from applications that are - configured for trusted identity propagation. -To ensure that the bootstrap application and identity bearer - users can properly work with each other, make sure to create two IAM - roles. One IAM role is required for the bootstrap application and - the other IAM role must be used for the identity bearer, or end users who are accessing - the web application that requests access through S3 Access Grants. The bootstrap - application receives the token issued by the identity provider and invokes the - CreateTokenWithIAM API, exchanging this token with the one - issued by the Identity Center. -Create an IAM role, such as bootstrap-role, with permissions such as - the following. The following example IAM policy gives permissions to the - bootstrap-role to perform the token exchange: -Then, create a second IAM role (such as identity-bearer-role), which - the identity broker uses to generate the IAM credentials. The IAM credentials returned - from the identity broker to the web application are used by the Storage Browser for S3 component - to allow access to S3 data: -This IAM role (identity-bearer-role) must use a trust policy with the - following statement: -Before you begin, make sure that you’ve created the required IAM roles from the - previous step. You’ll need to specify one of these IAM roles in this step. -To create and configure a customer managed application in AWS IAM Identity Center, - perform the following steps: -Open the IAM Identity - Center console. -Choose Applications. -Choose the Customer managed tab. -Choose Add application. -On the Select application type page, under Setup - preference, choose I have an application I want to set - up. -Under Application type, choose OAuth - 2.0. -Choose Next. The Specify application - page is displayed. -Under the Application name and descriptionsection, enter a - Display name for the application, such as - storage-browser-oauth. -Enter a Description. The application description appears in - the IAM Identity Center console and API requests, but not in the AWS access - portal. -Under User and group assignment method, choose Do - not require assignments. This option allows all authorized IAM Identity - Center users and groups access to this application. -Under AWS access portal, enter an application URL where - users can access the application. -(Optional) If you want to add a resource tag, enter the key and value pair. To add - multiple resource tags, choose Add new tag to generate a new - entry and enter the key and value pairs. -Choose Next. The Specify authentication - page displays. -Under Authentication with trusted token issuer, use the - checkbox to select the trusted token issuer that you previously created. -Under Configure selected trusted token issuers, enter the - aud claim. The aud claim identifies the audience of - the JSON Web Token (JWT), and it is the name by which the trusted token issuer - identifies this application. -You might need to contact the administrator of the external IdP to obtain this - information. -Choose Next. The Specify authentication - credentials page displays. -Under Configuration method, choose Enter one or - more IAM roles. -Under Enter IAM roles, add the IAM role or Amazon Resource Name - (ARN) for the identity bearer token. You must enter the IAM role that you created from - the previous step for the identity broker application (for example, - bootstrap-role). -Choose Next. -On the Review and configure page, review the details of your - application configuration. If you need to modify any of the settings, choose - Edit for the section that you want to edit and make your - changes to. -Choose Submit. The details page of the application that you - just added is displayed. -After you’ve set up your applications, your users can access your applications from - within their AWS access portal based on the permission sets that you’ve created and the user - access that you’ve assigned. -After you set up your customer managed application, you must specify S3 Access Grants for - identity propagation. S3 Access Grants vends credentials for users to access Amazon S3 data. When you sign in to your customer - managed application, S3 Access Grants will pass your user identity to the trusted application. - -Prerequisite: Make sure that you’ve set up S3 Access Grants (such - as creating an S3 Access Grants - instance and registering a - location) before following these steps. For more information, see Getting started with S3 Access Grants. -To add S3 Access Grants for identity propagation to your customer managed application, perform - the following steps: -Open the IAM Identity - Center console. -Choose Applications. -Choose the Customer managed tab. -In the Customer managed applications list, select the OAuth - 2.0 application for which you want to initiate access requests. This is the - application that your users will sign in to. -On the Details page, under Trusted applications for - identity propagation, choose Specify trusted - applications. -Under Setup type, select Individual applications and specify access, and then - choose Next. -On the Select service page, choose - S3 Access Grants. S3 Access Grants has applications that you can use to define your - own web application for trusted identity propagation. -Choose Next. You'll select your applications in the next - step. -On the Select applications page, choose Individual - applications, select the checkbox for each application that can receive - requests for access, and then choose Next. -On the Configure access page, under Configuration - method, choose either of the following: -Select access per application – Select - this option to configure different access levels for each application. Choose the - application for which you want to configure the access level, and then choose - Edit access. In Level of access to - apply, change the access levels as needed, and then choose - Save changes. -Apply same level of access to all - applications – Select this option if you don't - need to configure access levels on a per-application basis. -Choose Next. -On the Review configuration page, review the choices that you - made. -You’ll want to make sure the s3:access_grants:read_write permission - is granted for your users. This permission allows your users to retrieve credentials - to access Amazon S3. Make sure to use either the IAM policy you created previously, or - S3 Access Grants, to limit access to write operations. -To make changes, choose Edit for the configuration section - that you want to make changes to. Then, make the required changes and choose - Save changes. -Choose Trust applications to add the trusted application for - identity propagation. -In this step, you use IAM Identity Center to provision your users. You can use SCIM - for automated or manual - provisioning of users and groups. SCIM keeps your IAM Identity Center - identities in sync with identities from your identity provider (IdP). This includes any - provisioning, updates, and deprovisioning of users between your IdP and IAM Identity - Center. -This step is required because when S3 Access Grants is used with IAM Identity Center, local - IAM Identity Center users aren’t used. Instead, users must be synchronized from the - identity provider with IAM Identity Center. -To synchronize users from your identity provider with IAM Identity Center, perform - the following steps: -Enable automatic - provisioning. -Generate an access token. -For examples of how to set up the identity provider with IAM Identity Center for - your specific use case, see - IAM - Identity Center Identity source tutorials. -After you’ve set up your IAM Identity Center instance and created grants in S3 Access Grants, - open your React application. In the React application, use - createManagedAuthAdapter to set up the authorization rules. You must - provide a credentials provider to return the credentials you acquired from IAM Identity - Center. You can then call createStorageBrowser to initialize the - Storage Browser for S3 component: - -Then, create a mechanism to exchange the JSON web tokens (JWTs) from your web - application with the IAM credentials from IAM Identity Center. For more information - about how to exchange the JWT, see the following resources: -How to develop a user-facing data application with IAM Identity Center and - S3 Access Grants post in AWS Storage Blog -Scaling data access with S3 Access Grants post in AWS Storage - Blog -S3 Access Grants workshop on AWS workshop studio -S3 Access Grants workshop on GitHub -Then, set up an API endpoint to handle requests for fetching credentials. To validate - the JSON web token (JWT) exchange, perform the following steps: -Retrieve the JSON web token from the authorization header for incoming - requests. -Validate the token using the public keys from the specified JSON web key set - (JWKS) URL. -Verify the token's expiration, issuer, subject, and audience claims. -To exchange the identity provider’s JSON web token with AWS IAM credentials, - perform the following steps: -Make sure to avoid logging any sensitive information. We recommend that you use - error handling controls for missing authorization, expired tokens, and other exceptions. - For more information, see the Implementing AWS Lambda error handling patterns post in AWS Compute - Blog. -Verify that the required Permission and - Scope parameters are provided in the request. -Use the CreateTokenWithIAM API to exchange the JSON web token for an IAM Identity - Center token. -After the IdP JSON web token is used, it can’t be used again. A new token must - be used to exchange with IAM Identity Center. -Use the AssumeRole API operation - to assume a transient role using the IAM Identity Center token. Make sure to use the - identity bearer role, also known as the role that carries the identity context (for - example, identity-bearer-role) to request the - credentials. -Return the IAM credentials to the web application. -Make sure that you’ve set up a proper logging mechanism. Responses are returned - in a standardized JSON format with an appropriate HTTP status code. - Javascript is disabled or is unavailable in your browser. -To use the Amazon Web Services Documentation, Javascript must be enabled. Please refer to your browser's Help pages for instructions. -Thanks for letting us know we're doing a good job! -If you've got a moment, please tell us what we did right so we can do more of it. - -Thanks for letting us know this page needs work. We're sorry we let you down. -If you've got a moment, please tell us how we can make the documentation better. - diff --git a/crawl/crawled_data/Setting up permissions for configuring metadata tables - Amazon Simple Storage Service.txt b/crawl/crawled_data/Setting up permissions for configuring metadata tables - Amazon Simple Storage Service.txt deleted file mode 100644 index a602bde5..00000000 --- a/crawl/crawled_data/Setting up permissions for configuring metadata tables - Amazon Simple Storage Service.txt +++ /dev/null @@ -1,45 +0,0 @@ -Title: Setting up permissions for configuring metadata tables - Amazon Simple Storage Service - -The S3 Metadata feature is in preview release for Amazon S3 and is subject to change. -To create a metadata table configuration, you must have the necessary AWS Identity and Access Management (IAM) - permissions to both create and manage your metadata table configuration and to create and - manage your metadata table and the table bucket where your metadata table is stored. -To create and manage your metadata table configuration, you must have these permissions: -s3:CreateBucketMetadataTableConfiguration – This permission allows - you to create a metadata table configuration for your general purpose bucket. -s3:GetBucketMetadataTableConfiguration – This permission allows - you to retrieve information about your metadata table configuration. -s3:DeleteBucketMetadataTableConfiguration – This permission allows - you to delete your metadata table configuration. -To create and work with tables and table buckets, you must have certain - s3tables permissions. At a minimum, to create a metadata table configuration, - you must have the following s3tables permissions: -s3tables:CreateNamespace – This permission allows you to create a - namespace in a table bucket. Metadata tables use the default aws_s3_metadata - namespace. -s3tables:GetTable – This permission allows you to retrieve - information about your metadata table. -s3tables:CreateTable – This permission allows you to create your - metadata table. -s3tables:PutTablePolicy – This permission allows you to add or - update your metadata table policy. -For detailed information about all table and table bucket permissions, see Access - management for S3 Tables. -If you also want to integrate your table bucket with AWS analytics services so that you - can query your metadata table, you need additional permissions. For more information, see - - Integrating Amazon S3 Tables with AWS analytics services. -To create and work with metadata tables and table buckets, you can use the following - example policy. In this policy, the general purpose bucket that you're applying the metadata - table configuration to is referred to as amzn-s3-demo-source-bucket. The table - bucket where you're storing your metadata table is referred to as - amzn-s3-demo-bucket. To use this policy, replace these bucket names and the - user input placeholders with your own information: - Javascript is disabled or is unavailable in your browser. -To use the Amazon Web Services Documentation, Javascript must be enabled. Please refer to your browser's Help pages for instructions. -Thanks for letting us know we're doing a good job! -If you've got a moment, please tell us what we did right so we can do more of it. - -Thanks for letting us know this page needs work. We're sorry we let you down. -If you've got a moment, please tell us how we can make the documentation better. - diff --git a/crawl/crawled_data/Top DevOps Challenges and its SolutionsTop Challenges in DevOps and How to Solve Them | BrowserStack.txt b/crawl/crawled_data/Top DevOps Challenges and its SolutionsTop Challenges in DevOps and How to Solve Them | BrowserStack.txt deleted file mode 100644 index ae517c70..00000000 --- a/crawl/crawled_data/Top DevOps Challenges and its SolutionsTop Challenges in DevOps and How to Solve Them | BrowserStack.txt +++ /dev/null @@ -1,84 +0,0 @@ -Title: Top DevOps Challenges and its SolutionsTop Challenges in DevOps and How to Solve Them | BrowserStack - - Transform your testing process with: Real Device Cloud, Company-wide Licences & Accessibility Testing -App & Browser Testing Made Easy -Give your users a seamless experience by testing on 3000+ real devices and browsers. Don't compromise with emulators and simulators - January 25, 2023 -The primary goal of DevOps is to make the overall SDLC workflow more efficient. Since DevOps comprehensively refers to the overall culture surrounding Software Development it needs to optimise the pipelines by looking at the people, process, and technology that makes up the software ecosystem. Factors such as cultural obstacles, poor communication in a team, legacy infrastructure, process failures, etc all lead to roadblocks in DevOps. This article will dive into some of the top challenges in DevOps and how they can be solved.  -Must Read : What is a DevOps Pipeline? How to Build One -During DevOps the ownership of the codebase changes from one team to another; from development to testing, to deployment and production. During such transitions there is a general waste of time and effort as the environments used by these teams are separately configured and the codebase must be modified to work smoothly on all these environments. This sometimes causes teams to also waste time trying to investigate for problems in the code rather than in the environment on which it is run. -Solutions -Creating infrastructural blueprints for Continuous Delivery enablement and making sure that all environments are identical. This usually requires all teams to get together and plan the Continuous Delivery process to ensure smooth transition.  -A good solution to enable this is to adopt a cloud-based infrastructure for DevOps enablement. The different parts of the DevOps pipeline, viz. coding, builds, testing, deployment, and post-production monitoring require different tools to enable and are hosted on different environments.  -Hosting these pipelines on the cloud helps to create a centralised architecture that helps different teams to access the codebase and continue to develop the pipeline as the cloud environment manages the environment transition.  - -Pro Tip : BrowserStack is an industry leader in cloud-based infrastructure for DevOps with over 3000 real browsers, devices and OS available at the touch of a button. It offers integrations with popular CI/CD tools like Jenkins, Travis CI, TeamCity, Bamboo, etc and facilitates automated testing by leveraging the power of Appium, Espresso, XCUITest, EarlGrey, Cypress, Playwright,  Puppeteer. -Run DevOps Tests on Real Devices -The level of maturity and competence a software engineering team has with the Software Development Life Cycle is directly related to their ability to be able to adapt to the DevOps transformation of these processes.  -One of the key reasons behind the adoption of DevOps is the need to deliver quality software in shorter deadlines in a more reliable fashion. The DevOps process seeks to transform the conventional Software Development process by creating a continuous cycle of Code-Build-Test and to better integrate the development and operations processes to be able to achieve its goal of quality software delivered on time.  -Solutions -Organisations adopting DevOps need to adopt the correct tools and technologies and should be able to invest in the correct training and upskilling for their employees. A robust devops culture requires the following steps :  - -Organisations adopt DevOps looking to orchestrate a transformation in their SDLC processes, and a major reason for this is their struggle to modernise legacy processes and practices.  -Organisations historically have worked in silos, with specific teams dedicated to performing certain tasks such as development, testing, and operations. These teams usually work in silos and have minimal communication with each other. It does not help that these teams work with outdated tools that do not allow for greater flexibility of communication and pipeline efficiency. -Solutions -Though DevOps pipelines depend on automation, some parts still require human communication and cooperation. Having transparency and cooperation in teams helps smoothen the pipelines and improve overall efficiency. -Read More: 5 DevOps Tools Every Team Must Have -BrowserStack Live for Teams allows teams to collaborate effectively by allowing seamless connections with existing tools and helping to track bugs. Issues can be filed in an organised manner and assigned to the relevant members on the same dashboard thus creating a unified approach to handle the entire DevOps pipeline.  -Try BrowserStack Live for Teams -One of the most common problems with DevOps is the challenge in holistically monitoring the entire process. DevOps consists of several moving parts and each of these have different metrics to judge their effectiveness.  -For example a metric like number of code branches or deployment frequency might deal with the CI/CD process; whereas something like Defect Escape Rate is a part of the Continuous Testing pipeline.  -Often there is lack of clear visibility over the entire process and that often leads to finger-pointing and delays in production. Any manual processes made to envisage this leads to extensive labour and risks incorrect updates due to human error.  -Solutions -Continuous Monitoring tools like Nagios Core can allow the continuous overview of applications, metrics, services, network components, etc. It allows users to have a birds eye view of the infrastructure and detect critical issues. It also offers useful altering services to resolve these issues and provides a great degree of scalability and flexibility. It also provides up to date logs of any DevOps events, failures, code crashes, and system outages.  -This automated viewport allows teams to exactly know what has gone wrong and where and subsequently make the changes required to correct it.  -A suboptimal implementation of the CI/CD pipeline leads to recurring issues such as slow page loads for websites, delayed responses from servers, and poor memory optimization that hampers the overall performance of the application.  -A Deloitte study indicates that suboptimal website performance increases the bounce rate and decreases overall time on the site. A mere 1 second delay can result in  -Solutions -Automated testing principles can be extended by the QA team to check for performance using tools like Apache JMeter. -Must Read : JMeter vs Selenium: What is preferred by Testers? -Tools like BrowserStack SpeedLab allow real-time web performance testing and scores the website under test on key performance metrics for an overall enhanced UX. -BrowserStack Speed Lab -This free tool allows the running of speed tests for monitoring the events occurring during page load including CPU Processing Breakdown, Navigation Timing API, and Page Resource Summary. It also allows the comparison of mobile and desktop screenshots for comparing render time to provide insights and optimise performance. -Performance Measurement using BrowserStack Speed Lab -Try SpeedLab for Free -DevOps relies on version control for all components and processes to work on stable versions. However, the unexpected update or change in any process can cause the entire pipeline to break due to compatibility issues. This can commonly occur during automated system updates. -Suboptimal automated tests are the perfect storm for any Continuous Automation process in the DevOps pipeline. Since DevOps focuses on delivering production ready code in short sprints, it is imperative to have the correct automation testing tools and techniques in place to avoid flawed tests. -Read More: DevOps Testing Strategy -Solutions -One of the most common solutions to this is to put a forced stop to all auto-updates to make sure that no software updates happen without manual intervention and that unstable versions are not incorporated just for the sake of it. -It is of the utmost importance that the QA team chooses the correct testing tools which are compatible with the systems and the languages being used.  Also model-based testing helps in furthering this process by proactively identifying red flags even before they cause major issues to the overall system.  -Run Automation Tests on Real Devices -Security vulnerabilities in the DevOps pipeline make it susceptible to cyber-attacks and can lead to sensitive information being compromised.  -Solutions -Some potential solutions are: -One of the major challenges in Continuous Testing is to be able to scale the operations and the test capabilities at the same time to handle increased volumes of data, devices, etc -Solutions  -Choosing a Selenium based test infrastructure that flexibility handles version updates, increased device load, and manages data capacity in a stable manner.  - -Several test reports are too complex to understand by all the stakeholders and this delays bug fixes thereby affecting product release velocity. -Solutions -Pro Tip: BrowserStack’s Test Insights offers an interactive dashboard to help obtain actionable insights. This helps teams to recognize critical issues or bottlenecks and accelerate product release velocity. -BrowserStack Test Insights -Summary -Though there are a number of challenges in DevOps, with the right tools and training they can all be overcome with ease. Regardless of the DevOps challenges being faced by the organisation, having a robust testing methodology that allows for accurate results on real browsers and devices is a must. -BrowserStack provides more than 3000+ real browsers and devices on a Cloud Selenium Grid with 99% uptime and assists the product team in many ways:  -Learn more about DevOps configuration management, its importance, and why it is beneficial for enter... -DevOps work towards making Software Release faster with high quality. DevOps recommend Shift Left Pr... -Are you looking for a way to improve your DevOps team's efficiency and effectiveness? Cloud-based so... -Over 6 million developers and 50,000 teams test on BrowserStack. Join them. -PRODUCTS -WHY BROWSERSTACK -RESOURCES -COMPANY -Social -More Resources -Test on Devices -Tools -© 2024 BrowserStack. All rights reserved. - -Our team will get in touch with you within 12 hours -Please share some details regarding your query -We will respond back shortly to -In the meantime, here are some resources that might interest you: -Meanwhile, these links might interest you: diff --git a/crawl/crawled_data/Tracking a multipart upload with the AWS SDKs - Amazon Simple Storage Service.txt b/crawl/crawled_data/Tracking a multipart upload with the AWS SDKs - Amazon Simple Storage Service.txt deleted file mode 100644 index e1a759b3..00000000 --- a/crawl/crawled_data/Tracking a multipart upload with the AWS SDKs - Amazon Simple Storage Service.txt +++ /dev/null @@ -1,28 +0,0 @@ -Title: Tracking a multipart upload with the AWS SDKs - Amazon Simple Storage Service - -You can track an object's upload progress to Amazon S3 with a listen interface. The high-level - multipart upload API provides such a listen interface, called ProgressListener. - Progress events occur periodically and notify the listener that bytes have been transferred. - For more general information about multipart uploads, see Uploading and copying objects using multipart upload in Amazon S3. -For an end-to-end procedure on uploading an object with multipart upload with an additional checksum, see - Tutorial: Upload an object through multipart upload and - verify its data integrity. -The following section show how to track a multipart upload with the AWS SDKs. -The following Java code uploads a file and uses the ProgressListener - to track the upload progress. For instructions on how to create and test a - working sample, see Getting - Started in the AWS SDK for Java Developer Guide. - -The following C# example uploads a file to an S3 bucket using the - TransferUtility class, and tracks the progress of the upload. For - information about setting up and running the code examples, see Getting Started - with the AWS SDK for .NET in the AWS SDK for .NET Developer - Guide. - Javascript is disabled or is unavailable in your browser. -To use the Amazon Web Services Documentation, Javascript must be enabled. Please refer to your browser's Help pages for instructions. -Thanks for letting us know we're doing a good job! -If you've got a moment, please tell us what we did right so we can do more of it. - -Thanks for letting us know this page needs work. We're sorry we let you down. -If you've got a moment, please tell us how we can make the documentation better. - diff --git a/crawl/crawled_data/Troubleshooting Storage Browser for S3 - Amazon Simple Storage Service.txt b/crawl/crawled_data/Troubleshooting Storage Browser for S3 - Amazon Simple Storage Service.txt deleted file mode 100644 index e427bb81..00000000 --- a/crawl/crawled_data/Troubleshooting Storage Browser for S3 - Amazon Simple Storage Service.txt +++ /dev/null @@ -1,22 +0,0 @@ -Title: Troubleshooting Storage Browser for S3 - Amazon Simple Storage Service - -If you’re experiencing issues with Storage Browser for S3, make sure to review the following troubleshooting tips: -Avoid trying to use the same token (idToken or accessToken) for multiple requests. Tokens can't be reused. This will result in a request failure. -Make sure that the IAM credentials that you provide to the Storage Browser component includes permissions to invoke the s3:GetDataAccess operation. Otherwise, your end users won’t be able to access your data. -Alternatively, you can check the following resources: -Storage Browser for S3 is backed by AWS Support. If you need assistance, contact the - AWS Support - Center. -If you’re having trouble with Storage Browser for S3 or would like to submit feedback, visit - the Amplify GitHub page. - -If you discover a potential security issue in this project, you can notify AWS - Security through the AWS Vulnerability Reporting page. - Javascript is disabled or is unavailable in your browser. -To use the Amazon Web Services Documentation, Javascript must be enabled. Please refer to your browser's Help pages for instructions. -Thanks for letting us know we're doing a good job! -If you've got a moment, please tell us what we did right so we can do more of it. - -Thanks for letting us know this page needs work. We're sorry we let you down. -If you've got a moment, please tell us how we can make the documentation better. - diff --git a/crawl/crawled_data/Uploading a directory using the high-level .NET TransferUtility class - Amazon Simple Storage Service.txt b/crawl/crawled_data/Uploading a directory using the high-level .NET TransferUtility class - Amazon Simple Storage Service.txt deleted file mode 100644 index e08f173b..00000000 --- a/crawl/crawled_data/Uploading a directory using the high-level .NET TransferUtility class - Amazon Simple Storage Service.txt +++ /dev/null @@ -1,27 +0,0 @@ -Title: Uploading a directory using the high-level .NET TransferUtility class - Amazon Simple Storage Service - -You can use the TransferUtility class to upload an entire directory. By - default, the API uploads only the files at the root of the specified directory. You can, - however, specify recursively uploading files in all of the sub directories. -To select files in the specified directory based on filtering criteria, specify - filtering expressions. For example, to upload only the PDF files from a directory, - specify the "*.pdf" filter expression. -When uploading files from a directory, you don't specify the key names for the - resulting objects. Amazon S3 constructs the key names using the original file path. For - example, assume that you have a directory called c:\myfolder with the - following structure: -When you upload this directory, Amazon S3 uses the following key names: -The following C# example uploads a directory to an Amazon S3 bucket. It shows how to - use various TransferUtility.UploadDirectory overloads to upload the - directory. Each successive call to upload replaces the previous upload. For - information about setting up and running the code examples, see Getting Started - with the AWS SDK for .NET in the AWS SDK for .NET Developer - Guide. - Javascript is disabled or is unavailable in your browser. -To use the Amazon Web Services Documentation, Javascript must be enabled. Please refer to your browser's Help pages for instructions. -Thanks for letting us know we're doing a good job! -If you've got a moment, please tell us what we did right so we can do more of it. - -Thanks for letting us know this page needs work. We're sorry we let you down. -If you've got a moment, please tell us how we can make the documentation better. - diff --git a/crawl/crawled_data/Uploading and copying objects using multipart upload in Amazon S3 - Amazon Simple Storage Service.txt b/crawl/crawled_data/Uploading and copying objects using multipart upload in Amazon S3 - Amazon Simple Storage Service.txt deleted file mode 100644 index 06134949..00000000 --- a/crawl/crawled_data/Uploading and copying objects using multipart upload in Amazon S3 - Amazon Simple Storage Service.txt +++ /dev/null @@ -1,344 +0,0 @@ -Title: Uploading and copying objects using multipart upload in Amazon S3 - Amazon Simple Storage Service - -Multipart upload allows you to upload a single object to Amazon S3 as a set of parts. Each part is a - contiguous portion of the object's data. You can upload these object parts independently, and in - any order. For uploads, your updated AWS client automatically calculates a checksum of the - object and sends it to Amazon S3 along with the size of the object as a part of the request. If - transmission of any part fails, you can retransmit that part without affecting other parts. - After all parts of your object are uploaded, Amazon S3 assembles them to create the object. It's a - best practice to use multipart upload for objects that are 100 MB or larger instead of uploading them in a - single operation. -Using multipart upload provides the following advantages: -Improved throughput – You can upload parts in - parallel to improve throughput. -Quick recovery from any network issues – Smaller - part size minimizes the impact of restarting a failed upload due to a network error. -Pause and resume object uploads – You can upload - object parts over time. After you initiate a multipart upload, there is no expiry; you must - explicitly complete or stop the multipart upload. -Begin an upload before you know the final object size - – You can upload an object as you create it. -We recommend that you use multipart upload in the following ways: -If you upload large objects over a stable high-bandwidth network, use multipart upload - to maximize the use of your available bandwidth by uploading object parts in parallel for - multi-threaded performance. -If you upload over a spotty network, use multipart upload to increase resiliency against - network errors by avoiding upload restarts. When using multipart upload, you only need to - retry uploading the parts that are interrupted during the upload. You don't need to restart - uploading your object from the beginning. -For more information about using the Amazon S3 Express One Zone storage class with directory buckets, - see S3 Express One Zone and - Working with directory buckets. For more - information about using multipart upload with S3 Express One Zone and directory buckets, see - Using multipart uploads with - directory buckets. -Multipart upload is a three-step process: You initiate the upload, upload the object parts, - and—after you've uploaded all the parts—complete the multipart upload. Upon receiving the - complete multipart upload request, Amazon S3 constructs the object from the uploaded parts, and you can access - the object just as you would any other object in your bucket. -You can list all of your in-progress multipart uploads or get a list of the parts that you - have uploaded for a specific multipart upload. Each of these operations is explained in this - section. -When you send a request to initiate a multipart upload, make sure to specify a checksum type. Amazon S3 - will then return a response with an upload ID, which is a unique identifier for your multipart upload. - This upload ID is required when you upload parts, list parts, complete an upload, or stop an - upload. If you want to provide metadata describing the object being uploaded, you must - provide it in the request to initiate the multipart upload. -When uploading a part, you must specify a part number in addition to the upload ID. You - can choose any part number between 1 and 10,000. A part number uniquely identifies a part - and its position in the object you are uploading. The part number that you choose doesn’t - need to be in a consecutive sequence (for example, it can be 1, 5, and 14). Be aware that if - you upload a new part using the same part number as a previously uploaded part, the - previously uploaded part gets overwritten. -When you upload a part, Amazon S3 returns the checksum algorithm type with the checksum value - for each part as a header in the response. For each part upload, you must record the part - number and the ETag value. You must include these values in the subsequent request to complete - the multipart upload. Each part will have its own ETag at the time of upload. However, once the multipart upload is - complete and all parts are consolidated, all parts belong to one ETag as a checksum of - checksums. -After you initiate a multipart upload and upload one or more parts, you must either complete or - stop the multipart upload to stop incurring charges for storage of the uploaded parts. Only - after you complete or stop a multipart upload will Amazon S3 free up the parts - storage and stop billing you for the parts storage. -After stopping a multipart upload, you can't upload any part using that upload ID again. If part - uploads were in progress, they can still succeed or fail even after you stop the upload. To - make sure you free all storage consumed by all parts, you must stop a multipart upload only after all - part uploads have completed. -When you complete a multipart upload, Amazon S3 creates an object by concatenating the parts in - ascending order based on the part number. If any object metadata was provided in the - initiate multipart upload request, Amazon S3 associates that metadata with - the object. After a successful complete request, the parts no longer - exist. -Your complete multipart upload request must include the upload ID and a list of - part numbers and their corresponding ETag values. The Amazon S3 response includes an ETag that - uniquely identifies the combined object data. This ETag is not necessarily an MD5 hash of the - object data. -When you provide a full object checksum during a multipart upload, the AWS SDK passes the checksum to - Amazon S3, and S3 validates the object integrity server-side, comparing it to the received value. - Then, S3 stores the object if the values match. If the two values don’t match, Amazon S3 fails the - request with a BadDigest error. The checksum of your object is also stored in - object metadata that you'll later use to validate an object's data integrity. - For this example, assume that you're generating a multipart upload for a 100 GB file. - In this case, you would have the following API calls for the entire process. There would be - a total of 1,002 API calls. -A CreateMultipartUpload call to start the process. -1,000 individual UploadPart calls, each - uploading a part of 100 MB, for a total size of 100 GB. -A CompleteMultipartUpload call to complete the process. -You can list the parts of a specific multipart upload or all in-progress multipart uploads. The - list parts operation returns the parts information that you uploaded for a specific multipart upload. - For each list parts request, Amazon S3 returns the parts information for the specified multipart upload, up - to a maximum of 1,000 parts. If there are more than 1,000 parts in the multipart upload, you must send - a series of list part requests to retrieve all of the parts. Note that the returned list of - parts doesn't include parts that haven't finished uploading. Using the list - multipart uploads operation, you can obtain a list of multipart uploads that - are in progress. -An in-progress multipart upload is an upload that you have initiated, but have not yet completed or - stopped. Each request returns at most 1,000 multipart uploads. If there are more than 1,000 - multipart uploads in progress, you must send additional requests to retrieve the remaining - multipart uploads. Use the returned listing only for verification. -Do not use the result of - this listing when sending a complete multipart upload request. Instead, - maintain your own list of the part numbers that you specified when uploading parts and the - corresponding ETag values that Amazon S3 returns. -When you upload an object to Amazon S3, you can specify a checksum algorithm for Amazon S3 to use. - By default, the AWS SDK and S3 console use an algorithm for all object uploads, which you can - override. If you’re using an older SDK and your uploaded object doesn’t have a specified - checksum, Amazon S3 automatically uses the CRC-64NVME checksum algorithm. (This is - also the recommended option for efficient data integrity verification.) When using - CRC-64NVME, Amazon S3 calculates the checksum of the full object after the multipart - or single part upload is complete. The CRC-64NVME checksum algorithm is used to - calculate either a direct checksum of the entire object, or a checksum of the checksums, for - each individual part. -After you upload an object to S3 using multipart upload, Amazon S3 calculates the checksum valuefor each - part, or for the full object—and stores the values. You can use the S3 API or AWS SDK to - retrieve the checksum value in the following ways: -For individual parts, you can use GetObject or HeadObject. If you want to retrieve the checksum values for - individual parts of multipart uploads while they're still in process, you can use ListParts. -For the entire object, you can use PutObject. If you want - to perform a multipart upload with a full object checksum, use CreateMultipartUpload and CompleteMultipartUpload by specifying the full object checksum - type. To validate the checksum value of the entire object or to confirm which checksum - type is being used in the multipart upload, use ListParts. -If you're using a multipart upload with Checksums, the part numbers for each part upload - (in the multipart upload) must use consecutive part numbers. When using Checksums, - if you try to complete a multipart upload request with nonconsecutive part numbers, Amazon S3 generates an - HTTP 500 Internal Server error. - For more information about how checksums work with multipart upload objects, see Checking object integrity in Amazon S3. -For an end-to-end procedure that demonstrates how to upload an object using multipart upload with an - additional checksum, see Tutorial: Upload an object through multipart upload and - verify its data integrity. -In a distributed development environment, it is possible for your application to initiate - several updates on the same object at the same time. Your application might initiate several - multipart uploads using the same object key. For each of these uploads, your application can - then upload parts and send a complete upload request to Amazon S3 to create the object. When the - buckets have S3 Versioning enabled, completing a multipart upload always creates a new version. When you - initiate multiple multipart uploads that use the same object key in a versioning-enabled - bucket, the current version of the object is determined by which upload started most recently - (createdDate). -For example, you start a CreateMultipartUpload request for an object at 10:00 - AM. Then, you submit a second CreateMultipartUpload request for the same object - at 11:00 AM. Because the second request was submitted the most recently, the object uploaded - by the 11:00 AM request becomes the current version, even if the first upload is completed - after the second one. For buckets that don't have versioning enabled, it's possible that any - other request received between the time when the multipart upload is initiated and when it completes, the - other request might take precedence. -Another example of when a concurrent multipart upload request can take precedence is if another operation deletes a key after - you initiate a multipart upload with that key. Before you complete the operation, the complete multipart upload response - might indicate a successful object creation without you ever seeing the object. -You can check for the existence of an object in your bucket before creating it using a - conditional write on upload operations. This can prevent overwrites of existing data. Conditional writes - will validate that there is no existing object with the same key name already in your bucket - while uploading. -You can use conditional writes for PutObject or CompleteMultipartUpload requests. -For more information about conditional requests see, Add preconditions to S3 operations with conditional requests. -After you initiate a multipart upload, Amazon S3 retains all the parts until you either - complete or stop the upload. Throughout its lifetime, you are billed for all storage, - bandwidth, and requests for this multipart upload and its associated parts. -These parts are billed according to the storage class specified when the parts are - uploaded. However, you will not be billed for these parts if they're uploaded to - S3 Glacier Flexible Retrieval or S3 Glacier Deep Archive. In-progress multipart - parts - for a PUT request to the S3 Glacier Flexible Retrieval storage class are billed as - S3 Glacier Flexible Retrieval staging storage at S3 Standard storage rates until the upload - completes. In addition, both CreateMultipartUpload and UploadPart - are billed at S3 Standard rates. Only the CompleteMultipartUpload request is - billed at the S3 Glacier Flexible Retrieval rate. Similarly, in-progress multipart parts for a - PUT to the S3 Glacier Deep Archive storage class are billed as S3 Glacier Flexible Retrieval staging - storage at S3 Standard storage rates until the upload completes, with only the - CompleteMultipartUpload request charged at S3 Glacier Deep Archive rates. -If you stop the multipart upload, Amazon S3 deletes upload artifacts and all parts that you uploaded. You - will not be billed for those artifacts. There are no early delete charges for deleting - incomplete multipart uploads regardless of storage class specified. For more information about - pricing, see Amazon S3 pricing. -To minimize your storage costs, we recommend that you configure a lifecycle rule to - delete incomplete multipart uploads after a specified number of days by using the - AbortIncompleteMultipartUpload action. For more information about creating a - lifecycle rule to delete incomplete multipart uploads, see Configuring a bucket - lifecycle configuration to delete incomplete multipart uploads. -The following - sections in the Amazon Simple Storage Service API Reference describe the REST API for - multipart upload. -For a multipart upload walkthrough that uses AWS Lambda functions, see Uploading large objects to Amazon S3 using multipart upload and transfer acceleration. -Create Multipart - Upload -Upload Part -Upload Part (Copy) -Complete Multipart - Upload -Abort Multipart Upload -List Parts -List Multipart - Uploads -The following topics in the AWS Command Line Interface describe the operations for multipart upload. -Initiate Multipart - Upload -Upload Part -Upload Part - (Copy) -Complete Multipart - Upload -Abort Multipart - Upload -List Parts -List Multipart - Uploads - -You can use an AWS SDKs to upload an object in parts. For a list of AWS SDKs - supported by API action see: -Create Multipart - Upload -Upload Part -Upload Part (Copy) -Complete Multipart - Upload -Abort Multipart Upload -List Parts -List Multipart - Uploads -You must have the necessary permissions to use the multipart upload operations. You can use access - control lists (ACLs), the bucket policy, or the user policy to grant individuals permissions - to perform these operations. The following table lists the required permissions for various - multipart upload operations when using ACLs, a bucket policy, or a user policy. -Create Multipart Upload -You must be allowed to perform the s3:PutObject action on an object - to create a multipart upload request. -The bucket owner can allow other principals to perform the - s3:PutObject action. -Initiate Multipart Upload -You must be allowed to perform the s3:PutObject action on an object - to initiate a multipart upload. -The bucket owner can allow other principals to perform the - s3:PutObject action. -Container element that identifies who initiated the multipart upload. If the initiator - is an AWS account, this element provides the same information as the Owner - element. If the initiator is an IAM user, this element provides the user ARN and - display name. -You must be allowed to perform the s3:PutObject action on an - object to upload a part. -The bucket owner must allow the initiator to perform the - s3:PutObject action on an object in order for the initiator to upload - a part for that object. -You must be allowed to perform the s3:PutObject action on an - object to upload a part. Because you are uploading a part from an existing object, - you must be allowed s3:GetObject on the source object. -For the initiator to upload a part for an object, the owner of the bucket must - allow the initiator to perform the s3:PutObject action on the - object. -You must be allowed to perform the s3:PutObject action on an - object to complete a multipart upload. -The bucket owner must allow the initiator to perform the - s3:PutObject action on an object in order for the initiator to - complete a multipart upload for that object. -You must be allowed to perform the s3:AbortMultipartUpload - action to stop a multipart upload. -By default, the bucket owner and the initiator of the multipart upload are allowed to - perform this action as a part of IAM and S3 bucket polices. If the initiator is an - IAM user, that user's AWS account is also allowed to stop that multipart upload. With VPC - endpoint policies, the initiator of the multipart upload doesn't automatically gain the - permission to perform the s3:AbortMultipartUpload action. -In addition to these defaults, the bucket owner can allow other principals to - perform the s3:AbortMultipartUpload action on an object. The bucket - owner can deny any principal the ability to perform the - s3:AbortMultipartUpload action. -You must be allowed to perform the s3:ListMultipartUploadParts - action to list parts in a multipart upload. -By default, the bucket owner has permission to list parts for any multipart upload to the - bucket. The initiator of the multipart upload has the permission to list parts of the specific - multipart upload. If the multipart upload initiator is an IAM user, the AWS account controlling that - IAM user also has permission to list parts of that upload. - In addition to these defaults, the bucket owner can allow other principals to - perform the s3:ListMultipartUploadParts action on an object. The bucket - owner can also deny any principal the ability to perform the - s3:ListMultipartUploadParts action. -You must be allowed to perform the - s3:ListBucketMultipartUploads action on a bucket to list multipart - uploads in progress to that bucket. -In addition to the default, the bucket owner can allow other principals to - perform the s3:ListBucketMultipartUploads action on the - bucket. -To perform a multipart upload with encryption using an AWS Key Management Service (AWS KMS) KMS key, the - requester must have the following permissions: -The kms:Decrypt and kms:GenerateDataKey actions on - the key. -The kms:GenerateDataKey action for the CreateMultipartUpload API. -The kms:Decrypt action on the UploadPart and UploadPartCopy APIs. - These permissions are required because Amazon S3 must decrypt and read data from the - encrypted file parts before it completes the multipart upload. The kms:Decrypt - permission, and the server-side encryption with customer-provided encryption keys, are also required for you to obtain an object's - checksum value. If you don't have these required permissions when you use the CompleteMultipartUpload API, the object is created - without a checksum value. -If your IAM user or role is in the same AWS account as the KMS key, then - you must have these permissions on the key policy. If your IAM user or role - belongs to a different account than the KMS key, then you must have the - permissions on both the key policy and your IAM user or role. -When you use the CompleteMultipartUpload API, you must provide the SSE-C (server-side encryption with customer-provided encryption keys), or your object will be created without a checksum, and no checksum value is returned. -For information on the relationship between ACL permissions and permissions in access - policies, see Mapping of ACL permissions and access policy - permissions. For information about IAM users, - roles, and best practices, see IAM identities (users, user groups, and roles) in the IAM User Guide. -There are three Amazon S3 APIs that are used to perform the actual multipart upload: CreateMultipartUpload, UploadPart, and CompleteMultipartUpload. The following table indicates which - checksum headers and values must be provided for each of the APIs: -CRC-64NVME -x-amz-checsum-algorithm -Optional headers: -x-amz-checksum-CRC64nvme -Optional headers: -x-amz-checksum-algorithm -x-amz-crc64 -CRC-32 -CRC-32C -Required headers: -x-amz-checksum-algorithm -x-amz-checksum-type -Optional headers: -x-amz-checksum-crc64nvme -Optional headers: -x-amz-checksum-algorithm -x-amz-crc32 -x-amz-crc32c -CRC-32 -CRC-32C -SHA-1 -SHA-256 -Required headers: -x-amz-checksum-algorithm -Required headers: -x-amz-checksum-crc32 -x-amz-checksum-crc32c -x-amz-checksum-sha1 -x-amz-checksum-sha256 -Required headers: -All part-level checksums need to be included in the CompleteMultiPartUpload request. -Optional headers: -x-amz-crc32 -x-amz-crc32c -x-amz-sha1 -x-amz-sha256 - Javascript is disabled or is unavailable in your browser. -To use the Amazon Web Services Documentation, Javascript must be enabled. Please refer to your browser's Help pages for instructions. -Thanks for letting us know we're doing a good job! -If you've got a moment, please tell us what we did right so we can do more of it. - -Thanks for letting us know this page needs work. We're sorry we let you down. -If you've got a moment, please tell us how we can make the documentation better. - diff --git a/crawl/crawled_data/Using Storage Browser for S3 - Amazon Simple Storage Service.txt b/crawl/crawled_data/Using Storage Browser for S3 - Amazon Simple Storage Service.txt deleted file mode 100644 index 56228757..00000000 --- a/crawl/crawled_data/Using Storage Browser for S3 - Amazon Simple Storage Service.txt +++ /dev/null @@ -1,35 +0,0 @@ -Title: Using Storage Browser for S3 - Amazon Simple Storage Service - -In Storage Browser for S3, a location is an S3 bucket or prefix, that you - grant end users access to using S3 Access Grants, IAM policies, or your - own managed authorization service. After you've authorized your end users to access a specific - location, they can work with any data within that location. -The Storage Browser for S3 user interface has four main views: -Home page: The home page lists the S3 locations that - your users can access, as well as your permissions for each. This is the initial view for - users that shows the root level S3 resources that your end users have access to and the - permissions (READ/WRITE/READWRITE) for each S3 location. -Location details: This view allows users to browse - files and folders in S3, and upload or download files. (Note that in Storage Browser for S3, - objects are known as files, and prefixes and - buckets are known as folders.) -Location action: After a user chooses an action (such as - Upload) in Storage Browser, it opens up another view of the - file location. -Vertical ellipsis: The vertical ellipsis icon opens - the drop-down list of actions. -When using Storage Browser for S3, be aware of the following limitations: -Folders starting or ending with dots (.) aren’t supported. -S3 Access Grants with WRITE only permission isn't supported. -Storage Browser for S3 supports the PUT operation for files up to 160 GB in - size. -Storage Browser for S3 only supports the COPY operation for files smaller than 5 GB. If the file size - exceeds 5 GB, Storage Browser fails the request. - Javascript is disabled or is unavailable in your browser. -To use the Amazon Web Services Documentation, Javascript must be enabled. Please refer to your browser's Help pages for instructions. -Thanks for letting us know we're doing a good job! -If you've got a moment, please tell us what we did right so we can do more of it. - -Thanks for letting us know this page needs work. We're sorry we let you down. -If you've got a moment, please tell us how we can make the documentation better. - diff --git a/crawl/crawled_data/Virtual hosting of buckets - Amazon Simple Storage Service.txt b/crawl/crawled_data/Virtual hosting of buckets - Amazon Simple Storage Service.txt deleted file mode 100644 index 07469966..00000000 --- a/crawl/crawled_data/Virtual hosting of buckets - Amazon Simple Storage Service.txt +++ /dev/null @@ -1,229 +0,0 @@ -Title: Virtual hosting of buckets - Amazon Simple Storage Service - -Virtual hosting is the practice of serving multiple websites from a single web server. One - way to differentiate sites in your Amazon S3 REST API requests is by using the apparent hostname - of the Request-URI instead of just the path name part of the URI. An ordinary Amazon S3 REST - request specifies a bucket by using the first slash-delimited component of the Request-URI - path. Instead, you can use Amazon S3 virtual hosting to address a bucket in a REST API call by - using the HTTP Host header. In practice, Amazon S3 interprets Host as - meaning that most buckets are automatically accessible for limited types of requests at - https://bucket-name.s3.region-code.amazonaws.com. - For a complete list of Amazon S3 Regions and endpoints, see Amazon S3 endpoints and quotas in the Amazon Web Services General Reference. -Virtual hosting also has other benefits. By naming your bucket after your registered - domain name and by making that name a DNS alias for Amazon S3, you can completely customize the - URL of your Amazon S3 resources, for example, - http://my.bucket-name.com/. You can - also publish to the "root directory" of your bucket's virtual server. This ability can be - important because many existing applications search for files in this standard location. For - example, favicon.ico, robots.txt, and - crossdomain.xml are all expected to be found at the root. -When you're using virtual-hosted–style buckets with SSL, the SSL wildcard certificate - matches only buckets that do not contain dots (.). To work around this - limitation, use HTTP or write your own certificate-verification logic. For more - information, see Amazon S3 Path - Deprecation Plan on the AWS News - Blog. -Currently, Amazon S3 supports both virtual-hosted–style and path-style URL access in all AWS Regions. However, path-style - URLs will be discontinued in the future. For more information, see the following Important note. -In Amazon S3, path-style URLs use the following format: -For example, if you create a bucket named amzn-s3-demo-bucket1 in the US West (Oregon) Region, - and you want to access the puppy.jpg object in that bucket, you can use the - following path-style URL: -Update (September 23, 2020) – To make sure that customers have the time that they need to transition to virtual-hosted–style URLs, -we have decided to delay the deprecation of path-style URLs. For more information, -see Amazon S3 Path Deprecation Plan – The Rest of the Story in the AWS News Blog. -When hosting website content that will be accessed from a web browser, avoid using - path-style URLs, which might interfere with the browser same origin security model. - To host website content, we recommend that you use either S3 website endpoints or a - CloudFront distribution. For more information, see Website endpoints and Deploy a React-based single-page application to Amazon S3 and CloudFront in the - AWS Perspective Guidance Patterns. -In a virtual-hosted–style URI, the bucket name is part of the domain name in the - URL. -Amazon S3 virtual-hosted–style URLs use the following format: -In this example, amzn-s3-demo-bucket1 is the bucket name, US West (Oregon) is the Region, and puppy.png is the key name: -As long as your GET request does not use the SSL endpoint, you can - specify the bucket for the request by using the HTTP Host header. The - Host header in a REST request is interpreted as follows: -If the Host header is omitted or its value is - s3.region-code.amazonaws.com, the - bucket for the request will be the first slash-delimited component of the - Request-URI, and the key for the request will be the rest of the Request-URI. - This is the ordinary method, as illustrated by the first and second examples in - this section. Omitting the Host header is valid only for HTTP 1.0 - requests. -Otherwise, if the value of the Host header ends in - .s3.region-code.amazonaws.com, - the bucket name is the leading component of the Host header's value - up to .s3.region-code.amazonaws.com. The - key for the request is the Request-URI. This interpretation exposes buckets as - subdomains of - .s3.region-code.amazonaws.com, as - illustrated by the third and fourth examples in this section. -Otherwise, the bucket for the request is the lowercase value of the - Host header, and the key for the request is the Request-URI. - This interpretation is useful when you have registered the same DNS name as your - bucket name and have configured that name to be a canonical name (CNAME) alias - for Amazon S3. The procedure for registering domain names and configuring CNAME DNS - records is beyond the scope of this guide, but the result is illustrated by the - final example in this section. -This section provides example URLs and requests. -This example uses the following: -Bucket Name ‐ example.com -Region ‐ US East (N. Virginia) -Key Name ‐ homepage.html -The URL is as follows: -The request is as follows: -The request with HTTP 1.0 and omitting the Host header is as - follows: -For information about DNS-compatible names, see Limitations. For more information about - keys, see Keys. -This example uses the following: -Bucket name ‐ - amzn-s3-demo-bucket1 - -Region ‐ Europe (Ireland) -Key name ‐ - homepage.html -The URL is as follows: -The request is as follows: -To use this method, you must configure your DNS name as a CNAME alias for - bucket-name.s3.us-east-1.amazonaws.com. - For more information, see Customizing Amazon S3 URLs with CNAME - records. -This example uses the following: -Bucket Name ‐ example.com - -Key name ‐ - homepage.html -The URL is as follows: -The example is as follows: -Depending on your needs, you might not want - s3.region-code.amazonaws.com to appear on - your website or service. For example, if you're hosting website images on Amazon S3, you - might prefer http://images.example.com/ instead of - http://images.example.com.s3.us-east-1.amazonaws.com/. Any bucket - with a DNS-compatible name can be referenced as follows: - http://BucketName.s3.Region.amazonaws.com/[Filename], - for example, - http://images.example.com.s3.us-east-1.amazonaws.com/mydog.jpg. - By using CNAME, you can map images.example.com to an Amazon S3 - hostname so that the previous URL could become - http://images.example.com/mydog.jpg. -Your bucket name must be the same as the CNAME. For example, if you create a CNAME to - map images.example.com to - images.example.com.s3.us-east-1.amazonaws.com, both - http://images.example.com/filename and - http://images.example.com.s3.us-east-1.amazonaws.com/filename - will be the same. -The CNAME DNS record should alias your domain name to the appropriate virtual - hosted–style hostname. For example, if your bucket name and domain name are - images.example.com and your bucket is in the - US East (N. Virginia) Region, the CNAME record should alias to - images.example.com.s3.us-east-1.amazonaws.com. -Amazon S3 uses the hostname to determine the bucket name. So the CNAME and the bucket name - must be the same. For example, suppose that you have configured - www.example.com as a CNAME for - www.example.com.s3.us-east-1.amazonaws.com. When you access - http://www.example.com, Amazon S3 receives a request similar to - the following: -Amazon S3 sees only the original hostname www.example.com and is - unaware of the CNAME mapping used to resolve the request. -You can use any Amazon S3 endpoint in a CNAME alias. For example, - s3.ap-southeast-1.amazonaws.com can be used in CNAME aliases. - For more information about endpoints, see Request - Endpoints in the Amazon S3 API Reference. To create a static website by using a custom domain, see Tutorial: Configuring a static website using a - custom domain registered with Route 53. -When using custom URLs with CNAMEs, you will need to ensure a matching bucket - exists for any CNAME or alias record you configure. For example, if you create DNS - entries for www.example.com and - login.example.com to publish web content using S3, you - will need to create both buckets www.example.com and - login.example.com. -When a CNAME or alias records is configured pointing to an S3 endpoint without a - matching bucket, any AWS user can create that bucket and publish content under the - configured alias, even if ownership is not the same. -For the same reason, we recommend that you change or remove the corresponding - CNAME or alias when deleting a bucket. -Select a hostname that belongs to a domain that you control. -This example uses the images subdomain of the - example.com domain. -Create a bucket that matches the hostname. -In this example, the host and bucket names are - images.example.com. The bucket name must exactly match the hostname. -Create a CNAME DNS record that defines the hostname as an alias for the Amazon S3 - bucket. -For example: -images.example.com CNAME - images.example.com.s3.us-west-2.amazonaws.com -For request-routing reasons, the CNAME DNS record must be defined exactly - as shown in the preceding example. Otherwise, it might appear to operate - correctly, but it will eventually result in unpredictable behavior. -The procedure for configuring CNAME DNS records depends on your DNS server or - DNS provider. For specific information, see your server documentation or contact - your provider. - - SOAP support over HTTP is deprecated, but SOAP is still available over HTTPS. - New Amazon S3 features are not supported for SOAP. Instead of using SOAP, we recommend that you use - either the REST API or the AWS SDKs. - -The following sections cover various aspects of Amazon S3 backward compatibility that - relate to path-style and virtual-hosted–style URL requests. -Some Regions support legacy endpoints. You might see these endpoints in your - server access logs or AWS CloudTrail logs. For more information, review the following - information. For a complete list of Amazon S3 Regions and endpoints, see Amazon S3 endpoints and quotas in the Amazon Web Services General Reference. -Although you might see legacy endpoints in your logs, we recommend that you - always use the standard endpoint syntax to access your buckets. -Amazon S3 virtual-hosted–style URLs use the following format: -In Amazon S3, path-style URLs use the following format: -Some older Amazon S3 Regions support endpoints that contain a dash (-) - between s3 and the Region code (for example, - s3‐us-west-2), instead of a dot (for example, - s3.us-west-2). If your bucket is in one of these Regions, you - might see the following endpoint format in your server access logs or CloudTrail - logs: -In this example, the bucket name is amzn-s3-demo-bucket1 and the - Region is US West (Oregon): -For some Regions, you can use the legacy global endpoint to construct requests - that do not specify a Region-specific endpoint. The legacy global endpoint point - is as follows: -In your server access logs or CloudTrail logs, you might see requests that use the - legacy global endpoint. In this example, the bucket name is - amzn-s3-demo-bucket1 and the legacy global - endpoint is: -Requests made with the legacy global endpoint go to the US East (N. Virginia) - Region by default. Therefore, the legacy global endpoint is sometimes used - in place of the Regional endpoint for US East (N. Virginia). If you create a - bucket in US East (N. Virginia) and use the global endpoint, Amazon S3 routes your - request to this Region by default. -The legacy global endpoint is also used for virtual-hosted–style - requests in other supported Regions. If you create a bucket in a Region that - was launched before March 20, 2019, and use the legacy global endpoint, Amazon S3 - updates the DNS record to reroute the request to the correct location, which - might take time. In the meantime, the default rule applies, and your - virtual-hosted–style request goes to the US East (N. Virginia) Region. - Amazon S3 then redirects it with an HTTP 307 Temporary Redirect to the correct - Region. -For S3 buckets in Regions launched after March 20, 2019, the DNS server - doesn't route your request directly to the AWS Region where your bucket - resides. It returns an HTTP 400 Bad Request error instead. For more information, - see Making requests in the Amazon S3 API - Reference. -For the US East (N. Virginia) Region, you can use the legacy global endpoint - for path-style requests. -For all other Regions, the path-style syntax requires that you use the - Region-specific endpoint when attempting to access a bucket. If you try to - access a bucket with the legacy global endpoint or another endpoint that is - different than the one for the Region where the bucket resides, you receive an - HTTP response code 307 Temporary Redirect error and a message that indicates the - correct URI for your resource. For example, if you use - https://s3.amazonaws.com/bucket-name - for a bucket that was created in the US West (Oregon) Region, you will receive - an HTTP 307 Temporary Redirect error. - Javascript is disabled or is unavailable in your browser. -To use the Amazon Web Services Documentation, Javascript must be enabled. Please refer to your browser's Help pages for instructions. -Thanks for letting us know we're doing a good job! -If you've got a moment, please tell us what we did right so we can do more of it. - -Thanks for letting us know this page needs work. We're sorry we let you down. -If you've got a moment, please tell us how we can make the documentation better. - diff --git a/crawl/crawled_data/What is Amazon EC2? - Amazon Elastic Compute Cloud.txt b/crawl/crawled_data/What is Amazon EC2? - Amazon Elastic Compute Cloud.txt deleted file mode 100644 index d0e78fd3..00000000 --- a/crawl/crawled_data/What is Amazon EC2? - Amazon Elastic Compute Cloud.txt +++ /dev/null @@ -1,151 +0,0 @@ -Title: What is Amazon EC2? - Amazon Elastic Compute Cloud - -Amazon Elastic Compute Cloud (Amazon EC2) provides on-demand, scalable computing capacity in the Amazon Web - Services (AWS) Cloud. Using Amazon EC2 reduces hardware costs so you can develop and deploy - applications faster. You can use Amazon EC2 to launch as many or as few virtual servers as you - need, configure security and networking, and manage storage. You can add capacity (scale up) - to handle compute-heavy tasks, such as monthly or yearly processes, or spikes in website - traffic. When usage decreases, you can reduce capacity (scale down) again. -An EC2 instance is a virtual server in the AWS Cloud. When you launch an EC2 instance, - the instance type that you specify determines the hardware available to your instance. - Each instance type offers a different balance of compute, memory, network, and storage - resources. For more information, see the Amazon EC2 Instance Types Guide. -Amazon EC2 provides the following high-level features: -Virtual servers. -Preconfigured templates for your instances that package the components you - need for your server (including the operating system and additional - software). -Various configurations of CPU, memory, storage, networking capacity, and - graphics hardware for your instances. -Persistent storage volumes for your data using Amazon Elastic Block Store (Amazon EBS). -Storage volumes for temporary data that is deleted when you stop, - hibernate, or terminate your instance. -Secure login information for your instances. AWS stores the public key - and you store the private key in a secure place. -A virtual firewall that allows you to specify the protocols, ports, and - source IP ranges that can reach your instances, and the destination IP - ranges to which your instances can connect. -Amazon EC2 supports the processing, storage, and transmission -of credit card data by a merchant or service provider, and has been -validated as being compliant with Payment Card Industry (PCI) Data Security Standard (DSS). -For more information about PCI DSS, including how to request a copy of the AWS PCI Compliance Package, -see PCI DSS Level 1. - -You can use other AWS services with the instances that you deploy using Amazon EC2. -Helps ensure you have the correct number of Amazon EC2 instances available to - handle the load for your application. -Automate backing up your Amazon EC2 instances and the Amazon EBS volumes attached to - them. -Monitor your instances and Amazon EBS volumes. -Automatically distribute incoming application traffic across multiple - instances. -Detect potentially unauthorized or malicious use of your EC2 instances. -Automate the creation, management, and deployment of customized, secure, and - up-to-date server images. -Size, configure, and deploy AWS resources for third-party applications - without having to manually identify and provision individual AWS - resources. -Perform operations at scale on EC2 instances with this secure end-to-end - management solution. -You can launch instances using another AWS compute service instead of using Amazon EC2. -Build websites or web applications using Amazon Lightsail, a cloud platform - that provides the resources that you need to deploy your project quickly, for - a low, predictable monthly price. To compare Amazon EC2 and Lightsail, see - Amazon Lightsail or Amazon EC2. -Deploy, manage, and scale containerized applications on a cluster of EC2 - instances. For more information, see Choosing an AWS container service. -Run your Kubernetes applications on AWS. For more information, see - Choosing an AWS container service. -You can create and manage your Amazon EC2 instances using the following interfaces: -A simple web interface to create and manage Amazon EC2 instances and resources. - If you've signed up for an AWS account, you can access the Amazon EC2 console - by signing into the AWS Management Console and selecting EC2 from - the console home page. -Enables you to interact with AWS services using commands in your command-line shell. It - is supported on Windows, Mac, and Linux. For more information about the - AWS CLI , see AWS Command Line Interface User Guide. You can find the Amazon EC2 commands in the AWS CLI Command Reference. -Amazon EC2 supports creating resources using AWS CloudFormation. You create a template, in JSON or YAML - format, that describes your AWS resources, and AWS CloudFormation provisions and - configures those resources for you. You can reuse your CloudFormation - templates to provision the same resources multiple times, whether in the - same Region and account or in multiple Regions and accounts. For more - information about supported resource types and properties for Amazon EC2, see - EC2 resource type - reference in the AWS CloudFormation User Guide. -If you prefer to build applications using language-specific APIs instead - of submitting a request over HTTP or HTTPS, AWS provides libraries, sample - code, tutorials, and other resources for software developers. These - libraries provide basic functions that automate tasks such as - cryptographically signing your requests, retrying requests, and handling - error responses, making it easier for you to get started. For more - information, see - Tools to Build - on AWS. -A set of PowerShell modules that are built on the functionality exposed by - the AWS SDK for .NET. The Tools for PowerShell enable you to script operations on your AWS - resources from the PowerShell command line. To get started, see the - AWS Tools for Windows PowerShell User Guide. You can find the cmdlets for Amazon EC2, in the AWS Tools for PowerShell Cmdlet Reference. -Amazon EC2 provides a Query API. These requests are HTTP or HTTPS requests that - use the HTTP verbs GET or POST and a Query parameter named - Action. For more information about the API actions for - Amazon EC2, see Actions in the - Amazon EC2 API Reference. -Amazon EC2 provides the following pricing options: -You can get started with Amazon EC2 for free. To explore the Free Tier options, - see AWS Free Tier. -Pay for the instances that you use by the second, with a minimum of 60 - seconds, with no long-term commitments or upfront payments. -You can reduce your Amazon EC2 costs by making a commitment to a consistent - amount of usage, in USD per hour, for a term of 1 or 3 years. -You can reduce your Amazon EC2 costs by making a commitment to a specific - instance configuration, including instance type and Region, for a term of 1 - or 3 years. -Request unused EC2 instances, which can reduce your Amazon EC2 costs - significantly. -Reduce costs by using a physical EC2 server that is fully dedicated for - your use, either On-Demand or as part of a Savings Plan. You can use your - existing server-bound software licenses and get help meeting compliance - requirements. -Reserve compute capacity for your EC2 instances in a specific Availability - Zone for any duration of time. -Removes the cost of unused minutes and seconds from your bill. -For a complete list of charges and prices for Amazon EC2 and more information about the purchase - models, see Amazon EC2 pricing. -To create estimates for your AWS use cases, use the AWS Pricing Calculator. -To estimate the cost of transforming Microsoft - workloads to a modern architecture that uses open source and - cloud-native services deployed on AWS, use the AWS - Modernization Calculator for Microsoft Workloads. -To see your bill, go to the Billing and Cost Management - Dashboard in the AWS Billing and Cost Management - console. Your bill contains links to usage reports that provide details - about your bill. To learn more about AWS account billing, see AWS Billing and Cost Management User - Guide. -If you have questions concerning AWS billing, accounts, and events, contact AWS Support. -To calculate the cost of a sample provisioned - environment, see Cloud Economics - Center. When calculating the cost of a provisioned - environment, remember to include incidental costs such as snapshot storage for EBS - volumes. -You can optimize the cost, security, and performance of your AWS environment - using AWS Trusted Advisor. -You can use AWS Cost Explorer to analyze the cost and usage of your EC2 instances. You can view - data up to the last 13 months, and forecast how much you are likely to spend for the next - 12 months. For more information, see - Analyzing your costs with - AWS Cost Explorer in the AWS Cost Management User Guide. -Amazon EC2 features -AWS re:Post -AWS Skill Builder -AWS Support -Hands-on Tutorials -Web Hosting -Windows on AWS - Javascript is disabled or is unavailable in your browser. -To use the Amazon Web Services Documentation, Javascript must be enabled. Please refer to your browser's Help pages for instructions. -Thanks for letting us know we're doing a good job! -If you've got a moment, please tell us what we did right so we can do more of it. - -Thanks for letting us know this page needs work. We're sorry we let you down. -If you've got a moment, please tell us how we can make the documentation better. - diff --git a/crawl/crawled_data/What is Amazon S3? - Amazon Simple Storage Service.txt b/crawl/crawled_data/What is Amazon S3? - Amazon Simple Storage Service.txt deleted file mode 100644 index 5229223e..00000000 --- a/crawl/crawled_data/What is Amazon S3? - Amazon Simple Storage Service.txt +++ /dev/null @@ -1,431 +0,0 @@ -Title: What is Amazon S3? - Amazon Simple Storage Service - -Amazon Simple Storage Service (Amazon S3) is an object storage service that offers industry-leading scalability, - data availability, security, and performance. Customers of all sizes and industries can use - Amazon S3 to store and protect any amount of data for a range of use cases, such as data lakes, - websites, mobile applications, backup and restore, archive, enterprise applications, IoT - devices, and big data analytics. Amazon S3 provides management features so that you can optimize, - organize, and configure access to your data to meet your specific business, organizational, - and compliance requirements. -For more information about using the Amazon S3 Express One Zone storage class with directory buckets, see S3 Express One Zone and Working with directory buckets. -Amazon S3 offers a range of storage classes designed for different use cases. For - example, you can store mission-critical production data in S3 Standard or S3 Express One Zone for frequent - access, save costs by storing infrequently accessed data in S3 Standard-IA or - S3 One Zone-IA, and archive data at the lowest costs in S3 Glacier Instant Retrieval, - S3 Glacier Flexible Retrieval, and S3 Glacier Deep Archive. -Amazon S3 Express One Zone is a high-performance, single-zone Amazon S3 storage class that is purpose-built - to deliver consistent, single-digit millisecond data access for your most - latency-sensitive applications. S3 Express One Zone is the lowest latency cloud object - storage class available today, with data access - speeds - up to 10x faster and with request costs - 50 - percent lower than S3 Standard. S3 Express One Zone is the first S3 storage class where you can select a single Availability Zone with -the option to co-locate your object storage with your compute resources, which provides the highest possible access speed. - Additionally, to further increase access speed and support hundreds of thousands of - requests per second, data is stored in a new bucket type: an - Amazon S3 directory bucket. For more information, see S3 Express One Zone and Working with directory buckets. -You can store data with changing or unknown access patterns in - S3 Intelligent-Tiering, which optimizes storage costs by automatically moving your - data between four access tiers when your access patterns change. These four access - tiers include two low-latency access tiers optimized for frequent and infrequent - access, and two opt-in archive access tiers designed for asynchronous access for - rarely accessed data. -For more information, see Understanding and managing Amazon S3 storage classes. -Amazon S3 has storage management features that you can use to manage costs, meet - regulatory requirements, reduce latency, and save multiple distinct copies of your - data for compliance requirements. -S3 Lifecycle – Configure a lifecycle configuration to manage - your objects and store them cost effectively throughout their lifecycle. You - can transition objects to other S3 storage classes or expire objects that - reach the end of their lifetimes. -S3 Object Lock – Prevent Amazon S3 objects from being - deleted or overwritten for a fixed amount of time or indefinitely. You can - use Object Lock to help meet regulatory requirements that require write-once-read-many -(WORM) storage or to simply add another - layer of protection against object changes and deletions. -S3 Replication - – Replicate objects and their respective metadata and object tags to - one or more destination buckets in the same or different AWS Regions for - reduced latency, compliance, security, and other use cases. -S3 Batch Operations – Manage billions of objects at scale - with a single S3 API request or a few clicks in the Amazon S3 console. You can - use Batch Operations to perform operations such as Copy, Invoke AWS Lambda - function, and Restore on - millions or billions of objects. -Amazon S3 provides features for auditing and managing access to your buckets and - objects. By default, S3 buckets and the objects in them are private. You have access - only to the S3 resources that you create. To grant granular resource permissions - that support your specific use case or to audit the permissions of your Amazon S3 - resources, you can use the following features. -S3 Block Public Access – Block public access to S3 - buckets and objects. By default, Block Public Access settings are turned on - at the bucket level. We recommend that you keep all Block Public Access - settings enabled unless you know that you need to turn off one or more of - them for your specific use case. For more information, see Configuring block public access - settings for your S3 buckets. -AWS Identity and Access Management (IAM) – IAM is a web service that helps - you securely control access to AWS resources, including your Amazon S3 - resources. With IAM, you can centrally manage permissions that control - which AWS resources users can access. You use IAM to control who is - authenticated (signed in) and authorized (has permissions) to use - resources. -Bucket - policies – Use IAM-based policy language to configure - resource-based permissions for your S3 buckets and the objects in - them. - -Amazon S3 access points - – Configure named network endpoints with dedicated access policies to - manage data access at scale for shared datasets in Amazon S3. -Access control - lists (ACLs) – Grant read and write permissions for - individual buckets and objects to authorized users. As a general rule, we - recommend using S3 resource-based policies (bucket policies and access point - policies) or IAM user policies for access control instead of ACLs. - Policies are a simplified and more flexible access control option. With - bucket policies and access point policies, you can define rules that apply - broadly across all requests to your Amazon S3 resources. For more information - about the specific cases when you'd use ACLs instead of resource-based - policies or IAM user policies, see Managing access with ACLs. -S3 Object Ownership – Take ownership of every object - in your bucket, simplifying access management for data stored in Amazon S3. - S3 Object Ownership is an Amazon S3 bucket-level setting that you can use to - disable or enable ACLs. By default, ACLs are disabled. With ACLs disabled, - the bucket owner owns all the objects in the bucket and manages access to - data exclusively by using access-management policies. -IAM Access Analyzer for S3 - – Evaluate and monitor your S3 bucket access policies, ensuring that - the policies provide only the intended access to your S3 resources. -To transform data and trigger workflows to automate a variety of other processing - activities at scale, you can use the following features. -S3 Object Lambda - – Add your own code to S3 GET, HEAD, and LIST requests to modify and process data as - it is returned to an application. Filter rows, dynamically resize images, - redact confidential data, and much more. -Event - notifications – Trigger workflows that use Amazon Simple Notification Service - (Amazon SNS), Amazon Simple Queue Service (Amazon SQS), and AWS Lambda when a change is made to your S3 - resources. -Amazon S3 provides logging and monitoring tools that you can use to monitor and control - how your Amazon S3 resources are being used. For more information, see Monitoring - tools. -Amazon CloudWatch - metrics for Amazon S3 – Track the operational health of your - S3 resources and configure billing alerts when estimated charges reach a - user-defined threshold. -AWS CloudTrail - – Record actions taken by a user, a role, or an AWS service in - Amazon S3. CloudTrail logs provide you with detailed API tracking for S3 bucket-level - and object-level operations. -Server access - logging – Get detailed records for the requests that are - made to a bucket. You can use server access logs for many use cases, such as - conducting security and access audits, learning about your customer base, - and understanding your Amazon S3 bill. -AWS Trusted - Advisor – Evaluate your account by using AWS best - practice checks to identify ways to optimize your AWS infrastructure, - improve security and performance, reduce costs, and monitor service quotas. - You can then follow the recommendations to optimize your services and - resources. -Amazon S3 offers features to help you gain visibility into your storage usage, which - empowers you to better understand, analyze, and optimize your storage at - scale. -Amazon S3 Storage Lens - – Understand, analyze, and optimize your storage. S3 Storage Lens provides - 60+ usage and activity metrics and interactive dashboards to aggregate data - for your entire organization, specific accounts, AWS Regions, buckets, or - prefixes. -Storage - Class Analysis – Analyze storage access patterns to - decide when it's time to move data to a more cost-effective storage class. - -S3 Inventory with - Inventory reports – Audit and report on objects and their - corresponding metadata and configure other Amazon S3 features to take action in - Inventory reports. For example, you can report on the replication and - encryption status of your objects. For a list of all the metadata available - for each object in Inventory reports, see Amazon S3 Inventory list. -Amazon S3 provides strong read-after-write consistency for PUT and DELETE requests of - objects in your Amazon S3 bucket in all AWS Regions. This behavior applies to both - writes of new objects as well as PUT requests that overwrite existing objects and - DELETE requests. In addition, read operations on Amazon S3 Select, Amazon S3 access control - lists (ACLs), Amazon S3 Object Tags, and object metadata (for example, the HEAD object) - are strongly consistent. For more information, see Amazon S3 data consistency model. -Amazon S3 is an object storage service that stores data as objects within buckets. An - object is a file and any metadata that describes - the file. A bucket is a container for objects. -To store your data in Amazon S3, you first create a bucket and specify a bucket name and - AWS Region. Then, you upload your data to that bucket as objects in Amazon S3. Each object - has a key (or key - name), which is the unique identifier for the object within the - bucket. -S3 provides features that you can configure to support your specific use case. For - example, you can use S3 Versioning to keep multiple versions of an object in the same - bucket, which allows you to restore objects that are accidentally deleted or - overwritten. -Buckets and the objects in them are private and can be accessed only if you explicitly - grant access permissions. You can use bucket policies, AWS Identity and Access Management (IAM) policies, - access control lists (ACLs), and S3 Access Points to manage access. -A general purpose bucket is a container for objects stored in Amazon S3. You can store any number of - objects in a bucket and all accounts have a default bucket quota of 10,000 general purpose buckets. To see your bucket utilization, bucket quota, or request an - increase to this quota, visit the Service Quotas - console. -Every object is contained in a bucket. For example, if the object named - photos/puppy.jpg is stored in the - amzn-s3-demo-bucket bucket in the US West (Oregon) - Region, then it is addressable by using the URL - https://amzn-s3-demo-bucket.s3.us-west-2.amazonaws.com/photos/puppy.jpg. - For more information, see Accessing a - Bucket. -When you create a bucket, you enter a bucket name and choose the AWS Region - where the bucket will reside. After you create a bucket, you cannot change the name - of the bucket or its Region. Bucket names must follow the bucket naming rules. You can also configure a bucket to use S3 Versioning or other storage management - features. - Buckets also: -Organize the Amazon S3 namespace at the highest level. -Identify the account responsible for storage and data transfer - charges. -Provide access control options, such as bucket policies, access control - lists (ACLs), and S3 Access Points, that you can use to manage access to - your Amazon S3 resources. -Serve as the unit of aggregation for usage reporting. - For more information about buckets, see Buckets overview. -Objects are the fundamental entities stored in Amazon S3. Objects consist of object - data and metadata. The metadata is a set of name-value pairs that describe the - object. These pairs include some default metadata, such as the date last modified, - and standard HTTP metadata, such as Content-Type. You can also specify - custom metadata at the time that the object is stored. -An object is uniquely identified within a bucket by a key (name) and a version ID (if - S3 Versioning is enabled on the bucket). For more information about objects, see - Amazon S3 objects overview. -An object key (or key - name) is the unique identifier for an object within a bucket. Every - object in a bucket has exactly one key. The combination of a bucket, object key, and - optionally, version ID (if S3 Versioning is enabled for the bucket) uniquely identify - each object. So you can think of Amazon S3 as a basic data map between "bucket + key + - version" and the object itself. -Every object in Amazon S3 can be uniquely addressed through the combination of the web service - endpoint, bucket name, key, and optionally, a version. For example, in the URL - https://amzn-s3-demo-bucket.s3.us-west-2.amazonaws.com/photos/puppy.jpg, - amzn-s3-demo-bucket is the name of the bucket - and photos/puppy.jpg is the key. -For more information about object keys, see Naming Amazon S3 objects. -You can use S3 Versioning to keep multiple variants of an object in the same - bucket. With S3 Versioning, you can preserve, retrieve, and restore every version of - every object stored in your buckets. You can easily recover from both unintended - user actions and application failures. -For more information, see Retaining multiple versions of objects with S3 Versioning. -When you enable S3 Versioning in a bucket, Amazon S3 generates a unique version ID for - each object added to the bucket. Objects that already existed in the bucket at the - time that you enable versioning have a version ID of null. If you - modify these (or any other) objects with other operations, such as CopyObject and PutObject, the new objects - get a unique version ID. -For more information, see Retaining multiple versions of objects with S3 Versioning. -A bucket policy is a resource-based AWS Identity and Access Management (IAM) policy that you can use to - grant access permissions to your bucket and the objects in it. Only the bucket owner - can associate a policy with a bucket. The permissions attached to the bucket apply - to all of the objects in the bucket that are owned by the bucket owner. Bucket - policies are limited to 20 KB in size. -Bucket policies use JSON-based access policy language that is standard across - AWS. You can use bucket policies to add or deny permissions for the objects in a - bucket. Bucket policies allow or deny requests based on the elements in the policy, - including the requester, S3 actions, resources, and aspects or conditions of the - request (for example, the IP address used to make the request). For example, you can - create a bucket policy that grants cross-account permissions to upload objects to an - S3 bucket while ensuring that the bucket owner has full control of the uploaded - objects. For more information, see Examples of Amazon S3 bucket policies. -In your bucket policy, you can use wildcard characters on Amazon Resource Names - (ARNs) and other values to grant permissions to a subset of objects. For example, - you can control access to groups of objects that begin with a common prefix or end with a given extension, such as - .html. -Amazon S3 Access Points are named network endpoints with dedicated access policies that - describe how data can be accessed using that endpoint. Access Points are attached to - buckets that you can use to perform S3 object operations, such as GetObject and - PutObject. Access Points simplify managing data access at scale for shared datasets - in Amazon S3. -Each access point has its own access point policy. You can configure Block Public Access settings - for each access point. To restrict Amazon S3 data access to a private network, you can - also configure any access point to accept requests only from a virtual private cloud - (VPC). -For more information, see Managing access to shared datasets with access points. -You can use ACLs to grant read and write permissions to authorized users for - individual buckets and objects. Each bucket and object has an ACL attached to it as - a subresource. The ACL defines which AWS accounts or groups are granted access and - the type of access. ACLs are an access control mechanism that predates IAM. For - more information about ACLs, see Access control list (ACL) overview. -S3 Object Ownership is an Amazon S3 bucket-level setting that you can use to both control ownership of the objects that are - uploaded to your bucket and to disable or enable ACLs. By default, Object Ownership is set to the Bucket owner enforced setting, - and all ACLs are disabled. When ACLs are disabled, the bucket owner owns all the objects in the bucket and manages access to them - exclusively by using access-management policies. - A majority of modern use cases in Amazon S3 no longer require the use of ACLs. We recommend that you keep ACLs disabled, except - in unusual circumstances where you need to control access for each object individually. With ACLs disabled, you can use policies - to control access to all objects in your bucket, regardless of who uploaded the objects to your bucket. - For more information, see Controlling ownership of objects and disabling ACLs - for your bucket. -You can choose the geographical AWS Region where Amazon S3 stores the buckets that - you create. You might choose a Region to optimize latency, minimize costs, or - address regulatory requirements. Objects stored in an AWS Region never leave the - Region unless you explicitly transfer or replicate them to another Region. For example, objects stored in the Europe (Ireland) Region - never leave it. - -You can access Amazon S3 and its features only in the AWS Regions that are - enabled for your account. For more information about enabling a Region to create - and manage AWS resources, see Managing AWS Regions in - the AWS General Reference. -For a list of Amazon S3 Regions and endpoints, see Regions and endpoints in the - AWS General Reference. -Amazon S3 provides strong read-after-write consistency for PUT and DELETE requests of - objects in your Amazon S3 bucket in all AWS Regions. This behavior applies to both writes - to new objects as well as PUT requests that overwrite existing objects and DELETE - requests. In addition, read operations on Amazon S3 Select, Amazon S3 access controls lists - (ACLs), Amazon S3 Object Tags, and object metadata (for example, the HEAD object) are - strongly consistent. -Updates to a single key are atomic. For example, if you make a PUT request to an - existing key from one thread and perform a GET request on the same key from a second - thread concurrently, you will get either the old data or the new data, but never partial - or corrupt data. -Amazon S3 achieves high availability by replicating data across multiple servers within - AWS data centers. If a PUT request is successful, your data is safely stored. Any read - (GET or LIST request) that is initiated following the receipt of a successful PUT - response will return the data written by the PUT request. Here are examples of this - behavior: -A process writes a new object to Amazon S3 and immediately lists keys within its - bucket. The new object appears in the list. -A process replaces an existing object and immediately tries to read it. Amazon S3 - returns the new data. -A process deletes an existing object and immediately tries to read it. Amazon S3 - does not return any data because the object has been deleted. -A process deletes an existing object and immediately lists keys within its - bucket. The object does not appear in the listing. -Amazon S3 does not support object locking for concurrent writers. If two PUT - requests are simultaneously made to the same key, the request with the - latest timestamp wins. If this is an issue, you must build an object-locking - mechanism into your application. -Updates are key-based. There is no way to make atomic updates across keys. - For example, you cannot make the update of one key dependent on the update - of another key unless you design this functionality into your - application. -Bucket configurations have an eventual consistency model. Specifically, this means - that: -If you delete a bucket and immediately list all buckets, the deleted bucket - might still appear in the list. -If you enable versioning on a bucket for the first time, it might take a short - amount of time for the change to be fully propagated. We recommend that you wait - for 15 minutes after enabling versioning before issuing write operations (PUT or - DELETE requests) on objects in the bucket. -This section provides examples of behavior to be expected from Amazon S3 when multiple - clients are writing to the same items. -In this example, both W1 (write 1) and W2 (write 2) finish before the start of R1 - (read 1) and R2 (read 2). Because S3 is strongly consistent, R1 and R2 both return - color = ruby. -In the next example, W2 does not finish before the start of R1. Therefore, R1 - might return color = ruby or color = garnet. However, - because W1 and W2 finish before the start of R2, R2 returns color = - garnet. -In the last example, W2 begins before W1 has received an acknowledgment. - Therefore, these writes are considered concurrent. Amazon S3 internally uses - last-writer-wins semantics to determine which write takes precedence. However, the - order in which Amazon S3 receives the requests and the order in which applications - receive acknowledgments cannot be predicted because of various factors, such as - network latency. For example, W2 might be initiated by an Amazon EC2 instance in the same - Region, while W1 might be initiated by a host that is farther away. The best way to - determine the final value is to perform a read after both writes have been - acknowledged. -After you load your data into Amazon S3, you can use it with other AWS services. The - following are the services that you might use most frequently: -Amazon Elastic Compute Cloud - (Amazon EC2) – Provides secure and scalable computing - capacity in the AWS Cloud. Using Amazon EC2 eliminates your need to invest in - hardware upfront, so you can develop and deploy applications faster. You can - use Amazon EC2 to launch as many or as few virtual servers as you need, configure - security and networking, and manage storage. -Amazon EMR – Helps businesses, researchers, data - analysts, and developers easily and cost-effectively process vast amounts of - data. Amazon EMR uses a hosted Hadoop framework running on the web-scale - infrastructure of Amazon EC2 and Amazon S3. -AWS Snow - Family – Helps customers that need to run - operations in austere, non-data center environments, and in locations where - there's a lack of consistent network connectivity. You can use AWS Snow Family - devices to locally and cost-effectively access the storage and compute power of - the AWS Cloud in places where an internet connection might not be an option. - -AWS Transfer Family – Provides fully managed support for - file transfers directly into and out of Amazon S3 or Amazon Elastic File System (Amazon EFS) using Secure - Shell (SSH) File Transfer Protocol (SFTP), File Transfer Protocol over SSL - (FTPS), and File Transfer Protocol (FTP). -You can work with Amazon S3 in any of the following ways: -The console is a web-based user interface for managing Amazon S3 and AWS resources. - If you've signed up for an AWS account, you can access the Amazon S3 console by signing - into the AWS Management Console and choosing S3 from the AWS Management Console home - page. -You can use the AWS command line tools to issue commands or build scripts at - your system's command line to perform AWS (including S3) tasks. -The AWS Command Line Interface (AWS CLI) provides commands - for a broad set of AWS services. The AWS CLI is supported on Windows, macOS, and - Linux. To get started, see the AWS Command Line Interface User Guide. For more information about the commands for - Amazon S3, see s3api and s3control in the AWS CLI Command Reference. -AWS provides SDKs (software development kits) that consist of libraries and sample code - for various programming languages and platforms (Java, Python, Ruby, .NET, iOS, - Android, and so on). The AWS SDKs provide a convenient way to create programmatic - access to S3 and AWS. Amazon S3 is a REST service. You can send requests to Amazon S3 using - the AWS SDK libraries, which wrap the underlying Amazon S3 REST API and simplify your - programming tasks. For example, the SDKs take care of tasks such as calculating - signatures, cryptographically signing requests, managing errors, and retrying - requests automatically. For information about the AWS SDKs, including how to - download and install them, see Tools for - AWS. -Every interaction with Amazon S3 is either authenticated or anonymous. If you are using - the AWS SDKs, the libraries compute the signature for authentication from the keys - that you provide. For more information about how to make requests to Amazon S3, see Making requests - . -The architecture of Amazon S3 is designed to be programming language-neutral, using - AWS-supported interfaces to store and retrieve objects. You can access S3 and - AWS programmatically by using the Amazon S3 REST API. The REST API is an HTTP interface - to Amazon S3. With the REST API, you use standard HTTP requests to create, fetch, and - delete buckets and objects. -To use the REST API, you can use any toolkit that supports HTTP. You can even use - a browser to fetch objects, as long as they are anonymously readable. -The REST API uses standard HTTP headers and status codes, so that standard - browsers and toolkits work as expected. In some areas, we have added functionality - to HTTP (for example, we added headers to support access control). In these cases, - we have done our best to add the new functionality in a way that matches the style - of standard HTTP usage. -If you make direct REST API calls in your application, you must write the code to - compute the signature and add it to the request. For more information about how to - make requests to Amazon S3, see Making requests - in the Amazon S3 API Reference. -SOAP API support over HTTP is deprecated, but it is still available over - HTTPS. Newer Amazon S3 features are not supported for SOAP. We recommend that you use - either the REST API or the AWS SDKs. -Pricing for Amazon S3 is designed so that you don't have to plan for the storage - requirements of your application. Most storage providers require you to purchase a - predetermined amount of storage and network transfer capacity. In this scenario, if you - exceed that capacity, your service is shut off or you are charged high overage fees. If - you do not exceed that capacity, you pay as though you used it all. -Amazon S3 charges you only for what you actually use, with no hidden fees and no overage - charges. This model gives you a variable-cost service that can grow with your business - while giving you the cost advantages of the AWS infrastructure. For more information, - see Amazon S3 Pricing. -When you sign up for AWS, your AWS account is automatically signed up for all - services in AWS, including Amazon S3. However, you are charged only for the services that - you use. If you are a new Amazon S3 customer, you can get started with Amazon S3 for free. For - more information, see AWS free tier. -To see your bill, go to the Billing and Cost Management Dashboard in the AWS Billing and Cost Management console. To learn more about AWS account billing, see the AWS Billing User Guide. If you have - questions concerning AWS billing and AWS accounts, contact AWS Support. -Amazon S3 supports the processing, storage, and transmission -of credit card data by a merchant or service provider, and has been -validated as being compliant with Payment Card Industry (PCI) Data Security Standard (DSS). -For more information about PCI DSS, including how to request a copy of the AWS PCI Compliance Package, -see PCI DSS Level 1. - - Javascript is disabled or is unavailable in your browser. -To use the Amazon Web Services Documentation, Javascript must be enabled. Please refer to your browser's Help pages for instructions. -Thanks for letting us know we're doing a good job! -If you've got a moment, please tell us what we did right so we can do more of it. - -Thanks for letting us know this page needs work. We're sorry we let you down. -If you've got a moment, please tell us how we can make the documentation better. - diff --git a/crawl/main.py b/crawl/main.py deleted file mode 100644 index ce378269..00000000 --- a/crawl/main.py +++ /dev/null @@ -1,42 +0,0 @@ -import requests -from bs4 import BeautifulSoup -import os - -# List of URLs to crawl -urls = [ - -] - -# Directory to save the files -save_dir = "crawled_data" -os.makedirs(save_dir, exist_ok=True) - -def fetch_and_save(url): - try: - response = requests.get(url) - response.raise_for_status() # Check if the request was successful - - # Parse the HTML content - soup = BeautifulSoup(response.text, 'html.parser') - - # For demonstration, we are fetching the page title and all paragraphs - title = soup.title.string if soup.title else "no_title" - paragraphs = soup.find_all('p') - - # Prepare the file name - file_name = os.path.join(save_dir, f"{title}.txt") - - # Write the content to the file - with open(file_name, 'w', encoding='utf-8') as file: - file.write(f"Title: {title}\n\n") - for para in paragraphs: - file.write(para.get_text() + "\n") - - print(f"Saved content from {url} to {file_name}") - - except requests.RequestException as e: - print(f"Failed to fetch {url}: {e}") - -# Fetch and save data from each URL -for url in urls: - fetch_and_save(url) From f2f419030ebfa0a0674523b0b6f0b0bb30d53721 Mon Sep 17 00:00:00 2001 From: Abolfazl Andalib <79583121+abolfazl8131@users.noreply.github.com> Date: Mon, 16 Dec 2024 20:14:45 +0330 Subject: [PATCH 22/22] Delete app/media/MyAnsible/group_vars/all --- app/media/MyAnsible/group_vars/all | 39 ------------------------------ 1 file changed, 39 deletions(-) delete mode 100644 app/media/MyAnsible/group_vars/all diff --git a/app/media/MyAnsible/group_vars/all b/app/media/MyAnsible/group_vars/all deleted file mode 100644 index 3be1741d..00000000 --- a/app/media/MyAnsible/group_vars/all +++ /dev/null @@ -1,39 +0,0 @@ - -# General -install_ansible_modules: "true" -disable_transparent_huge_pages: "true" - -setup_interface: "false" - -# Network Calico see here for more details https://github.com/projectcalico/calico/releases -calico_operator_url: "https://raw.githubusercontent.com/projectcalico/calico/v3.29.0/manifests/tigera-operator.yaml" -calico_crd_url: "https://raw.githubusercontent.com/projectcalico/calico/v3.29.0/manifests/custom-resources.yaml" -pod_network_cidr: "192.168.0.0/16" - -# DNS -resolv_nameservers: [8.8.8.8, 4.2.2.4] # 403.online - -# Sanction shekan -use_iran: "true" # change it to "false" if you are outside of iran - -# Docker -docker_gpg_key_url: "https://download.docker.com/linux/ubuntu/gpg" -docker_gpg_key_path: "/etc/apt/keyrings/docker.gpg" -docker_apt_repo: "https://download.docker.com/linux/ubuntu" - -# Kubernetes -kubernetes_gpg_keyring_path: "/etc/apt/keyrings/kubernetes-apt-keyring.gpg" -kubernetes_gpg_key_url: "https://pkgs.k8s.io/core:/stable:/v1.31/deb/Release.key" -kubernetes_apt_repo: "https://pkgs.k8s.io/core:/stable:/v1.31/deb/" -k8s_version: 1.31 # see here https://kubernetes.io/releases/patch-releases/ and https://github.com/kubernetes/kubernetes/releases - -# CRI -cri_socket: unix:///var/run/containerd/containerd.sock - -# Ansible Connection -ansible_user: root -ansible_port: 22 -ansible_python_interpreter: "/usr/bin/python3" -domain: "devopsgpt.com" -apiserver_url: "devopsgpt.com" - \ No newline at end of file