docs: Release v3.32.0

doc/config/_default/config.toml

@@ -24,7 +24,7 @@ pygmentsUseClasses = true
   github_repository = "https://github.com/TheThingsIndustries/lorawan-stack-docs"
   github_repository_edit = "https://github.com/TheThingsIndustries/lorawan-stack-docs/blob/master/doc/content"
   tts_github_repository = "https://github.com/TheThingsNetwork/lorawan-stack"
-  version = "3.31.1"
+  version = "3.32.0"
 
 [markup]
   [markup.goldmark]

doc/content/the-things-stack/host/aws/ecs/changelog/index.md

@@ -8,6 +8,13 @@ All meaningful changes to templates are documented in this file.
 
 ## Unreleased
 
+- Add support for managed gateways via The Things Gateway Controller. The Gateway Configuration Server and Device Claiming Server use TLS client authentication.
+  - When using AWS Private CA (`CertificateAuthorityARN` in `4-2a-configuration`), the client certificate can be issued automatically.
+  - To specify a custom TLS client certificate, enable `EnableTTGCCustomCertificate` in `4-1-secrets` and specify the certificate and key according to the format in the description.
+- Add support for gateways using The Things Industries Gateway Protocol. This requires TLS mutual authentication and TLS termination by the proxy. Make sure that `SupportProxyTLS` is enabled. This adds a new public listener (port `8889`) that is mapped to the proxy that forwards traffic to the Gateway Server (port `1889`).
+- Fixed the rate-limiting profile for the `ApplicationUpStorage` service in the Application Server.
+- Add default values for the default and maximum page sizes in the `ApplicationUpStorage` service in the Application Server.
+
 ## 3.31.1
 
 ### Proxy

doc/content/ttn-lw-cli/ttn-lw-cli_end-devices_list.md

@@ -24,6 +24,7 @@ ttn-lw-cli end-devices list [application-id] [flags]
       --claim-authentication-code.valid-to     select the claim_authentication_code.valid_to field
       --claim-authentication-code.value        select the claim_authentication_code.value field
       --description                            select the description field
+      --filter.updated-since updated_at        filter the updated_at field by this timestamp (YYYY-MM-DDTHH:MM:SSZ)
   -h, --help                                   help for list
       --join-server-address                    select the join_server_address field
       --last-seen-at                           select the last_seen_at field

doc/content/ttn-lw-cli/ttn-lw-cli_gateways_claim.md

@@ -39,9 +39,8 @@ ttn-lw-cli gateways claim [gateway-eui] [flags]
       --gateway-id string                      
   -h, --help                                   help for claim
       --organization-id string                 
-      --target-cups-trust-local-file string    (optional) Target CUPS trust in PEM format (local file name)
-      --target-cups-uri string                 
       --target-frequency-plan-id string        
+      --target-frequency-plan-ids string       
       --target-gateway-id string               gateway ID for the claimed gateway
       --target-gateway-server-address string   
       --user-id string                         
@@ -87,7 +86,5 @@ ttn-lw-cli gateways claim [gateway-eui] [flags]
 ### SEE ALSO
 
 * [ttn-lw-cli gateways]({{< relref "ttn-lw-cli_gateways" >}})	 - Gateway commands
-* [ttn-lw-cli gateways claim authorize]({{< relref "ttn-lw-cli_gateways_claim_authorize" >}})	 - Authorize a gateway for claiming (EXPERIMENTAL)
 * [ttn-lw-cli gateways claim info]({{< relref "ttn-lw-cli_gateways_claim_info" >}})	 - Check if claiming is supported for the EUI (EXPERIMENTAL)
-* [ttn-lw-cli gateways claim unauthorize]({{< relref "ttn-lw-cli_gateways_claim_unauthorize" >}})	 - Unauthorize a gateway for claiming (EXPERIMENTAL)
 

doc/content/ttn-lw-cli/ttn-lw-cli_gateways_create.md

@@ -63,6 +63,7 @@ ttn-lw-cli gateways create [gateway-id] [flags]
       --version-ids.firmware-version string                                  
       --version-ids.hardware-version string                                  
       --version-ids.model-id string                                          
+      --version-ids.runtime-version string                                   
 ```
 
 ### Options inherited from parent commands

doc/content/ttn-lw-cli/ttn-lw-cli_gateways_get.md

@@ -70,6 +70,7 @@ ttn-lw-cli gateways get [gateway-id] [flags]
       --version-ids.firmware-version                                  select the version_ids.firmware_version field
       --version-ids.hardware-version                                  select the version_ids.hardware_version field
       --version-ids.model-id                                          select the version_ids.model_id field
+      --version-ids.runtime-version                                   select the version_ids.runtime_version field
 ```
 
 ### Options inherited from parent commands

doc/content/ttn-lw-cli/ttn-lw-cli_gateways_list.md

@@ -39,6 +39,7 @@ ttn-lw-cli gateways list [flags]
       --downlink-path-constraint                                      select the downlink_path_constraint field
       --enforce-duty-cycle                                            select the enforce_duty_cycle field
       --field-mask strings                                            
+      --filter.updated-since updated_at                               filter the updated_at field by this timestamp (YYYY-MM-DDTHH:MM:SSZ)
       --frequency-plan-id                                             select the frequency_plan_id field
       --frequency-plan-ids                                            select the frequency_plan_ids field
       --gateway-server-address                                        select the gateway_server_address field
@@ -75,6 +76,7 @@ ttn-lw-cli gateways list [flags]
       --version-ids.firmware-version                                  select the version_ids.firmware_version field
       --version-ids.hardware-version                                  select the version_ids.hardware_version field
       --version-ids.model-id                                          select the version_ids.model_id field
+      --version-ids.runtime-version                                   select the version_ids.runtime_version field
 ```
 
 ### Options inherited from parent commands

doc/content/ttn-lw-cli/ttn-lw-cli_gateways_search.md

@@ -79,6 +79,7 @@ ttn-lw-cli gateways search [flags]
       --version-ids.firmware-version                                  select the version_ids.firmware_version field
       --version-ids.hardware-version                                  select the version_ids.hardware_version field
       --version-ids.model-id                                          select the version_ids.model_id field
+      --version-ids.runtime-version                                   select the version_ids.runtime_version field
 ```
 
 ### Options inherited from parent commands

doc/content/ttn-lw-cli/ttn-lw-cli_gateways_set.md

@@ -64,6 +64,7 @@ ttn-lw-cli gateways set [gateway-id] [flags]
       --version-ids.firmware-version string                                  
       --version-ids.hardware-version string                                  
       --version-ids.model-id string                                          
+      --version-ids.runtime-version string                                   
 ```
 
 ### Options inherited from parent commands

doc/content/whats-new/3.32.0.md

@@ -0,0 +1,70 @@
+---
+date: 2024-09-09T14:11:11Z
+title: "3.32.0"
+featured: {
+  added: [],
+  changed: [],
+  fixed: []
+}
+---
+
+### Added
+
+- Newly redesigned The Things Stack Console. The most notable changes are as follows:
+  - New look and feel
+  - New sidebar-based navigation and structure
+  - Top entities are automatically stored and displayed for faster access
+  - More useful panels in the overviews of gateways, applications and end devices
+  - Quick search feature to search globally for any entity
+  - New event overlay for better in-context event examination
+  - Notifications can now be viewed from within the Console
+- Add `ResendAdminValidation` RPC to the `TenantAccess` service. Allows tenant administrators to re-send the tenant's email validation email to their initial user.
+- Add `ResendAdminValidation` RPC to the `TenantAccess` service. Allows tenant administrators to re-send the tenant's email validation email to their initial user.
+- Support for managed gateways and claiming through The Things Gateway Controller.
+- Support for The Things Industries gateway protocol. This is adds a new pair of ports to Gateway Server: `1889` for Envoy or Traefik terminated TLS mutual authentication, and `8889` for The Things Stack terminated TLS mutual authentication.
+- Option to filter the output of the `List` RPCs for end devices and gateways based on when they were updated.
+  - `ListGatewaysRequest` and `ListEndDevicesRequest` RPCs have a new `Filter` field that supports an `updated_since` timestamp.
+- Preparation for universal rights assigned to users.
+  - This requires a database schema migration (`ttn-lw-stack is-db migrate`).
+
+### Changed
+
+- Enforce `NOT NULL` constraint on Network Operations Center's `end_device_data_uplink` hypertable.
+  - This requires a Network Operations Center migration (`tti-lw-stack noc-db migrate`).
+
+### Deprecated
+
+- Experimental RPCs `AuthorizeGateway` and `UnauthorizeGateway` of the Gateway Claiming Server (GCLS).
+- `CUPSRedirection` field of `ClaimGatewayRequest`.
+
+### Removed
+
+- `authorize` and `unauthorize` sub-commands of the gateway `claim` command.
+
+### Fixed
+
+- Fixed an issue where an downlink message was scheduled by the Application Layer Clock Synchronization (ALCS) implementation when there is no answer to send (i.e. `AnsRequired` is not set and the difference with what the end device reports falls within the threshold).
+- Fix unhandled video stream in QR code scanner
+- Recreate Network Operations Center's `gateway_end_device_uplink_statistics_1h` aggregate with a proper time bucket.
+  - This requires a Network Operations Center migration (`tti-lw-stack noc-db migrate`).
+  - Decreases aggregate's total data size to be less than 10% of the previous size.
+  - To generate data for the new table, run the following command:
+    - `CALL refresh_continuous_aggregate('gateway_end_device_uplink_statistics_1h', NOW() - interval '30d', null);`
+  - Deployments that have retention period for the hypertable `gateway_end_device_uplink` shorter than 30 days will see a loss of data in the migration. Specifically the data loss happens in the period between ['30d', retention_period).
+  - Grafana graphs will be impacted but slowly replenished by the new aggregate.
+- The Network Operation Center no longer returns an incorrect amount of active Gateways, End-devices and Applications on the bigger bucket intervals.
+  - This requires a Network Operations Center migration (`tti-lw-stack noc-db migrate`).
+  - To generate data for the new aggregates, run the following commands:
+    - `CALL refresh_continuous_aggregate('active_gateways_1h', NOW() - interval '30d', null);`
+    - `CALL refresh_continuous_aggregate('active_gateways_1d', NOW() - interval '30d', null);`
+    - `CALL refresh_continuous_aggregate('active_end_devices_1h', NOW() - interval '30d', null);`
+    - `CALL refresh_continuous_aggregate('active_end_devices_1d', NOW() - interval '30d', null);`
+    - `CALL refresh_continuous_aggregate('active_applications_1h', NOW() - interval '30d', null);`
+    - `CALL refresh_continuous_aggregate('active_applications_1d', NOW() - interval '30d', null);`
+    - `CALL refresh_continuous_aggregate('active_gateway_count_1d', NOW() - interval '30d', null);`
+    - `CALL refresh_continuous_aggregate('active_gateway_count_1w', NOW() - interval '30d', null);`
+    - `CALL refresh_continuous_aggregate('active_end_device_count_1d', NOW() - interval '30d', null);`
+    - `CALL refresh_continuous_aggregate('active_end_device_count_1w', NOW() - interval '30d', null);`
+    - `CALL refresh_continuous_aggregate('active_application_count_1d', NOW() - interval '30d', null);`
+    - `CALL refresh_continuous_aggregate('active_application_count_1w', NOW() - interval '30d', null);`
+- Network Operations Center no longer returns only the last three days of data of gateways' connection events.

doc/data/api/ttn.lorawan.v3/enums/ManagedGatewayNetworkInterfaceStatus.yml

@@ -0,0 +1,10 @@
+name: ManagedGatewayNetworkInterfaceStatus
+values:
+- name: MANAGED_GATEWAY_NETWORK_INTERFACE_STATUS_UNSPECIFIED
+  value: 0
+- name: MANAGED_GATEWAY_NETWORK_INTERFACE_STATUS_DOWN
+  value: 1
+- name: MANAGED_GATEWAY_NETWORK_INTERFACE_STATUS_UP
+  value: 2
+- name: MANAGED_GATEWAY_NETWORK_INTERFACE_STATUS_FAILED
+  value: 3

doc/data/api/ttn.lorawan.v3/enums/ManagedGatewayNetworkInterfaceType.yml

@@ -0,0 +1,10 @@
+name: ManagedGatewayNetworkInterfaceType
+values:
+- name: MANAGED_GATEWAY_NETWORK_INTERFACE_TYPE_UNSPECIFIED
+  value: 0
+- name: MANAGED_GATEWAY_NETWORK_INTERFACE_TYPE_CELLULAR
+  value: 1
+- name: MANAGED_GATEWAY_NETWORK_INTERFACE_TYPE_WIFI
+  value: 2
+- name: MANAGED_GATEWAY_NETWORK_INTERFACE_TYPE_ETHERNET
+  value: 3

doc/data/api/ttn.lorawan.v3/messages/ApplicationDownlink.yml

@@ -22,6 +22,8 @@ fields:
     The payload is encrypted if the skip_payload_crypto field of the EndDevice
     is true.
   type: bytes
+  rules:
+    max_len: 255
   default: ""
 - name: decoded_payload
   comment: |-

doc/data/api/ttn.lorawan.v3/messages/CUPSRedirection.yml

@@ -1,4 +1,6 @@
 name: CUPSRedirection
+comment: 'DEPRECATED: This message is deprecated and will be removed in a future version
+  of The Things Stack.'
 fields:
 - name: target_cups_uri
   comment: CUPS URI for LoRa Basics Station CUPS redirection.

doc/data/api/ttn.lorawan.v3/messages/ClaimGatewayRequest.yml

@@ -32,7 +32,9 @@ fields:
     pattern: ^(?:(?:[a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*(?:[A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])(?::[0-9]{1,5})?$|^$
   default: ""
 - name: cups_redirection
-  comment: Parameters to set CUPS redirection for the gateway.
+  comment: |-
+    Parameters to set CUPS redirection for the gateway.
+    DEPRECATED: This field is deprecated and will be removed in a future version of The Things Stack.
   message:
     name: CUPSRedirection
   default: {}

doc/data/api/ttn.lorawan.v3/messages/CreateManagedGatewayEthernetProfileRequest.yml

@@ -0,0 +1,14 @@
+name: CreateManagedGatewayEthernetProfileRequest
+fields:
+- name: profile
+  message:
+    name: ManagedGatewayEthernetProfile
+  rules:
+    required: true
+  default: {}
+- name: collaborator
+  message:
+    name: OrganizationOrUserIdentifiers
+  rules:
+    required: true
+  default: {}

doc/data/api/ttn.lorawan.v3/messages/CreateManagedGatewayWiFiProfileRequest.yml

@@ -0,0 +1,14 @@
+name: CreateManagedGatewayWiFiProfileRequest
+fields:
+- name: profile
+  message:
+    name: ManagedGatewayWiFiProfile
+  rules:
+    required: true
+  default: {}
+- name: collaborator
+  message:
+    name: OrganizationOrUserIdentifiers
+  rules:
+    required: true
+  default: {}

doc/data/api/ttn.lorawan.v3/messages/DeleteManagedGatewayEthernetProfileRequest.yml

@@ -0,0 +1,13 @@
+name: DeleteManagedGatewayEthernetProfileRequest
+fields:
+- name: collaborator
+  message:
+    name: OrganizationOrUserIdentifiers
+  rules:
+    required: true
+  default: {}
+- name: profile_id
+  type: string
+  rules:
+    uuid: true
+  default: ""

doc/data/api/ttn.lorawan.v3/messages/DeleteManagedGatewayWiFiProfileRequest.yml

@@ -0,0 +1,13 @@
+name: DeleteManagedGatewayWiFiProfileRequest
+fields:
+- name: collaborator
+  message:
+    name: OrganizationOrUserIdentifiers
+  rules:
+    required: true
+  default: {}
+- name: profile_id
+  type: string
+  rules:
+    uuid: true
+  default: ""

doc/data/api/ttn.lorawan.v3/messages/GatewayVersionIdentifiers.yml

@@ -25,3 +25,8 @@ fields:
   rules:
     max_len: 32
   default: ""
+- name: runtime_version
+  type: string
+  rules:
+    max_len: 32
+  default: ""

doc/data/api/ttn.lorawan.v3/messages/GetInfoByGatewayEUIResponse.yml

@@ -9,3 +9,9 @@ fields:
 - name: supports_claiming
   type: bool
   default: false
+- name: is_managed
+  comment: |-
+    Indicates whether the gateway is a managed gateway.
+    If true, when the gateway is successfully claimed, it can be managed with ManagedGatewayConfigurationService.
+  type: bool
+  default: false

doc/data/api/ttn.lorawan.v3/messages/GetManagedGatewayEthernetProfileRequest.yml

@@ -0,0 +1,19 @@
+name: GetManagedGatewayEthernetProfileRequest
+fields:
+- name: collaborator
+  message:
+    name: OrganizationOrUserIdentifiers
+  rules:
+    required: true
+  default: {}
+- name: profile_id
+  type: string
+  rules:
+    uuid: true
+  default: ""
+- name: field_mask
+  comment: The names of the gateway fields that should be returned.
+  message:
+    package: google.protobuf
+    name: FieldMask
+  default: {}

doc/data/api/ttn.lorawan.v3/messages/GetManagedGatewayWiFiProfileRequest.yml

@@ -0,0 +1,19 @@
+name: GetManagedGatewayWiFiProfileRequest
+fields:
+- name: collaborator
+  message:
+    name: OrganizationOrUserIdentifiers
+  rules:
+    required: true
+  default: {}
+- name: profile_id
+  type: string
+  rules:
+    uuid: true
+  default: ""
+- name: field_mask
+  comment: The names of the gateway fields that should be returned.
+  message:
+    package: google.protobuf
+    name: FieldMask
+  default: {}

doc/data/api/ttn.lorawan.v3/messages/ListEndDevicesRequest.Filter.yml

@@ -0,0 +1,14 @@
+name: ListEndDevicesRequest.Filter
+comment: Filter end devices by fields.
+fields:
+- name: updated_since
+  message:
+    package: google.protobuf
+    name: Timestamp
+  rules:
+    lt_now: true
+  default: "0001-01-01T00:00:00Z"
+oneofs:
+- name: field
+  field_names:
+  - updated_since

doc/data/api/ttn.lorawan.v3/messages/ListEndDevicesRequest.yml

@@ -45,3 +45,8 @@ fields:
   comment: Page number for pagination. 0 is interpreted as 1.
   type: uint32
   default: 0
+- name: filters
+  repeated:
+    message:
+      name: ListEndDevicesRequest.Filter
+  default: []

doc/data/api/ttn.lorawan.v3/messages/ListGatewaysRequest.Filter.yml

@@ -0,0 +1,14 @@
+name: ListGatewaysRequest.Filter
+comment: Filter gateways by fields.
+fields:
+- name: updated_since
+  message:
+    package: google.protobuf
+    name: Timestamp
+  rules:
+    lt_now: true
+  default: "0001-01-01T00:00:00Z"
+oneofs:
+- name: field
+  field_names:
+  - updated_since

doc/data/api/ttn.lorawan.v3/messages/ListGatewaysRequest.yml

@@ -45,3 +45,8 @@ fields:
   comment: Only return recently deleted gateways.
   type: bool
   default: false
+- name: filters
+  repeated:
+    message:
+      name: ListGatewaysRequest.Filter
+  default: []