From 0c207af8881071896b2905d4e2398b7eef6abc7c Mon Sep 17 00:00:00 2001 From: Kosuke Morimoto Date: Tue, 7 May 2024 12:55:01 +0900 Subject: [PATCH] implement prototype of agent grpc server --- rust/Cargo.lock | 142 +++++++++++++++++++++++---- rust/bin/agent/Cargo.toml | 5 + rust/bin/agent/src/handler.rs | 12 +++ rust/bin/agent/src/handler/common.rs | 6 ++ rust/bin/agent/src/handler/index.rs | 45 +++++++++ rust/bin/agent/src/handler/insert.rs | 32 ++++++ rust/bin/agent/src/handler/remove.rs | 41 ++++++++ rust/bin/agent/src/handler/search.rs | 112 +++++++++++++++++++++ rust/bin/agent/src/handler/update.rs | 33 +++++++ rust/bin/agent/src/handler/upsert.rs | 33 +++++++ rust/bin/agent/src/main.rs | 16 ++- rust/libs/proto/Cargo.toml | 4 +- rust/libs/proto/src/lib.rs | 8 +- 13 files changed, 460 insertions(+), 29 deletions(-) create mode 100644 rust/bin/agent/src/handler.rs create mode 100644 rust/bin/agent/src/handler/common.rs create mode 100644 rust/bin/agent/src/handler/index.rs create mode 100644 rust/bin/agent/src/handler/insert.rs create mode 100644 rust/bin/agent/src/handler/remove.rs create mode 100644 rust/bin/agent/src/handler/search.rs create mode 100644 rust/bin/agent/src/handler/update.rs create mode 100644 rust/bin/agent/src/handler/upsert.rs diff --git a/rust/Cargo.lock b/rust/Cargo.lock index ec441bb1204..5556008fa0d 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -22,6 +22,11 @@ name = "agent" version = "0.1.0" dependencies = [ "ngt", + "prost", + "proto", + "tokio", + "tokio-stream", + "tonic", ] [[package]] @@ -164,9 +169,9 @@ checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" [[package]] name = "cc" -version = "1.0.95" +version = "1.0.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d32a725bc159af97c3e629873bb9f88fb8cf8a4867175f76dc987815ea07c83b" +checksum = "065a29261d53ba54260972629f9ca6bffa69bac13cd1fed61420f7fa68b9f8bd" [[package]] name = "cfg-if" @@ -339,9 +344,15 @@ checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" [[package]] name = "hashbrown" -version = "0.14.3" +version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + +[[package]] +name = "hermit-abi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" [[package]] name = "http" @@ -430,7 +441,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" dependencies = [ "equivalent", - "hashbrown 0.14.3", + "hashbrown 0.14.5", ] [[package]] @@ -456,9 +467,9 @@ checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" [[package]] name = "libc" -version = "0.2.153" +version = "0.2.154" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" +checksum = "ae743338b92ff9146ce83992f766a31066a91a8c84a45e0e9f21e7cf6de6d346" [[package]] name = "link-cplusplus" @@ -475,6 +486,16 @@ version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" +[[package]] +name = "lock_api" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" +dependencies = [ + "autocfg", + "scopeguard", +] + [[package]] name = "matchit" version = "0.7.3" @@ -562,6 +583,16 @@ dependencies = [ "rand", ] +[[package]] +name = "num_cpus" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +dependencies = [ + "hermit-abi", + "libc", +] + [[package]] name = "object" version = "0.32.2" @@ -583,6 +614,29 @@ version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "caff54706df99d2a78a5a4e3455ff45448d81ef1bb63c22cd14052ca0e993a3f" +[[package]] +name = "parking_lot" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e4af0ca4f6caed20e900d564c242b8e5d4903fdacf31d3daf527b66fe6f42fb" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-targets 0.52.5", +] + [[package]] name = "percent-encoding" version = "2.3.1" @@ -717,6 +771,15 @@ dependencies = [ "getrandom", ] +[[package]] +name = "redox_syscall" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "469052894dcb553421e483e4209ee581a45100d31b4018de03e5a7ad86374a7e" +dependencies = [ + "bitflags 2.5.0", +] + [[package]] name = "rustc-demangle" version = "0.1.23" @@ -742,6 +805,12 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "80af6f9131f277a45a3fba6ce8e2258037bb0477a67e610d3c1fe046ab31de47" +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + [[package]] name = "scratch" version = "1.0.7" @@ -750,24 +819,33 @@ checksum = "a3cf7c11c38cb994f3d40e8a8cde3bbd1f72a435e4c49e85d6553d8312306152" [[package]] name = "serde" -version = "1.0.198" +version = "1.0.199" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9846a40c979031340571da2545a4e5b7c4163bdae79b301d5f86d03979451fcc" +checksum = "0c9f6e76df036c77cd94996771fb40db98187f096dd0b9af39c6c6e452ba966a" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.198" +version = "1.0.199" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e88edab869b01783ba905e7d0153f9fc1a6505a96e4ad3018011eedb838566d9" +checksum = "11bd257a6541e141e42ca6d24ae26f7714887b47e89aa739099104c7e4d3b7fc" dependencies = [ "proc-macro2", "quote", "syn", ] +[[package]] +name = "signal-hook-registry" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" +dependencies = [ + "libc", +] + [[package]] name = "slab" version = "0.4.9" @@ -777,6 +855,12 @@ dependencies = [ "autocfg", ] +[[package]] +name = "smallvec" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" + [[package]] name = "smawk" version = "0.3.2" @@ -785,9 +869,9 @@ checksum = "b7c388c1b5e93756d0c740965c41e8822f866621d41acbdf6336a6a168f8840c" [[package]] name = "socket2" -version = "0.5.6" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05ffd9c0a93b7543e062e759284fcf5f5e3b098501104bfbdde4d404db792871" +checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" dependencies = [ "libc", "windows-sys 0.52.0", @@ -891,8 +975,12 @@ dependencies = [ "bytes", "libc", "mio", + "num_cpus", + "parking_lot", "pin-project-lite", + "signal-hook-registry", "socket2", + "tokio-macros", "windows-sys 0.48.0", ] @@ -906,6 +994,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-macros" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "tokio-stream" version = "0.1.15" @@ -915,6 +1014,7 @@ dependencies = [ "futures-core", "pin-project-lite", "tokio", + "tokio-util", ] [[package]] @@ -933,9 +1033,9 @@ dependencies = [ [[package]] name = "tonic" -version = "0.10.2" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d560933a0de61cf715926b9cac824d4c883c2c43142f787595e48280c40a1d0e" +checksum = "76c4eb7a4e9ef9d4763600161f12f5070b92a578e1b634db88a6887844c91a13" dependencies = [ "async-stream", "async-trait", @@ -960,9 +1060,9 @@ dependencies = [ [[package]] name = "tonic-types" -version = "0.10.2" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b39bd850e4bf99146b3fd244019562cafd30338db068c5795c55b448eb02411" +checksum = "f4aa089471d8d4c60ec3aef047739713a4695f0b309d4cea0073bc55201064f4" dependencies = [ "prost", "prost-types", @@ -1052,9 +1152,9 @@ checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f" [[package]] name = "unicode-width" -version = "0.1.11" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" +checksum = "68f5e5f3158ecfd4b8ff6fe086db7c8467a2dfdac97fe420f2b7c4aa97af66d6" [[package]] name = "want" @@ -1073,9 +1173,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "winapi-util" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "134306a13c5647ad6453e8deaec55d3a44d6021970129e6188735e74bf546697" +checksum = "4d4cc384e1e73b93bafa6fb4f1df8c41695c8a91cf9c4c64358067d15a7b6c6b" dependencies = [ "windows-sys 0.52.0", ] diff --git a/rust/bin/agent/Cargo.toml b/rust/bin/agent/Cargo.toml index 5fc426d1960..fdfce09733e 100644 --- a/rust/bin/agent/Cargo.toml +++ b/rust/bin/agent/Cargo.toml @@ -22,3 +22,8 @@ edition = "2021" [dependencies] ngt = { version = "0.1.0", path = "../../libs/ngt" } +prost = "0.12.4" +proto = { version = "0.1.0", path = "../../libs/proto" } +tokio = { version = "1.37.0", features = ["full"] } +tokio-stream = { version = "0.1.15", features = ["full"] } +tonic = "0.11.0" diff --git a/rust/bin/agent/src/handler.rs b/rust/bin/agent/src/handler.rs new file mode 100644 index 00000000000..6156942d1ab --- /dev/null +++ b/rust/bin/agent/src/handler.rs @@ -0,0 +1,12 @@ +mod common; +mod index; +mod insert; +mod remove; +mod search; +mod update; +mod upsert; + +#[derive(Default, Debug)] +pub struct Agent { + +} diff --git a/rust/bin/agent/src/handler/common.rs b/rust/bin/agent/src/handler/common.rs new file mode 100644 index 00000000000..99d1d918902 --- /dev/null +++ b/rust/bin/agent/src/handler/common.rs @@ -0,0 +1,6 @@ +#[macro_export] +macro_rules! stream_type { + ($t:ty) => { + std::pin::Pin> + Send>> + }; +} diff --git a/rust/bin/agent/src/handler/index.rs b/rust/bin/agent/src/handler/index.rs new file mode 100644 index 00000000000..610fc5d1a43 --- /dev/null +++ b/rust/bin/agent/src/handler/index.rs @@ -0,0 +1,45 @@ +use proto::{ + core::v1::agent_server, + payload::v1::{control, info, object, Empty}, +}; + +#[tonic::async_trait] +impl agent_server::Agent for super::Agent { + async fn create_index( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status> { + todo!() + } + + async fn save_index( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status> { + todo!() + } + + #[doc = " Represent the creating and saving index RPC.\n"] + async fn create_and_save_index( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status> { + todo!() + } + + #[doc = " Represent the RPC to get the agent index information.\n"] + async fn index_info( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status> { + todo!() + } + + #[doc = " Represent the RPC to get the vector metadata. This RPC is mainly used for index correction process\n"] + async fn get_timestamp( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status> { + todo!() + } +} diff --git a/rust/bin/agent/src/handler/insert.rs b/rust/bin/agent/src/handler/insert.rs new file mode 100644 index 00000000000..0a43f0f2665 --- /dev/null +++ b/rust/bin/agent/src/handler/insert.rs @@ -0,0 +1,32 @@ +use proto::{ + payload::v1::{insert, object}, + vald::v1::insert_server, +}; +#[tonic::async_trait] +impl insert_server::Insert for super::Agent { + async fn insert( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status> { + todo!() + } + + #[doc = " Server streaming response type for the StreamInsert method."] + type StreamInsertStream = crate::stream_type!(object::StreamLocation); + + #[doc = " A method to add new multiple vectors by bidirectional streaming.\n"] + async fn stream_insert( + &self, + request: tonic::Request>, + ) -> std::result::Result, tonic::Status> { + todo!() + } + + #[doc = " A method to add new multiple vectors in a single request.\n"] + async fn multi_insert( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status> { + todo!() + } +} diff --git a/rust/bin/agent/src/handler/remove.rs b/rust/bin/agent/src/handler/remove.rs new file mode 100644 index 00000000000..16bf406205a --- /dev/null +++ b/rust/bin/agent/src/handler/remove.rs @@ -0,0 +1,41 @@ +use proto::{ + payload::v1::{object, remove}, + vald::v1::remove_server, +}; + +#[tonic::async_trait] +impl remove_server::Remove for super::Agent { + async fn remove( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status> { + todo!() + } + + #[doc = " A method to remove an indexed vector based on timestamp.\n"] + async fn remove_by_timestamp( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status> { + todo!() + } + + #[doc = " Server streaming response type for the StreamRemove method."] + type StreamRemoveStream = crate::stream_type!(object::StreamLocation); + + #[doc = " A method to remove multiple indexed vectors by bidirectional streaming.\n"] + async fn stream_remove( + &self, + request: tonic::Request>, + ) -> std::result::Result, tonic::Status> { + todo!() + } + + #[doc = " A method to remove multiple indexed vectors in a single request.\n"] + async fn multi_remove( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status> { + todo!() + } +} diff --git a/rust/bin/agent/src/handler/search.rs b/rust/bin/agent/src/handler/search.rs new file mode 100644 index 00000000000..374e9be9424 --- /dev/null +++ b/rust/bin/agent/src/handler/search.rs @@ -0,0 +1,112 @@ +use proto::{payload::v1::search, vald::v1::search_server}; + +#[tonic::async_trait] +impl search_server::Search for super::Agent { + async fn search( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + todo!() + } + + #[doc = " A method to search indexed vectors by ID.\n"] + async fn search_by_id( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + todo!() + } + + #[doc = " Server streaming response type for the StreamSearch method."] + type StreamSearchStream = crate::stream_type!(search::StreamResponse); + + #[doc = " A method to search indexed vectors by multiple vectors.\n"] + async fn stream_search( + &self, + request: tonic::Request>, + ) -> std::result::Result, tonic::Status> { + todo!() + } + + #[doc = " Server streaming response type for the StreamSearchByID method."] + type StreamSearchByIDStream = crate::stream_type!(search::StreamResponse); + + #[doc = " A method to search indexed vectors by multiple IDs.\n"] + async fn stream_search_by_id( + &self, + request: tonic::Request>, + ) -> std::result::Result, tonic::Status> { + todo!() + } + + #[doc = " A method to search indexed vectors by multiple vectors in a single request.\n"] + async fn multi_search( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status> { + todo!() + } + + #[doc = " A method to search indexed vectors by multiple IDs in a single request.\n"] + async fn multi_search_by_id( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status> { + todo!() + } + + #[doc = " A method to linear search indexed vectors by a raw vector.\n"] + async fn linear_search( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status> { + todo!() + } + + #[doc = " A method to linear search indexed vectors by ID.\n"] + async fn linear_search_by_id( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status> { + todo!() + } + + #[doc = " Server streaming response type for the StreamLinearSearch method."] + type StreamLinearSearchStream = crate::stream_type!(search::StreamResponse); + + #[doc = " A method to linear search indexed vectors by multiple vectors.\n"] + async fn stream_linear_search( + &self, + request: tonic::Request>, + ) -> std::result::Result, tonic::Status> { + todo!() + } + + #[doc = " Server streaming response type for the StreamLinearSearchByID method."] + type StreamLinearSearchByIDStream = crate::stream_type!(search::StreamResponse); + + #[doc = " A method to linear search indexed vectors by multiple IDs.\n"] + async fn stream_linear_search_by_id( + &self, + request: tonic::Request>, + ) -> std::result::Result, tonic::Status> + { + todo!() + } + + #[doc = " A method to linear search indexed vectors by multiple vectors in a single\n request.\n"] + async fn multi_linear_search( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status> { + todo!() + } + + #[doc = " A method to linear search indexed vectors by multiple IDs in a single\n request.\n"] + async fn multi_linear_search_by_id( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status> { + todo!() + } +} diff --git a/rust/bin/agent/src/handler/update.rs b/rust/bin/agent/src/handler/update.rs new file mode 100644 index 00000000000..1a6e9f48417 --- /dev/null +++ b/rust/bin/agent/src/handler/update.rs @@ -0,0 +1,33 @@ +use proto::{ + payload::v1::{object, update}, + vald::v1::update_server, +}; + +#[tonic::async_trait] +impl update_server::Update for super::Agent { + async fn update( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status> { + todo!() + } + + #[doc = " Server streaming response type for the StreamUpdate method."] + type StreamUpdateStream = crate::stream_type!(object::StreamLocation); + + #[doc = " A method to update multiple indexed vectors by bidirectional streaming.\n"] + async fn stream_update( + &self, + request: tonic::Request>, + ) -> std::result::Result, tonic::Status> { + todo!() + } + + #[doc = " A method to update multiple indexed vectors in a single request.\n"] + async fn multi_update( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status> { + todo!() + } +} diff --git a/rust/bin/agent/src/handler/upsert.rs b/rust/bin/agent/src/handler/upsert.rs new file mode 100644 index 00000000000..a6e8a3e362f --- /dev/null +++ b/rust/bin/agent/src/handler/upsert.rs @@ -0,0 +1,33 @@ +use proto::{ + payload::v1::{object, upsert}, + vald::v1::upsert_server, +}; + +#[tonic::async_trait] +impl upsert_server::Upsert for super::Agent { + async fn upsert( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status> { + todo!() + } + + #[doc = " Server streaming response type for the StreamUpsert method."] + type StreamUpsertStream = crate::stream_type!(object::StreamLocation); + + #[doc = " A method to insert/update multiple vectors by bidirectional streaming.\n"] + async fn stream_upsert( + &self, + request: tonic::Request>, + ) -> std::result::Result, tonic::Status> { + todo!() + } + + #[doc = " A method to insert/update multiple vectors in a single request.\n"] + async fn multi_upsert( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status> { + todo!() + } +} diff --git a/rust/bin/agent/src/main.rs b/rust/bin/agent/src/main.rs index 7546cec8106..5ef03c184da 100644 --- a/rust/bin/agent/src/main.rs +++ b/rust/bin/agent/src/main.rs @@ -13,6 +13,18 @@ // See the License for the specific language governing permissions and // limitations under the License. // -fn main() { - println!("Hello, world!"); + +mod handler; + +#[tokio::main] +async fn main() -> Result<(), Box> { + let addr = "[::1]:8081".parse()?; + let agent = handler::Agent::default(); + + tonic::transport::Server::builder() + .add_service(proto::core::v1::agent_server::AgentServer::new(agent)) + .serve(addr) + .await?; + + Ok(()) } diff --git a/rust/libs/proto/Cargo.toml b/rust/libs/proto/Cargo.toml index d185dbf6559..72ea9d88f06 100644 --- a/rust/libs/proto/Cargo.toml +++ b/rust/libs/proto/Cargo.toml @@ -23,5 +23,5 @@ edition = "2021" [dependencies] futures-core = "0.3.30" prost = "0.12.3" -tonic = "0.10.2" -tonic-types = "0.10.2" +tonic = "0.11.0" +tonic-types = "0.11.0" diff --git a/rust/libs/proto/src/lib.rs b/rust/libs/proto/src/lib.rs index a25f31ae094..b9b1d0bac08 100644 --- a/rust/libs/proto/src/lib.rs +++ b/rust/libs/proto/src/lib.rs @@ -31,8 +31,8 @@ pub mod vald { } } -#[cfg(test)] -mod tests { - #[test] - fn it_works() {} +pub mod core { + pub mod v1 { + include!("core.v1.tonic.rs"); + } }