Merge pull request #17 from gobicycle/develop

v0.5.0

.github/workflows/go.yml

@@ -33,7 +33,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v3
         with:
-          go-version: 1.19
+          go-version: 1.22
 
       - name: Build
         run: go build -v ./...
@@ -48,12 +48,18 @@ jobs:
       - name: Install dependencies
         run: |
           sudo apt-get update
-          sudo apt-get -y install git zlib1g-dev libssl-dev
+          sudo apt-get -y install openssl ca-certificates libsecp256k1-0 libsodium23
 
       - name: Install libs
         run: |
-          git clone https://github.com/tonkeeper/tongo /tmp/tongo
-          sudo cp /tmp/tongo/lib/linux/libemulator.so /lib
+          wget -O libemulator.so https://github.com/ton-blockchain/ton/releases/download/v2024.02/libemulator-linux-x86-64.so
+          sudo cp libemulator.so /lib
+
+      - name: Update Library Cache
+        run: sudo ldconfig
+
+      - name: Verify Library Presence
+        run: ls /lib | grep libemulator.so
 
       - name: Run Test
         env:

Dockerfile

@@ -1,12 +1,4 @@
-FROM docker.io/library/ubuntu:20.04 AS emulator-builder
-ENV DEBIAN_FRONTEND=noninteractive
-RUN apt-get update
-RUN apt-get -y install git cmake g++ zlib1g-dev libssl-dev
-RUN git clone --recurse-submodules -b emulator_vm_verbosity https://github.com/dungeon-master-666/ton.git
-RUN mkdir build && (cd build && cmake ../ton -DCMAKE_BUILD_TYPE=Release && cmake --build . --target emulator)
-RUN mkdir /output && cp build/emulator/libemulator.so /output
-
-FROM docker.io/library/golang:1.19.2 AS builder
+FROM docker.io/library/golang:1.22-bullseye AS builder
 WORKDIR /build-dir
 COPY go.mod .
 COPY go.sum .
@@ -20,21 +12,23 @@ COPY db db
 COPY audit audit
 COPY queue queue
 COPY webhook webhook
+COPY metrics metrics
+RUN apt-get update && apt-get install -y libsecp256k1-0 libsodium23
 RUN go build -o /tmp/processor github.com/gobicycle/bicycle/cmd/processor
 RUN go build -o /tmp/testutil github.com/gobicycle/bicycle/cmd/testutil
 
 FROM docker.io/library/ubuntu:20.04 AS payment-processor
-RUN apt-get update && apt-get -y install zlib1g-dev libssl-dev openssl ca-certificates && rm -rf /var/lib/apt/lists/*
-RUN mkdir -p /lib
+RUN apt-get update && apt-get install -y openssl ca-certificates libsecp256k1-0 libsodium23 wget && rm -rf /var/lib/apt/lists/*
+RUN mkdir -p /app/lib
+RUN wget -O /app/lib/libemulator.so https://github.com/ton-blockchain/ton/releases/download/v2024.02/libemulator-linux-x86-64.so
+ENV LD_LIBRARY_PATH=/app/lib
 COPY --from=builder /tmp/processor /app/processor
-COPY --from=emulator-builder /output/libemulator.so /lib
-ENV LD_LIBRARY_PATH=/lib
 CMD ["/app/processor", "-v"]
 
 FROM docker.io/library/ubuntu:20.04 AS payment-test
-RUN apt-get update && apt-get -y install zlib1g-dev libssl-dev openssl ca-certificates && rm -rf /var/lib/apt/lists/*
-RUN mkdir -p /lib
+RUN apt-get update && apt-get install -y openssl ca-certificates libsecp256k1-0 libsodium23 wget && rm -rf /var/lib/apt/lists/*
+RUN mkdir -p /app/lib
+RUN wget -O /app/lib/libemulator.so https://github.com/ton-blockchain/ton/releases/download/v2024.02/libemulator-linux-x86-64.so
+ENV LD_LIBRARY_PATH=/app/lib
 COPY --from=builder /tmp/testutil /app/testutil
-COPY --from=emulator-builder /output/libemulator.so /lib
-ENV LD_LIBRARY_PATH=/lib
-CMD ["/app/testutil", "-v"]
+CMD ["/app/testutil", "-v"]

README.md

@@ -18,6 +18,8 @@ Service is ADNL based and interacts directly with node and do not use any third
 - [Deployment](#Deployment)
   - [Configurable parameters](#Configurable-parameters)
   - [Service deploy](#Service-deploy)
+- [Payment notifications](#Payment-notifications)
+- [Binary comment support](#Binary-comment-support)
 - [REST API](https://gobicycle.github.io/bicycle/)
 - [Technical notes](/technical_notes.md)
 - [Threat model](/threat_model.md)
@@ -89,6 +91,7 @@ For more information on Jettons compatibility, see [Jettons compatibility](/jett
 | `WEBHOOK_ENDPOINT`     | endpoint to send webhooks, example: `http://hostname:3333/webhook`. If the value is not set, then webhooks are not sent.                                                                                                                                                                                                                                                      |
 | `WEBHOOK_TOKEN`        | Bearer token for webhook request. If not set then not used.                                                                                                                                                                                                                                                                                                                   |
 | `ALLOWABLE_LAG`        | allowable time lag between service time and last block time in seconds, default: 15                                                                                                                                                                                                                                                                                           |
+| `FORWARD_TON_AMOUNT`   | forward ton amount for jetton withdrawals, default: 20000000 nanoton                                                                                                                                                                                                                                                                                                          |
 
 **! Be careful with `IS_TESTNET` variable.** This does not guarantee that a testnet node is being used. It is only for address checking purposes.
 
@@ -115,28 +118,32 @@ If the `hot_wallet_residual_balance` is not set, then it is calculated using the
 ```console
 make -f Makefile
 ```
-2. Prepare `.env` file for `payment-postgres` service or fill environment variables in `docker-compose-main.yml` file.
+2. Prepare `.env` file for `payment-postgres` service or fill environment variables in `docker-compose.yml` file.
 Database scheme automatically init.
 ```console
-docker-compose -f docker-compose-main.yml up -d payment-postgres
+docker-compose -f docker-compose.yml up -d payment-postgres
 ```
-3. Prepare `.env` file for `payment-processor` service or fill environment variables in `docker-compose-main.yml` file.
+3. Prepare `.env` file for `payment-processor` service or fill environment variables in `docker-compose.yml` file.
 ```console
-docker-compose -f docker-compose-main.yml up -d payment-processor
+docker-compose -f docker-compose.yml up -d payment-processor
 ```
 4. Optionally you can start Grafana for service monitoring. Prepare `.env` file for `payment-grafana` service or 
-fill environment variables in `docker-compose-main.yml` file.
+fill environment variables in `docker-compose.yml` file.
 ```console
-docker-compose -f docker-compose-main.yml up -d payment-grafana
+docker-compose -f docker-compose.yml up -d payment-grafana
 ```
 5. Optionally you can start RabbitMQ to collect payment notifications (if `QUEUE_ENABLED` env var is `true` for payment-processor). 
-Prepare `.env` file for `payment-rabbitmq` service or fill environment variables in `docker-compose-main.yml` file.
+Prepare `.env` file for `payment-rabbitmq` service or fill environment variables in `docker-compose.yml` file.
 ```console
-docker-compose -f docker-compose-main.yml up -d payment-rabbitmq
+docker-compose -f docker-compose.yml up -d payment-rabbitmq
 ```
 
 ## Payment notifications
 
+ATTENTION! Sending notifications does not guarantee that all notifications will be sent. 
+If the service is restarted after the data is saved to the database and before the notification data is sent, 
+these notifications will not be sent after restart.
+
 The service has several mechanisms for notification of payments. These are webhooks and a AMQP (to RabbitMQ). 
 Depending on the `DEPOSIT_SIDE_BALANCE` setting, a notification is received either about the payment to the 
 deposit address, or about the withdrawal from the deposit to the hot wallet. Source address and comment returned if known.
@@ -182,6 +189,23 @@ wait for a response with a `200` code and an empty body. If a successful deliver
 several attempts, the service will stop with an error. If the variable `WEBHOOK_TOKEN` is set, it will also 
 add header `Authorization: Bearer {token}`.
 
+## Binary comment support
+
+Method `/v1/withdrawal/send` also supports `binary_comment`. The comment is written in a hex form. If the bits qty is not 
+a multiple of a byte, then the record form with a flip bit is supported, for example `9fe7_`.
+A `binary_comment` is writing directly to the body of the message (for the TON transfer) and to the `forward_payload` 
+(for the Jetton transfer) with its opcode according to the following TLB scheme:
+
+`binary_comment#b3ddcf7d {n:#} data:(SnakeData ~n) = InternalMsgBody;`
+
+`crc32('binary_comment n:# data:SnakeData ~n = InternalMsgBody') = 0xb3ddcf7d`
+
+This comment will not be displayed by the explorer as text and can be useful for transmitting metadata that 
+will be read by indexers.
+
+The documentation [contains](https://docs.ton.org/develop/smart-contracts/guidelines/internal-messages#simple-message-with-comment) a standard way of writing a binary comment, but due to the fact that it is not supported 
+by services, an alternative recording method was chosen.
+
 <!-- Badges -->
 [ton-svg]: https://img.shields.io/badge/Based%20on-TON-blue
 [ton]: https://ton.org

api/example.http

@@ -21,7 +21,7 @@ GET {{url}}/v1/income?user_id=TestUser
 Authorization: Bearer {{token}}
 
 ###
-GET {{url}}/v1/deposit/history?user_id=TestUser&currency=TON&limit=3&offset=0
+GET {{url}}/v1/deposit/history?user_id=TestUser&currency=TON&limit=3&offset=0&sort_order=asc
 Authorization: Bearer {{token}}
 
 ###
@@ -38,6 +38,14 @@ Content-Type: application/json
 
 {"user_id": "TestUser", "query_id": "2", "currency": "TGR", "amount":  1000, "destination": "kQBFETbGASx3-6QYpPuQAKQM1s32AfSkWzbsADqt3bKDlN1A", "comment":  "test_jetton_withdrawal"}
 
+###
+POST {{url}}/v1/withdrawal/send
+Authorization: Bearer {{token}}
+Content-Type: application/json
+
+{"user_id": "TestUser", "query_id": "3", "currency": "TON", "amount":  1000, "destination": "kQBFETbGASx3-6QYpPuQAKQM1s32AfSkWzbsADqt3bKDlN1A", "binary_comment":  "9fe7_"}
+
+
 ###
 POST {{url}}/v1/withdrawal/service/jetton
 Authorization: Bearer {{token}}
@@ -57,4 +65,23 @@ GET {{url}}/v1/withdrawal/status?id=2
 Authorization: Bearer {{token}}
 
 ###
-GET {{url}}/v1/system/sync
+GET {{url}}/v1/system/sync
+
+###
+GET {{url}}/metrics
+
+###
+GET {{url}}//v1/deposit/income?tx_hash=54e61136c33b94372030de8c7d02bc23a60e3de7cfad46f26258e8e722dc66b1
+Authorization: Bearer {{token}}
+
+###
+GET {{url}}//v1/balance?currency=TON&address=kQAbMQzuuGiCne0R7QEj9nrXsjM7gNjeVmrlBZouyC-SCALE
+Authorization: Bearer {{token}}
+
+###
+GET {{url}}//v1/balance?currency=TON
+Authorization: Bearer {{token}}
+
+###
+GET {{url}}//v1/resolve?domain=wallet.ton
+Authorization: Bearer {{token}}