From 6221728d394af80911d297c67f553ad5ae5e6ae0 Mon Sep 17 00:00:00 2001 From: Wojak Date: Thu, 24 Oct 2024 23:06:24 +0700 Subject: [PATCH] Fix gateway + hb instances communication --- .gitignore | 3 + .../master_account/conf_client.yml | 2 +- conf/conf_client copy.yml | 196 ------------ conf/conf_client.yml | 2 +- conf/conf_fee_overrides copy.yml | 298 ------------------ conf/hummingbot_logs copy.yml | 83 ----- conf/mango.yml | 2 + conf/root.yml | 8 + conf/server.yml | 34 +- conf/solana.yml | 24 ++ docker-compose.yml | 10 +- routers/instances.py | 6 +- services/accounts_service.py | 95 +++--- services/docker_service.py | 102 +++--- utils/gateway.py | 13 +- 15 files changed, 171 insertions(+), 707 deletions(-) delete mode 100644 conf/conf_client copy.yml delete mode 100644 conf/conf_fee_overrides copy.yml delete mode 100755 conf/hummingbot_logs copy.yml create mode 100644 conf/mango.yml create mode 100644 conf/solana.yml diff --git a/.gitignore b/.gitignore index 68bc17f..d2d4c28 100644 --- a/.gitignore +++ b/.gitignore @@ -158,3 +158,6 @@ cython_debug/ # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ +secret_key.txt +bots/.DS_Store +.DS_Store diff --git a/bots/credentials/master_account/conf_client.yml b/bots/credentials/master_account/conf_client.yml index 7df2b63..451a01c 100644 --- a/bots/credentials/master_account/conf_client.yml +++ b/bots/credentials/master_account/conf_client.yml @@ -111,7 +111,7 @@ gateway: gateway_api_host: localhost gateway_api_port: '15888' -certs_path: /certs +certs_path: ./certs # Whether to enable aggregated order and trade data collection anonymized_metrics_mode: diff --git a/conf/conf_client copy.yml b/conf/conf_client copy.yml deleted file mode 100644 index 7df2b63..0000000 --- a/conf/conf_client copy.yml +++ /dev/null @@ -1,196 +0,0 @@ -#################################### -### client_config_map config ### -#################################### - -instance_id: da1a30622aec40d9f1a9f1333c2b845f12ec456d - -# Fetch trading pairs from all exchanges if True, otherwise fetch only from connected exchanges. -fetch_pairs_from_all_exchanges: false - -log_level: INFO - -debug_console: false - -strategy_report_interval: 900.0 - -logger_override_whitelist: -- hummingbot.strategy.arbitrage -- hummingbot.strategy.cross_exchange_market_making -- conf - -log_file_path: /backend-api/logs - -kill_switch_mode: {} - -# What to auto-fill in the prompt after each import command (start/config) -autofill_import: disabled - -telegram_mode: {} - -# MQTT Bridge configuration. -mqtt_bridge: - mqtt_host: localhost - mqtt_port: 1883 - mqtt_username: '' - mqtt_password: '' - mqtt_namespace: hbot - mqtt_ssl: false - mqtt_logger: true - mqtt_notifier: true - mqtt_commands: true - mqtt_events: true - mqtt_external_events: true - mqtt_autostart: true - -# Error log sharing -send_error_logs: true - -# Can store the previous strategy ran for quick retrieval. -previous_strategy: some-strategy.yml - -# Advanced database options, currently supports SQLAlchemy's included dialects -# Reference: https://docs.sqlalchemy.org/en/13/dialects/ -# To use an instance of SQLite DB the required configuration is -# db_engine: sqlite -# To use a DBMS the required configuration is -# db_host: 127.0.0.1 -# db_port: 3306 -# db_username: username -# db_password: password -# db_name: dbname -db_mode: - db_engine: sqlite - -# Balance Limit Configurations -# e.g. Setting USDT and BTC limits on Binance. -# balance_asset_limit: -# binance: -# BTC: 0.1 -# USDT: 1000 -balance_asset_limit: - kucoin: {} - ndax_testnet: {} - huobi: {} - bitmart: {} - polkadex: {} - hitbtc: {} - ndax: {} - foxbit: {} - bitmex_testnet: {} - coinbase_pro: {} - bybit: {} - binance_paper_trade: {} - bitfinex: {} - kucoin_paper_trade: {} - okx: {} - binance_us: {} - injective_v2: {} - ascend_ex: {} - binance: {} - bybit_testnet: {} - kraken: {} - mexc: {} - vertex: {} - gate_io_paper_trade: {} - gate_io: {} - woo_x: {} - woo_x_testnet: {} - btc_markets: {} - vertex_testnet: {} - mock_paper_exchange: {} - bitmex: {} - ascend_ex_paper_trade: {} - -# Fixed gas price (in Gwei) for Ethereum transactions -manual_gas_price: 50.0 - -# Gateway API Configurations -# default host to only use localhost -# Port need to match the final installation port for Gateway -gateway: - gateway_api_host: localhost - gateway_api_port: '15888' - -certs_path: /certs - -# Whether to enable aggregated order and trade data collection -anonymized_metrics_mode: - anonymized_metrics_interval_min: 15.0 - -# Command Shortcuts -# Define abbreviations for often used commands -# or batch grouped commands together -command_shortcuts: -- command: spreads - help: Set bid and ask spread - arguments: - - Bid Spread - - Ask Spread - output: - - config bid_spread $1 - - config ask_spread $2 - -# A source for rate oracle, currently ascend_ex, binance, binance_us, coin_gecko, coin_cap, kucoin, gate_io, coinbase_advanced_trade, cube -rate_oracle_source: - name: binance - -# A universal token which to display tokens values in, e.g. USD,EUR,BTC -global_token: - global_token_name: USDT - global_token_symbol: $ - -# Percentage of API rate limits (on any exchange and any end point) allocated to this bot instance. -# Enter 50 to indicate 50%. E.g. if the API rate limit is 100 calls per second, and you allocate -# 50% to this setting, the bot will have a maximum (limit) of 50 calls per second -rate_limits_share_pct: 100.0 - -commands_timeout: - create_command_timeout: 10.0 - other_commands_timeout: 30.0 - -# Tabulate table format style (https://github.com/astanin/python-tabulate#table-format) -tables_format: psql - -paper_trade: - paper_trade_exchanges: - - binance - - kucoin - - ascend_ex - - gate_io - paper_trade_account_balance: - BTC: 1.0 - USDT: 1000.0 - ONE: 1000.0 - USDQ: 1000.0 - TUSD: 1000.0 - ETH: 10.0 - WETH: 10.0 - USDC: 1000.0 - DAI: 1000.0 - -color: - top_pane: '#000000' - bottom_pane: '#000000' - output_pane: '#262626' - input_pane: '#1C1C1C' - logs_pane: '#121212' - terminal_primary: '#5FFFD7' - primary_label: '#5FFFD7' - secondary_label: '#FFFFFF' - success_label: '#5FFFD7' - warning_label: '#FFFF00' - info_label: '#5FD7FF' - error_label: '#FF0000' - gold_label: '#FFD700' - silver_label: '#C0C0C0' - bronze_label: '#CD7F32' - -# The tick size is the frequency with which the clock notifies the time iterators by calling the -# c_tick() method, that means for example that if the tick size is 1, the logic of the strategy -# will run every second. -tick_size: 1.0 - -market_data_collection: - market_data_collection_enabled: true - market_data_collection_interval: 60 - market_data_collection_depth: 20 diff --git a/conf/conf_client.yml b/conf/conf_client.yml index 6f81169..dd8e81e 100644 --- a/conf/conf_client.yml +++ b/conf/conf_client.yml @@ -107,7 +107,7 @@ gateway: gateway_api_host: localhost gateway_api_port: '15888' -certs_path: /backend-api/certs +certs_path: ./certs # Whether to enable aggregated order and trade data collection anonymized_metrics_mode: diff --git a/conf/conf_fee_overrides copy.yml b/conf/conf_fee_overrides copy.yml deleted file mode 100644 index 1dbff8f..0000000 --- a/conf/conf_fee_overrides copy.yml +++ /dev/null @@ -1,298 +0,0 @@ -######################################## -### Fee overrides configurations ### -######################################## - -# For more detailed information: https://docs.hummingbot.io -template_version: 14 - -# Example of the fields that can be specified to override the `TradeFeeFactory` default settings. -# If the field is missing or the value is left blank, the default value will be used. -# The percentage values are specified as 0.1 for 0.1%. -# -# [exchange name]_percent_fee_token: -# [exchange name]_maker_percent_fee: -# [exchange name]_taker_percent_fee: -# [exchange name]_buy_percent_fee_deducted_from_returns: # if False, the buy fee is added to the order costs -# [exchange name]_maker_fixed_fees: # a list of lists of token-fee pairs (e.g. [["ETH", 1]]) -# [exchange name]_taker_fixed_fees: # a list of lists of token-fee pairs (e.g. [["ETH", 1]]) - -binance_percent_fee_token: # BNB -binance_maker_percent_fee: # 0.75 -binance_taker_percent_fee: # 0.75 -binance_buy_percent_fee_deducted_from_returns: # True - -# List of supported Exchanges for which the user's conf/conf_fee_override.yml -# will work. This file currently needs to be in sync with hummingbot list of -# supported exchanges -ascend_ex_buy_percent_fee_deducted_from_returns: -ascend_ex_maker_fixed_fees: -ascend_ex_maker_percent_fee: -ascend_ex_percent_fee_token: -ascend_ex_taker_fixed_fees: -ascend_ex_taker_percent_fee: -binance_maker_fixed_fees: -binance_perpetual_buy_percent_fee_deducted_from_returns: -binance_perpetual_maker_fixed_fees: -binance_perpetual_maker_percent_fee: -binance_perpetual_percent_fee_token: -binance_perpetual_taker_fixed_fees: -binance_perpetual_taker_percent_fee: -binance_perpetual_testnet_buy_percent_fee_deducted_from_returns: -binance_perpetual_testnet_maker_fixed_fees: -binance_perpetual_testnet_maker_percent_fee: -binance_perpetual_testnet_percent_fee_token: -binance_perpetual_testnet_taker_fixed_fees: -binance_perpetual_testnet_taker_percent_fee: -binance_taker_fixed_fees: -binance_us_buy_percent_fee_deducted_from_returns: -binance_us_maker_fixed_fees: -binance_us_maker_percent_fee: -binance_us_percent_fee_token: -binance_us_taker_fixed_fees: -binance_us_taker_percent_fee: -bitfinex_buy_percent_fee_deducted_from_returns: -bitfinex_maker_fixed_fees: -bitfinex_maker_percent_fee: -bitfinex_percent_fee_token: -bitfinex_taker_fixed_fees: -bitfinex_taker_percent_fee: -bitmart_buy_percent_fee_deducted_from_returns: -bitmart_maker_fixed_fees: -bitmart_maker_percent_fee: -bitmart_percent_fee_token: -bitmart_taker_fixed_fees: -bitmart_taker_percent_fee: -btc_markets_percent_fee_token: -btc_markets_maker_percent_fee: -btc_markets_taker_percent_fee: -btc_markets_buy_percent_fee_deducted_from_returns: -bybit_perpetual_buy_percent_fee_deducted_from_returns: -bybit_perpetual_maker_fixed_fees: -bybit_perpetual_maker_percent_fee: -bybit_perpetual_percent_fee_token: -bybit_perpetual_taker_fixed_fees: -bybit_perpetual_taker_percent_fee: -bybit_perpetual_testnet_buy_percent_fee_deducted_from_returns: -bybit_perpetual_testnet_maker_fixed_fees: -bybit_perpetual_testnet_maker_percent_fee: -bybit_perpetual_testnet_percent_fee_token: -bybit_perpetual_testnet_taker_fixed_fees: -bybit_perpetual_testnet_taker_percent_fee: -coinbase_pro_buy_percent_fee_deducted_from_returns: -coinbase_pro_maker_fixed_fees: -coinbase_pro_maker_percent_fee: -coinbase_pro_percent_fee_token: -coinbase_pro_taker_fixed_fees: -coinbase_pro_taker_percent_fee: -dydx_perpetual_buy_percent_fee_deducted_from_returns: -dydx_perpetual_maker_fixed_fees: -dydx_perpetual_maker_percent_fee: -dydx_perpetual_percent_fee_token: -dydx_perpetual_taker_fixed_fees: -dydx_perpetual_taker_percent_fee: -gate_io_buy_percent_fee_deducted_from_returns: -gate_io_maker_fixed_fees: -gate_io_maker_percent_fee: -gate_io_percent_fee_token: -gate_io_taker_fixed_fees: -gate_io_taker_percent_fee: -hitbtc_buy_percent_fee_deducted_from_returns: -hitbtc_maker_fixed_fees: -hitbtc_maker_percent_fee: -hitbtc_percent_fee_token: -hitbtc_taker_fixed_fees: -hitbtc_taker_percent_fee: -huobi_buy_percent_fee_deducted_from_returns: -huobi_maker_fixed_fees: -huobi_maker_percent_fee: -huobi_percent_fee_token: -huobi_taker_fixed_fees: -huobi_taker_percent_fee: -kraken_buy_percent_fee_deducted_from_returns: -kraken_maker_fixed_fees: -kraken_maker_percent_fee: -kraken_percent_fee_token: -kraken_taker_fixed_fees: -kraken_taker_percent_fee: -kucoin_buy_percent_fee_deducted_from_returns: -kucoin_maker_fixed_fees: -kucoin_maker_percent_fee: -kucoin_percent_fee_token: -kucoin_taker_fixed_fees: -kucoin_taker_percent_fee: -mexc_buy_percent_fee_deducted_from_returns: -mexc_maker_fixed_fees: -mexc_maker_percent_fee: -mexc_percent_fee_token: -mexc_taker_fixed_fees: -mexc_taker_percent_fee: -ndax_buy_percent_fee_deducted_from_returns: -ndax_maker_fixed_fees: -ndax_maker_percent_fee: -ndax_percent_fee_token: -ndax_taker_fixed_fees: -ndax_taker_percent_fee: -ndax_testnet_buy_percent_fee_deducted_from_returns: -ndax_testnet_maker_fixed_fees: -ndax_testnet_maker_percent_fee: -ndax_testnet_percent_fee_token: -ndax_testnet_taker_fixed_fees: -ndax_testnet_taker_percent_fee: -okx_buy_percent_fee_deducted_from_returns: -okx_maker_fixed_fees: -okx_maker_percent_fee: -okx_percent_fee_token: -okx_taker_fixed_fees: -okx_taker_percent_fee: -phemex_perpetual_percent_fee_token: -phemex_perpetual_maker_percent_fee: -phemex_perpetual_taker_percent_fee: -phemex_perpetual_buy_percent_fee_deducted_from_returns: -phemex_perpetual_maker_fixed_fees: -phemex_perpetual_taker_fixed_fees: -phemex_perpetual_testnet_percent_fee_token: -phemex_perpetual_testnet_maker_percent_fee: -phemex_perpetual_testnet_taker_percent_fee: -phemex_perpetual_testnet_buy_percent_fee_deducted_from_returns: -phemex_perpetual_testnet_maker_fixed_fees: -phemex_perpetual_testnet_taker_fixed_fees: -injective_v2_perpetual_percent_fee_token: -injective_v2_perpetual_maker_percent_fee: -injective_v2_perpetual_taker_percent_fee: -injective_v2_perpetual_buy_percent_fee_deducted_from_returns: -injective_v2_perpetual_maker_fixed_fees: -injective_v2_perpetual_taker_fixed_fees: -bitget_perpetual_percent_fee_token: -bitget_perpetual_maker_percent_fee: -bitget_perpetual_taker_percent_fee: -bitget_perpetual_buy_percent_fee_deducted_from_returns: -bitget_perpetual_maker_fixed_fees: -bitget_perpetual_taker_fixed_fees: -bit_com_perpetual_percent_fee_token: -bit_com_perpetual_maker_percent_fee: -bit_com_perpetual_taker_percent_fee: -bit_com_perpetual_buy_percent_fee_deducted_from_returns: -bit_com_perpetual_maker_fixed_fees: -bit_com_perpetual_taker_fixed_fees: -bit_com_perpetual_testnet_percent_fee_token: -bit_com_perpetual_testnet_maker_percent_fee: -bit_com_perpetual_testnet_taker_percent_fee: -bit_com_perpetual_testnet_buy_percent_fee_deducted_from_returns: -bit_com_perpetual_testnet_maker_fixed_fees: -bit_com_perpetual_testnet_taker_fixed_fees: -gate_io_perpetual_percent_fee_token: -gate_io_perpetual_maker_percent_fee: -gate_io_perpetual_taker_percent_fee: -gate_io_perpetual_buy_percent_fee_deducted_from_returns: -gate_io_perpetual_maker_fixed_fees: -gate_io_perpetual_taker_fixed_fees: -bitmex_perpetual_percent_fee_token: -bitmex_perpetual_maker_percent_fee: -bitmex_perpetual_taker_percent_fee: -bitmex_perpetual_buy_percent_fee_deducted_from_returns: -bitmex_perpetual_maker_fixed_fees: -bitmex_perpetual_taker_fixed_fees: -bitmex_perpetual_testnet_percent_fee_token: -bitmex_perpetual_testnet_maker_percent_fee: -bitmex_perpetual_testnet_taker_percent_fee: -bitmex_perpetual_testnet_buy_percent_fee_deducted_from_returns: -bitmex_perpetual_testnet_maker_fixed_fees: -bitmex_perpetual_testnet_taker_fixed_fees: -kucoin_perpetual_percent_fee_token: -kucoin_perpetual_maker_percent_fee: -kucoin_perpetual_taker_percent_fee: -kucoin_perpetual_buy_percent_fee_deducted_from_returns: -kucoin_perpetual_maker_fixed_fees: -kucoin_perpetual_taker_fixed_fees: -bitmex_percent_fee_token: -bitmex_maker_percent_fee: -bitmex_taker_percent_fee: -bitmex_buy_percent_fee_deducted_from_returns: -bitmex_maker_fixed_fees: -bitmex_taker_fixed_fees: -bitmex_testnet_percent_fee_token: -bitmex_testnet_maker_percent_fee: -bitmex_testnet_taker_percent_fee: -bitmex_testnet_buy_percent_fee_deducted_from_returns: -bitmex_testnet_maker_fixed_fees: -bitmex_testnet_taker_fixed_fees: -btc_markets_maker_fixed_fees: -btc_markets_taker_fixed_fees: -woo_x_percent_fee_token: -woo_x_maker_percent_fee: -woo_x_taker_percent_fee: -woo_x_buy_percent_fee_deducted_from_returns: -woo_x_maker_fixed_fees: -woo_x_taker_fixed_fees: -woo_x_testnet_percent_fee_token: -woo_x_testnet_maker_percent_fee: -woo_x_testnet_taker_percent_fee: -woo_x_testnet_buy_percent_fee_deducted_from_returns: -woo_x_testnet_maker_fixed_fees: -woo_x_testnet_taker_fixed_fees: -polkadex_percent_fee_token: -polkadex_maker_percent_fee: -polkadex_taker_percent_fee: -polkadex_buy_percent_fee_deducted_from_returns: -polkadex_maker_fixed_fees: -polkadex_taker_fixed_fees: -bybit_percent_fee_token: -bybit_maker_percent_fee: -bybit_taker_percent_fee: -bybit_buy_percent_fee_deducted_from_returns: -bybit_maker_fixed_fees: -bybit_taker_fixed_fees: -bybit_testnet_percent_fee_token: -bybit_testnet_maker_percent_fee: -bybit_testnet_taker_percent_fee: -bybit_testnet_buy_percent_fee_deducted_from_returns: -bybit_testnet_maker_fixed_fees: -bybit_testnet_taker_fixed_fees: -injective_v2_percent_fee_token: -injective_v2_maker_percent_fee: -injective_v2_taker_percent_fee: -injective_v2_buy_percent_fee_deducted_from_returns: -injective_v2_maker_fixed_fees: -injective_v2_taker_fixed_fees: -vertex_percent_fee_token: -vertex_maker_percent_fee: -vertex_taker_percent_fee: -vertex_buy_percent_fee_deducted_from_returns: -vertex_maker_fixed_fees: -vertex_taker_fixed_fees: -vertex_testnet_percent_fee_token: -vertex_testnet_maker_percent_fee: -vertex_testnet_taker_percent_fee: -vertex_testnet_buy_percent_fee_deducted_from_returns: -vertex_testnet_maker_fixed_fees: -vertex_testnet_taker_fixed_fees: -mock_paper_exchange_percent_fee_token: -mock_paper_exchange_maker_percent_fee: -mock_paper_exchange_taker_percent_fee: -mock_paper_exchange_buy_percent_fee_deducted_from_returns: -mock_paper_exchange_maker_fixed_fees: -mock_paper_exchange_taker_fixed_fees: -kucoin_perpetual_testnet_percent_fee_token: -kucoin_perpetual_testnet_maker_percent_fee: -kucoin_perpetual_testnet_taker_percent_fee: -kucoin_perpetual_testnet_buy_percent_fee_deducted_from_returns: -kucoin_perpetual_testnet_maker_fixed_fees: -kucoin_perpetual_testnet_taker_fixed_fees: -foxbit_percent_fee_token: -foxbit_maker_percent_fee: -foxbit_taker_percent_fee: -foxbit_buy_percent_fee_deducted_from_returns: -foxbit_maker_fixed_fees: -foxbit_taker_fixed_fees: -hyperliquid_perpetual_percent_fee_token: -hyperliquid_perpetual_maker_percent_fee: -hyperliquid_perpetual_taker_percent_fee: -hyperliquid_perpetual_buy_percent_fee_deducted_from_returns: -hyperliquid_perpetual_maker_fixed_fees: -hyperliquid_perpetual_taker_fixed_fees: -hyperliquid_perpetual_testnet_percent_fee_token: -hyperliquid_perpetual_testnet_maker_percent_fee: -hyperliquid_perpetual_testnet_taker_percent_fee: -hyperliquid_perpetual_testnet_buy_percent_fee_deducted_from_returns: -hyperliquid_perpetual_testnet_maker_fixed_fees: -hyperliquid_perpetual_testnet_taker_fixed_fees: diff --git a/conf/hummingbot_logs copy.yml b/conf/hummingbot_logs copy.yml deleted file mode 100755 index 8e65271..0000000 --- a/conf/hummingbot_logs copy.yml +++ /dev/null @@ -1,83 +0,0 @@ ---- -version: 1 -template_version: 12 - -formatters: - simple: - format: "%(asctime)s - %(process)d - %(name)s - %(levelname)s - %(message)s" - -handlers: - console: - class: hummingbot.logger.cli_handler.CLIHandler - level: DEBUG - formatter: simple - stream: ext://sys.stdout - console_warning: - class: hummingbot.logger.cli_handler.CLIHandler - level: WARNING - formatter: simple - stream: ext://sys.stdout - console_info: - class: hummingbot.logger.cli_handler.CLIHandler - level: INFO - formatter: simple - stream: ext://sys.stdout - file_handler: - class: logging.handlers.TimedRotatingFileHandler - level: DEBUG - formatter: simple - filename: $PROJECT_DIR/logs/logs_$STRATEGY_FILE_PATH.log - encoding: utf8 - when: "D" - interval: 1 - backupCount: 7 - "null": - class: logging.NullHandler - level: DEBUG - -loggers: - hummingbot.core.utils.eth_gas_station_lookup: - level: NETWORK - propagate: false - handlers: [console, file_handler] - mqtt: true - hummingbot.logger.log_server_client: - level: WARNING - propagate: false - handlers: [console, file_handler] - mqtt: true - hummingbot.logger.reporting_proxy_handler: - level: WARNING - propagate: false - handlers: [console, file_handler] - mqtt: true - hummingbot.strategy: - level: NETWORK - propagate: false - handlers: [console, file_handler] - mqtt: true - hummingbot.connector: - level: NETWORK - propagate: false - handlers: [console, file_handler] - mqtt: true - hummingbot.client: - level: NETWORK - propagate: false - handlers: [console, file_handler] - mqtt: true - hummingbot.core.event.event_reporter: - level: EVENT_LOG - propagate: false - handlers: [file_handler] - mqtt: false - conf: - level: NETWORK - handlers: ["null"] - propagate: false - mqtt: false - -root: - level: INFO - handlers: [console, file_handler] - mqtt: true diff --git a/conf/mango.yml b/conf/mango.yml new file mode 100644 index 0000000..f34d3b3 --- /dev/null +++ b/conf/mango.yml @@ -0,0 +1,2 @@ +defaultGroup: 78b8f4cGCwmZ9ysPFMWLaLTkkaYnUjwMJYStWe5RTSSX +tradeHistoryApiUrl: https://api.mngo.cloud/data/v4/stats/trade-history \ No newline at end of file diff --git a/conf/root.yml b/conf/root.yml index f62e5c2..41454a5 100644 --- a/conf/root.yml +++ b/conf/root.yml @@ -139,3 +139,11 @@ configurations: $namespace telos: configurationPath: telos.yml schemaPath: ethereum-schema.json + + $namespace solana: + configurationPath: solana.yml + schemaPath: solana-schema.json + + $namespace mango: + configurationPath: mango.yml + schemaPath: mango-schema.json \ No newline at end of file diff --git a/conf/server.yml b/conf/server.yml index 5f348a5..b468aa2 100644 --- a/conf/server.yml +++ b/conf/server.yml @@ -1,33 +1,11 @@ -# Path to folder where Hummingbot generates self-signed certificates -certificatePath: ./certs/ - -# Path to folder where logs will be stored. -logPath: './logs' - -# Port to expose the gateway server on +certificatePath: ./certs +logPath: ./logs port: 15888 - -# IPs allowed to access gateway. localhost is allowed by default. ipWhitelist: [] - -# GMT Offset -GMTOffset: +0800 - -# If true, logs will be stored in logPath and printed to stdout. If false, they -# will only be stored in logPath and not printed to stdout. +GMTOffset: 800 logToStdOut: true - -# Runs Gateway in unsafe dev mode with HTTP if true unsafeDevModeWithHTTP: false - -# Collects data about API usage if true telemetry_enabled: false - -# Nonce database -nonceDbPath: 'nonce.level' - -# Transaction database -transactionDbPath: 'transaction.level' - - - +nonceDbPath: nonce.level +transactionDbPath: transaction.level +id: 4457e1ff63912 diff --git a/conf/solana.yml b/conf/solana.yml new file mode 100644 index 0000000..faa15ca --- /dev/null +++ b/conf/solana.yml @@ -0,0 +1,24 @@ +networks: + mainnet-beta: + nodeURL: https://api.mainnet-beta.solana.com + tokenListType: 'FILE' + tokenListSource: 'conf/lists/solana_tokens.json' + nativeCurrencySymbol: SOL + maxLRUCacheInstances: 10 + testnet: + nodeURL: https://api.testnet.solana.com + tokenListType: 'FILE' + tokenListSource: 'conf/lists/solana_tokens.json' + nativeCurrencySymbol: SOL + maxLRUCacheInstances: 10 + devnet: + nodeURL: https://api.devnet.solana.com + tokenListType: 'FILE' + tokenListSource: 'conf/lists/solana_tokens.json' + nativeCurrencySymbol: SOL + maxLRUCacheInstances: 10 + +tokenProgram: TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA +transactionLamports: 5000 +lamportsToSol: 0.000000001 +timeToLive: 75 \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index b8f233f..5857e92 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -27,28 +27,26 @@ services: # Add these new environment variables for Hummingbot instances - GATEWAY_HOST=gateway - GATEWAY_PORT=15888 - - GATEWAY_CERTS_PATH=/certs + - GATEWAY_CERTS_PATH=./certs - HUMMINGBOT_INSTANCE_TYPE=docker networks: - emqx-bridge - gateway-network gateway: - image: mlguys/gateway:mango-markets-latest + image: hummingbot/gateway:mango-markets-latest restart: unless-stopped ports: - "15888:15888" - "8080:8080" volumes: - "./certs:/home/gateway/certs:ro" + - "./conf:/home/gateway/conf" environment: - - GATEWAY_CERT=/home/gateway/certs/gateway_cert.pem - - GATEWAY_KEY=/home/gateway/certs/gateway_key.pem - GATEWAY_PASSPHRASE=${GATEWAY_CERT_PASSPHRASE} - - CERTS_PATH=/home/gateway/certs networks: - gateway-network - + transaction-service: image: ricardocr987/mango-server:latest volumes: diff --git a/routers/instances.py b/routers/instances.py index e824d6e..432f22b 100644 --- a/routers/instances.py +++ b/routers/instances.py @@ -29,7 +29,7 @@ async def create_instance(request: CreateBotRequest, wallet_auth: JWTWalletAuthD strategy_name=request.strategy_name, parameters=request.strategy_parameters, market=request.market, - wallet_address=wallet_auth.address + wallet_address=wallet_auth.address, ) accounts_service.save_bot_config(bot_account, bot_config) # Create Hummingbot instance @@ -99,9 +99,7 @@ async def stop_instance(instance_id: str, wallet_auth: JWTWalletAuthDep): # Cancel all orders through the gateway bot_wallet = accounts_service.get_bot_wallet_address(instance_id) - cancel_orders_response = await gateway_client.clob_perp_get_orders( - "solana", "mainnet", "mango_perpetual_solana_mainnet-beta" - ) + cancel_orders_response = await gateway_client.clob_perp_get_orders("solana", "mainnet", "mango_perpetual_solana_mainnet-beta") if not cancel_orders_response["success"]: raise HTTPException(status_code=500, detail="Failed to cancel all orders") diff --git a/services/accounts_service.py b/services/accounts_service.py index 47a02c3..b88fc2c 100644 --- a/services/accounts_service.py +++ b/services/accounts_service.py @@ -9,7 +9,12 @@ from fastapi import HTTPException from hummingbot.client.config.client_config_map import ClientConfigMap from hummingbot.client.config.config_crypt import ETHKeyFileSecretManger -from hummingbot.client.config.config_helpers import ClientConfigAdapter, ReadOnlyClientConfigAdapter, get_connector_class, load_client_config_map_from_file +from hummingbot.client.config.config_helpers import ( + ClientConfigAdapter, + ReadOnlyClientConfigAdapter, + get_connector_class, + load_client_config_map_from_file, +) from hummingbot.client.settings import AllConnectorSettings, CONF_DIR_PATH from config import BANNED_TOKENS, CONFIG_PASSWORD @@ -27,12 +32,14 @@ file_system = FileSystemUtil() + class BotConfig(BaseModel): strategy_name: str parameters: dict market: str wallet_address: str + class AccountsService: """ This class is responsible for managing all the accounts that are connected to the trading system. It is responsible @@ -40,11 +47,13 @@ class AccountsService: update the balances of each account. """ - def __init__(self, - update_account_state_interval_minutes: int = 1, - default_quote: str = "USDC", - account_history_file: str = "account_state_history.json", - account_history_dump_interval_minutes: int = 1): + def __init__( + self, + update_account_state_interval_minutes: int = 1, + default_quote: str = "USDC", + account_history_file: str = "account_state_history.json", + account_history_dump_interval_minutes: int = 1, + ): # TODO: Add database to store the balances of each account each time it is updated. print("conf passw", CONFIG_PASSWORD) print("conf path", CONF_DIR_PATH) @@ -134,10 +143,8 @@ async def dump_account_state_loop(self): logging.error(f"Error dumping account state: {e}") finally: now = datetime.now() - next_log_time = (now + timedelta(minutes=self.account_history_dump_interval)).replace(second=0, - microsecond=0) - next_log_time = next_log_time - timedelta( - minutes=next_log_time.minute % self.account_history_dump_interval) + next_log_time = (now + timedelta(minutes=self.account_history_dump_interval)).replace(second=0, microsecond=0) + next_log_time = next_log_time - timedelta(minutes=next_log_time.minute % self.account_history_dump_interval) sleep_duration = (next_log_time - now).total_seconds() await asyncio.sleep(sleep_duration) @@ -258,8 +265,11 @@ async def update_account_state(self): for connector_name, connector in connectors.items(): tokens_info = [] try: - balances = [{"token": key, "units": value} for key, value in connector.get_all_balances().items() if - value != Decimal("0") and key not in BANNED_TOKENS] + balances = [ + {"token": key, "units": value} + for key, value in connector.get_all_balances().items() + if value != Decimal("0") and key not in BANNED_TOKENS + ] unique_tokens = [balance["token"] for balance in balances] trading_pairs = [self.get_default_market(token) for token in unique_tokens if "USD" not in token] last_traded_prices = await self._safe_get_last_traded_prices(connector, trading_pairs) @@ -270,17 +280,18 @@ async def update_account_state(self): else: market = self.get_default_market(balance["token"]) price = Decimal(last_traded_prices.get(market, 0)) - tokens_info.append({ - "token": balance["token"], - "units": float(balance["units"]), - "price": float(price), - "value": float(price * balance["units"]), - "available_units": float(connector.get_available_balance(balance["token"])) - }) + tokens_info.append( + { + "token": balance["token"], + "units": float(balance["units"]), + "price": float(price), + "value": float(price * balance["units"]), + "available_units": float(connector.get_available_balance(balance["token"])), + } + ) self.account_state_update_event.set() except Exception as e: - logging.error( - f"Error updating balances for connector {connector_name} in account {account_name}: {e}") + logging.error(f"Error updating balances for connector {connector_name} in account {account_name}: {e}") self.accounts_state[account_name][connector_name] = tokens_info async def _safe_get_last_traded_prices(self, connector, trading_pairs, timeout=5): @@ -340,16 +351,16 @@ def get_connector(self, account_name: str, connector_name: str): connector_class = get_connector_class(connector_name) connector = connector_class(**init_params) return connector - + @staticmethod def list_accounts(): """ List all the accounts that are connected to the trading system. :return: List of accounts. """ - if not file_system.path_exists('credentials'): + if not file_system.path_exists("credentials"): return [] - return file_system.list_folders('credentials') + return file_system.list_folders("credentials") def list_credentials(self, account_name: str): """ @@ -358,8 +369,7 @@ def list_credentials(self, account_name: str): :return: List of credentials. """ try: - return [file for file in file_system.list_files(f'credentials/{account_name}/connectors') if - file.endswith('.yml')] + return [file for file in file_system.list_files(f"credentials/{account_name}/connectors") if file.endswith(".yml")] except FileNotFoundError as e: raise HTTPException(status_code=404, detail=str(e)) @@ -386,8 +396,8 @@ def add_account(self, account_name: str): if account_name in self.accounts: raise HTTPException(status_code=400, detail="Account already exists.") files_to_copy = ["conf_client.yml", "conf_fee_overrides.yml", "hummingbot_logs.yml", ".password_verification"] - file_system.create_folder('credentials', account_name) - file_system.create_folder(f'credentials/{account_name}', "connectors") + file_system.create_folder("credentials", account_name) + file_system.create_folder(f"credentials/{account_name}", "connectors") for file in files_to_copy: file_system.copy_file(f"credentials/master_account/{file}", f"credentials/{account_name}/{file}") self.accounts[account_name] = {"wallet": None} @@ -399,7 +409,7 @@ def delete_account(self, account_name: str): :param account_name: :return: """ - file_system.delete_folder('credentials', account_name) + file_system.delete_folder("credentials", account_name) self.accounts.pop(account_name) self.accounts_state.pop(account_name) @@ -415,18 +425,18 @@ async def generate_bot_wallet(self, account_name: str) -> str: def _save_private_key(self, account_name: str, wallet_address: str, private_key: str): # Encrypt the private key encrypted_private_key = self._encrypt_private_key(private_key) - + # Prepare the file path credentials_dir = os.path.join("bots", "credentials", account_name) os.makedirs(credentials_dir, exist_ok=True) wallet_path = os.path.join(credentials_dir, "solana") os.makedirs(wallet_path, exist_ok=True) key_file_path = os.path.join(wallet_path, f"{wallet_address}.json") - + # Save the encrypted private key with open(key_file_path, "w") as f: json.dump(encrypted_private_key, f) - + # Set restrictive permissions on the file os.chmod(key_file_path, 0o600) @@ -439,7 +449,7 @@ def _decrypt_private_key(self, encrypted_private_key: str) -> str: box = SecretBox(self.secret_key) decrypted = box.decrypt(base64.b64decode(encrypted_private_key)) return decrypted.decode() - + def get_gateway_client(self, account_name: Optional[str] = None): config_map = load_client_config_map_from_file() if account_name is None: @@ -452,19 +462,16 @@ def get_gateway_client(self, account_name: Optional[str] = None): async def _add_wallet_to_gateway(self, account_name: str, wallet_address: str, private_key: str): # TODO: FIX the /backend/api/certs/ca_cert.pem path gateway_client = self.get_gateway_client(account_name) - + # Assuming there's a method to add a Solana wallet to the gateway # You may need to adjust this based on the actual Gateway API response = await gateway_client.add_wallet( - chain="solana", - network="mainnet", - private_key=private_key, - address=wallet_address + chain="solana", network="mainnet", private_key=private_key, address=wallet_address ) - + if not response["success"]: raise Exception(f"Failed to add wallet to gateway: {response['message']}") - + def _add_wallet_to_account(self, account_name: str, wallet_address: str): if account_name not in self.accounts: raise ValueError(f"Account {account_name} does not exist.") @@ -485,16 +492,16 @@ def get_bot_wallet_address(self, account_name: str) -> str: self.accounts[account_name] = account_info except FileNotFoundError: raise ValueError(f"No wallet found for bot account: {account_name}") - + return self.accounts[account_name]["wallet"] def get_bot_wallet_private_key(self, credentials_profile: str, wallet_address: str) -> str: wallet_path = os.path.join("bots", "credentials", credentials_profile, "solana") key_file_path = os.path.join(wallet_path, f"{wallet_address}.json") - + with open(key_file_path, "r") as f: encrypted_private_key = json.load(f) - + return self._decrypt_private_key(encrypted_private_key) def save_bot_config(self, account_name: str, config: BotConfig): @@ -513,4 +520,4 @@ def get_bot_config(self, account_name: str) -> BotConfig: def get_bot_market(self, account_name: str): config = self.get_bot_config(account_name) - return config.market \ No newline at end of file + return config.market diff --git a/services/docker_service.py b/services/docker_service.py index 761ef4a..a988941 100644 --- a/services/docker_service.py +++ b/services/docker_service.py @@ -23,9 +23,11 @@ def __init__(self): def get_active_containers(self): try: - containers_info = [{"id": container.id, "name": container.name, "status": container.status} for - container in self.client.containers.list(filters={"status": "running"}) if - "hummingbot" in container.name and "broker" not in container.name] + containers_info = [ + {"id": container.id, "name": container.name, "status": container.status} + for container in self.client.containers.list(filters={"status": "running"}) + if "hummingbot" in container.name and "broker" not in container.name + ] return {"active_instances": containers_info} except DockerException as e: return str(e) @@ -45,9 +47,11 @@ def pull_image(self, image_name): def get_exited_containers(self): try: - containers_info = [{"id": container.id, "name": container.name, "status": container.status} for - container in self.client.containers.list(filters={"status": "exited"}) if - "hummingbot" in container.name and "broker" not in container.name] + containers_info = [ + {"id": container.id, "name": container.name, "status": container.status} + for container in self.client.containers.list(filters={"status": "exited"}) + if "hummingbot" in container.name and "broker" not in container.name + ] return {"exited_instances": containers_info} except DockerException as e: return str(e) @@ -88,58 +92,77 @@ def remove_container(self, container_name, force=True): return {"success": False, "message": str(e)} def create_hummingbot_instance(self, config: HummingbotInstanceConfig): - bots_path = os.environ.get('HOST_BOTS_PATH', self.SOURCE_PATH) + bots_path = os.environ.get("HOST_BOTS_PATH", self.SOURCE_PATH) instance_name = f"hummingbot-{config.instance_name}" - instance_dir = os.path.join(bots_path, "bots", 'instances', instance_name) - if not os.path.exists(instance_dir): - os.makedirs(instance_dir) - os.makedirs(os.path.join(instance_dir, 'data')) - os.makedirs(os.path.join(instance_dir, 'logs')) # Copy credentials to instance directory - source_credentials_dir = os.path.join("bots", 'credentials', config.credentials_profile) - script_config_dir = os.path.join("bots", 'conf', 'scripts') - controllers_config_dir = os.path.join("bots", 'conf', 'controllers') - destination_credentials_dir = os.path.join(instance_dir, 'conf') - destination_scripts_config_dir = os.path.join(instance_dir, 'conf', 'scripts') - destination_controllers_config_dir = os.path.join(instance_dir, 'conf', 'controllers') + instance_to_copy_dir = os.path.join("bots", "instances", instance_name) + + if not os.path.exists(instance_to_copy_dir): + os.makedirs(instance_to_copy_dir) + os.makedirs(os.path.join(instance_to_copy_dir, "data")) + os.makedirs(os.path.join(instance_to_copy_dir, "logs")) + + source_credentials_dir = os.path.join("bots", "credentials", config.credentials_profile) + script_config_dir = os.path.join("bots", "conf", "scripts") + controllers_config_dir = os.path.join("bots", "conf", "controllers") + destination_credentials_dir = os.path.join(instance_to_copy_dir, "conf") + destination_scripts_config_dir = os.path.join(instance_to_copy_dir, "conf", "scripts") + destination_controllers_config_dir = os.path.join(instance_to_copy_dir, "conf", "controllers") # Remove the destination directory if it already exists if os.path.exists(destination_credentials_dir): shutil.rmtree(destination_credentials_dir) - # Copy the entire contents of source_credentials_dir to destination_credentials_dir + # Copy the entire contents of source_credentials_dir to destination_credentials_dir shutil.copytree(source_credentials_dir, destination_credentials_dir) shutil.copytree(script_config_dir, destination_scripts_config_dir) shutil.copytree(controllers_config_dir, destination_controllers_config_dir) - conf_file_path = f"{instance_dir}/conf/conf_client.yml" + conf_file_path = f"{instance_to_copy_dir}/conf/conf_client.yml" client_config = FileSystemUtil.read_yaml_file(conf_file_path) - client_config['instance_id'] = instance_name + client_config["instance_id"] = instance_name FileSystemUtil.dump_dict_to_yaml(conf_file_path, client_config) environment = { - "CONFIG_PASSWORD": os.environ.get('CONFIG_PASSWORD'), - "GATEWAY_CERT_PATH": os.environ.get('GATEWAY_CERT_PATH'), - "GATEWAY_CERT_PASSPHRASE": os.environ.get('GATEWAY_CERT_PASSPHRASE'), - "GATEWAY_HOST": os.environ.get('GATEWAY_HOST', 'gateway'), - "GATEWAY_PORT": os.environ.get('GATEWAY_PORT', '15888'), - "CERTS_PATH": os.environ.get('GATEWAY_CERTS_PATH', '/certs'), + "CONFIG_PASSWORD": os.environ.get("CONFIG_PASSWORD"), + "DEFAULT_GATEWAY_CERTS_PATH": os.environ.get("GATEWAY_CERTS_PATH", "./certs"), } - - host_bots_path = os.environ.get('HOST_BOTS_PATH', './bots') - host_gateway_certs_path = os.environ.get('HOST_GATEWAY_CERTS_PATH', './certs') - + + host_gateway_certs_path = os.environ.get("HOST_GATEWAY_CERTS_PATH", "./certs") + + instance_dir = os.path.join(bots_path, "instances", instance_name) + volumes = { - f"{host_bots_path}/instances/{instance_name}": {"bind": "/conf", "mode": "rw"}, - host_gateway_certs_path: {"bind": "/certs", "mode": "ro"}, + host_gateway_certs_path: {"bind": "/home/hummingbot/certs", "mode": "ro"}, + os.path.abspath(os.path.join(instance_dir, "conf")): {"bind": "/home/hummingbot/conf", "mode": "rw"}, + os.path.abspath(os.path.join(instance_dir, "conf", "connectors")): { + "bind": "/home/hummingbot/conf/connectors", + "mode": "rw", + }, + os.path.abspath(os.path.join(instance_dir, "conf", "scripts")): { + "bind": "/home/hummingbot/conf/scripts", + "mode": "rw", + }, + os.path.abspath(os.path.join(instance_dir, "conf", "controllers")): { + "bind": "/home/hummingbot/conf/controllers", + "mode": "rw", + }, + os.path.abspath(os.path.join(instance_dir, "data")): {"bind": "/home/hummingbot/data", "mode": "rw"}, + os.path.abspath(os.path.join(instance_dir, "logs")): {"bind": "/home/hummingbot/logs", "mode": "rw"}, + os.path.abspath(os.path.join(bots_path, "scripts")): {"bind": "/home/hummingbot/scripts", "mode": "rw"}, + os.path.abspath(os.path.join(bots_path, "controllers")): { + "bind": "/home/hummingbot/controllers", + "mode": "rw", + }, } - + log_config = LogConfig( type="json-file", config={ - 'max-size': '10m', - 'max-file': "5", - }) + "max-size": "10m", + "max-file": "5", + }, + ) self.client.containers.run( image=config.image, name=instance_name, @@ -152,12 +175,11 @@ def create_hummingbot_instance(self, config: HummingbotInstanceConfig): log_config=log_config, ) return {"success": True, "message": f"Instance {instance_name} created successfully."} - return {"success": False, "message": str(e)} def setup_hummingbot_config(self): config_dir = "/backend-api/conf" os.makedirs(config_dir, exist_ok=True) - + config_file = os.path.join(config_dir, "conf_client.yml") if not os.path.exists(config_file): with open(config_file, "w") as f: @@ -166,4 +188,4 @@ def setup_hummingbot_config(self): # Add other necessary default configurations # Set the HUMMINGBOT_CONFIG_PATH environment variable - os.environ["HUMMINGBOT_CONFIG_PATH"] = config_dir \ No newline at end of file + os.environ["HUMMINGBOT_CONFIG_PATH"] = config_dir diff --git a/utils/gateway.py b/utils/gateway.py index 987bb52..c520408 100644 --- a/utils/gateway.py +++ b/utils/gateway.py @@ -7,29 +7,30 @@ load_environment_variables() + class CustomGatewayHttpClient: def __init__(self, client_config_map: ClientConfigAdapter, secrets_manager): self.client_config_map = client_config_map self.secrets_manager = secrets_manager self.original_secrets_manager = Security.secrets_manager - + # Get gateway configuration from environment variables self.gateway_host = os.getenv("GATEWAY_HOST", "localhost") self.gateway_port = int(os.getenv("GATEWAY_PORT", "15888")) - self.gateway_certs_path = os.getenv("GATEWAY_CERTS_PATH", "/certs") + self.gateway_certs_path = os.getenv("GATEWAY_CERTS_PATH", "./certs") self.gateway_cert_path = os.path.join(self.gateway_certs_path, "gateway_cert.pem") def __enter__(self): Security.secrets_manager = self.secrets_manager self.client = GatewayHttpClient.get_instance(self.client_config_map) - + self.client.base_url = f"https://{self.gateway_host}:{self.gateway_port}" self.client.certs_path = self.gateway_certs_path self.client.cert_file = self.gateway_cert_path - + # Use SSL verification with the new certificate self.client.ssl = self.gateway_cert_path - + return self.client def __exit__(self, exc_type, exc_val, exc_tb): @@ -39,4 +40,4 @@ async def __aenter__(self): return self.__enter__() async def __aexit__(self, exc_type, exc_val, exc_tb): - self.__exit__(exc_type, exc_val, exc_tb) \ No newline at end of file + self.__exit__(exc_type, exc_val, exc_tb)