Initial commit

.dockerignore

+node_modules
+.git
+private-tests
\ No newline at end of file

.gitignore

+db-scripts/updates/
+keys/index.js
+keys/sslcert/
+node_modules/
+private-tests/
+*.log
+package-lock.json

README.md

+# Samourai Dojo
+
+Samourai Dojo is the backing server for Samourai Wallet. Provides HD account & loose addresses (BIP47) balances & transactions lists. Provides unspent output lists to the wallet. PushTX endpoint broadcasts transactions through the backing bitcoind node.
+
+[View API documentation](../master/doc/README.md)
+
+
+## Installation ##
+
+### MyDojo (installation with Docker and Docker Compose)
+
+This setup is recommended to Samourai users who feel comfortable with a few command lines.
+
+It provides in a single command the setup of a full Samourai backend composed of:
+
+* a bitcoin full node only accessible as an ephemeral Tor hidden service,
+* the backend database,
+* the backend modules with an API accessible as a static Tor hidden service,
+* a maintenance tool accessible through a Tor web browser.
+
+See [the documentation](./doc/DOCKER_setup.md) for detailed setup instructions.
+
+
+### Manual installation (developers only)
+
+A full manual setup isn't recommended if you don't intend to install a local development environment.
+
+
+## Theory of Operation
+
+Tracking wallet balances via `xpub` requires conforming to [BIP44](https://github.com/bitcoin/bips/blob/master/bip-0044.mediawiki), [BIP49](https://github.com/bitcoin/bips/blob/master/bip-0049.mediawiki) or [BIP84](https://github.com/bitcoin/bips/blob/master/bip-0084.mediawiki) address derivation scheme. Public keys received by Dojo correspond to single accounts and derive all addresses in the account and change chains. These addresses are at `M/0/x` and `M/1/y`, respectively.
+
+Dojo relies on the backing bitcoind node to maintain privacy.
+
+
+### Architecture
+
+Dojo is composed of 3 modules:
+* API (/account): web server providing a REST API and web sockets used by Samourai Wallet and Sentinel.
+* PushTx (/pushtx): web server providing a REST API used to push transactions on the Bitcoin P2P network.
+* Tracker (/tracker): process listening to the bitcoind node and indexing transactions of interest.
+
+API and PushTx modules are able to operate behind a web server (e.g. nginx) or as frontend http servers (not recommended). Both support HTTP or HTTPS (if SSL has been properly configured in /keys/index.js). These modules can also operate as a Tor hidden service (recommended).
+
+Authentication is enforced by an API key and Json Web Tokens.
+
+
+### Implementation Notes
+
+**Tracker**
+
+* ZMQ notifications send raw transactions and block hashes. Keep track of txids with timestamps, clearing out old txids after a timeout
+* On realtime transaction:
+  * Query database with all output addresses to see if an account has received a transaction. Notify client via WebSocket.
+  * Query database with all input txids to see if an account has sent coins. Make proper database entries and notify via WebSocket.
+* On a block notification, query database for txids included and update confirmed height
+* On a blockchain reorg (orphan block), previous block hash will not match last known block hash in the app. Need to mark transactions as unconfirmed and rescan blocks from new chain tip to last known hash. Note that many of the transactions from the orphaned block may be included in the new chain.
+* When an input spending a known output is confirmed in a block, delete any other inputs referencing that output, since this would be a double-spend.
+
+
+**Import of HD Accounts and data sources**
+
+* First import of an unknown HD account relies on a data source (local bitcoind or OXT). After that, the tracker will keep everything current.
+
+* Default option relies on the local bitcoind and makes you 100% independent of Samourai Wallet's infrastructure. This option is recommended for better privacy.
+
+* Activation of bitcoind as the data source:
+  * Edit /keys/index.js and set "explorers.bitcoind" to "active". OXT API will be ignored.
+
+* Activation of OXT as the data source (through socks5):
+  * Edit /keys/index.js and set "explorers.bitcoind" to "inactive".
+
+* Main drawbacks of using your local bitcoind for these imports:
+  * It doesn't return the full transactional history associated to the HD account but only transactions having an unspent output controlled by the HD account.
+  * It's slightly slower than using the option relying on the OXT API.
+  * In some specific cases, the importer might miss the most recent unspent outputs. Higher values of gap.external and gap.internal in /keys/index.js should help to mitigate this issue. Another workaround is to request the endpoint /support/xpub/.../rescan provided by the REST API with the optional gap parameter.
+  * This option is considered as experimental. 

accounts/api-helper.js

+/*!
+ * accounts/api-helper.js
+ * Copyright © 2019 – Katana Cryptographic Ltd. All Rights Reserved.
+ */
+'use strict'
+
+const bitcoin = require('bitcoinjs-lib')
+const validator = require('validator')
+const Logger = require('../lib/logger')
+const errors = require('../lib/errors')
+const WalletEntities = require('../lib/wallet/wallet-entities')
+const network = require('../lib/bitcoin/network')
+const activeNet = network.network
+const hdaHelper = require('../lib/bitcoin/hd-accounts-helper')
+const addrHelper = require('../lib/bitcoin/addresses-helper')
+const HttpServer = require('../lib/http-server/http-server')
+
+
+/**
+ * A singleton providing util methods used by the API
+ */
+class ApiHelper {
+
+  /**
+   * Parse a string and extract (x|y|z|t|u|v)pubs, addresses and pubkeys
+   * @param {string} str - list of entities separated by '|'
+   * @returns {object} returns a WalletEntities object
+   */
+  parseEntities(str) {
+    const ret = new WalletEntities()
+    
+    if (typeof str !== 'string')
+      return ret
+    
+    for (let item of str.split('|')) {
+      try {
+
+        if (hdaHelper.isValid(item) && !ret.hasXPub(item)) {
+          const xpub = hdaHelper.xlatXPUB(item)
+
+          if (hdaHelper.isYpub(item))
+            ret.addHdAccount(xpub, item, false)
+          else if (hdaHelper.isZpub(item))
+            ret.addHdAccount(xpub, false, item)
+          else
+            ret.addHdAccount(item, false, false)
+
+        } else if (addrHelper.isSupportedPubKey(item) && !ret.hasPubKey(item)) {
+          // Derive pubkey as 3 addresses (P1PKH, P2WPKH/P2SH, BECH32)
+          const bufItem = new Buffer(item, 'hex')
+
+          const funcs = [
+            addrHelper.p2pkhAddress,
+            addrHelper.p2wpkhP2shAddress,
+            addrHelper.p2wpkhAddress
+          ]
+
+          for (let f of funcs) {
+            const addr = f(bufItem)
+            if (ret.hasAddress(addr))
+              ret.updatePubKey(addr, item)
+            else
+              ret.addAddress(addr, item)
+          }
+
+        } else if (bitcoin.address.toOutputScript(item, activeNet) && !ret.hasAddress(item)) {
+
+          // Bech32 addresses are managed in lower case
+          if (addrHelper.isBech32(item))
+            item = item.toLowerCase()
+          ret.addAddress(item, false)
+        }
+      } catch(e) {}
+    }
+
+    return ret
+  }
+
+  /**
+   * Check entities passed as url params
+   * @param {object} params - request query or body object
+   * @returns {boolean} return true if conditions are met, false otherwise
+   */
+  checkEntitiesParams(params) {
+    return params.active
+      || params.new
+      || params.pubkey
+      || params.bip49
+      || params.bip84
+  }
+
+  /**
+   * Parse the entities passed as arguments of an url
+   * @param {object} params - request query or body object
+   * @returns {object} return a mapping object
+   *    {active:..., legacy:..., pubkey:..., bip49:..., bip84:...}
+   */
+  parseEntitiesParams(params) {
+    return {
+      active: this.parseEntities(params.active),
+      legacy: this.parseEntities(params.new),
+      pubkey: this.parseEntities(params.pubkey),
+      bip49: this.parseEntities(params.bip49),
+      bip84: this.parseEntities(params.bip84)
+    }
+  }
+
+  /**
+   * Express middleware validating if entities params are well formed
+   * @param {object} req - http request object
+   * @param {object} res - http response object
+   * @param {function} next - next express middleware
+   */
+  validateEntitiesParams(req, res, next) {
+    const params = this.checkEntitiesParams(req.query) ? req.query : req.body
+
+    let isValid = true
+
+    if (params.active && !this.subValidateEntitiesParams(params.active))
+      isValid &= false
+
+    if (params.new && !this.subValidateEntitiesParams(params.new))
+      isValid &= false
+
+    if (params.pubkey && !this.subValidateEntitiesParams(params.pubkey))
+      isValid &= false
+
+    if (params.bip49 && !this.subValidateEntitiesParams(params.bip49))
+      isValid &= false
+
+    if (params.bip84 && !this.subValidateEntitiesParams(params.bip84))
+      isValid &= false
+
+    if (isValid) {
+      next()
+    } else {
+      HttpServer.sendError(res, errors.body.INVDATA)
+      Logger.error(
+        params,
+        `ApiHelper.validateEntitiesParams() : Invalid arguments`
+      )
+    }
+  }
+
+  /**
+   * Validate a request argument
+   * @param {string} arg - request argument
+   */
+  subValidateEntitiesParams(arg) {
+    for (let item of arg.split('|')) {
+      const isValid = validator.isAlphanumeric(item)
+      if (!isValid)
+        return false
+    }
+    return true
+  }
+
+}
+
+module.exports = new ApiHelper()

accounts/fees-rest-api.js

+/*!
+ * accounts/get-fees-rest-api.js
+ * Copyright © 2019 – Katana Cryptographic Ltd. All Rights Reserved.
+ */
+'use strict'
+
+const Logger = require('../lib/logger')
+const rpcFees = require('../lib/bitcoind-rpc/fees')
+const authMgr = require('../lib/auth/authorizations-manager')
+const HttpServer = require('../lib/http-server/http-server')
+
+const debugApi = !!(process.argv.indexOf('api-debug') > -1)
+
+
+/**
+ * A singleton providing util methods used by the API
+ */
+class FeesRestApi {
+
+  /**
+   * Constructor
+   * @param {pushtx.HttpServer} httpServer - HTTP server
+   */
+  constructor(httpServer) {
+    this.httpServer = httpServer
+    // Establish routes
+    this.httpServer.app.get(
+      '/fees',
+      authMgr.checkAuthentication.bind(authMgr),
+      this.getFees.bind(this),
+      HttpServer.sendAuthError
+    )
+    // Refresh the network fees
+    rpcFees.refresh()
+  }
+
+  /**
+   * Refresh and return the current fees
+   * @param {object} req - http request object
+   * @param {object} res - http response object
+   */
+  async getFees(req, res) {
+    try {
+      const fees = await rpcFees.getFees()
+      HttpServer.sendOkDataOnly(res, fees)
+    } catch (e) {
+      HttpServer.sendError(res, e)
+    } finally {
+      debugApi && Logger.info(`Completed GET /fees`)
+    }
+  }
+
+}
+
+module.exports = FeesRestApi

accounts/headers-rest-api.js

+/*!
+ * accounts/headers-fees-rest-api.js
+ * Copyright © 2019 – Katana Cryptographic Ltd. All Rights Reserved.
+ */
+'use strict'
+
+const validator = require('validator')
+const Logger = require('../lib/logger')
+const errors = require('../lib/errors')
+const rpcHeaders = require('../lib/bitcoind-rpc/headers')
+const authMgr = require('../lib/auth/authorizations-manager')
+const HttpServer = require('../lib/http-server/http-server')
+const apiHelper = require('./api-helper')
+
+const debugApi = !!(process.argv.indexOf('api-debug') > -1)
+
+
+/**
+ * Headers API endpoints
+ */
+class HeadersRestApi {
+
+  /**
+   * Constructor
+   * @param {pushtx.HttpServer} httpServer - HTTP server
+   */
+  constructor(httpServer) {
+    this.httpServer = httpServer
+
+    // Establish routes
+    this.httpServer.app.get(
+      '/header/:hash',
+      authMgr.checkAuthentication.bind(authMgr),
+      this.validateArgsGetHeader.bind(this),
+      this.getHeader.bind(this),
+      HttpServer.sendAuthError
+    )
+  }
+
+  /**
+   * Retrieve the block header for a given hash
+   * @param {object} req - http request object
+   * @param {object} res - http response object
+   */
+  async getHeader(req, res) {
+    try {
+      const header = await rpcHeaders.getHeader(req.params.hash)
+      HttpServer.sendRawData(res, header)
+    } catch(e) {
+      HttpServer.sendError(res, e)
+    } finally {
+      debugApi && Logger.info(`Completed GET /header/${req.params.hash}`)
+    }
+  }
+
+  /**
+   * Validate request arguments
+   * @param {object} req - http request object
+   * @param {object} res - http response object
+   * @param {function} next - next express middleware
+   */
+  validateArgsGetHeader(req, res, next) {
+    const isValidHash = validator.isHash(req.params.hash, 'sha256')
+
+    if (!isValidHash) {
+      HttpServer.sendError(res, errors.body.INVDATA)
+      Logger.error(
+        req.params.hash,
+        'HeadersRestApi.validateArgsGetHeader() : Invalid hash'
+      )
+    } else {
+      next()
+    }
+  }
+
+}
+
+module.exports = HeadersRestApi

accounts/index-cluster.js

+/*!
+ * accounts/index-cluster.js
+ * Copyright © 2019 – Katana Cryptographic Ltd. All Rights Reserved.
+ */
+'use strict'
+
+const os = require('os')
+const cluster = require('cluster')
+const Logger = require('../lib/logger')
+
+
+/**
+ * Launch a cluster of Samourai API
+ */
+const nbCPUS = os.cpus()
+
+if (cluster.isMaster) {  
+  nbCPUS.forEach(function() {
+    cluster.fork()
+  })
+
+  cluster.on('listening', function(worker) {
+    Logger.info(`Cluster ${worker.process.pid} connected`)
+  })
+  
+  cluster.on('disconnect', function(worker) {
+    Logger.info(`Cluster ${worker.process.pid} disconnected`)
+  })
+  
+  cluster.on('exit', function(worker) {
+    Logger.info(`Cluster ${worker.process.pid} is dead`)
+    // Ensuring a new cluster will start if an old one dies
+    cluster.fork()
+  })
+} else {
+  require('./index.js')
+}

accounts/index.js

+/*!
+ * accounts/index.js
+ * Copyright © 2019 – Katana Cryptographic Ltd. All Rights Reserved.
+ */
+(async () => {
+
+  'use strict'
+
+  const Logger = require('../lib/logger')
+  const RpcClient = require('../lib/bitcoind-rpc/rpc-client')
+  const network = require('../lib/bitcoin/network')
+  const keys = require('../keys')[network.key]
+  const db = require('../lib/db/mysql-db-wrapper')
+  const hdaHelper = require('../lib/bitcoin/hd-accounts-helper')
+  const HttpServer = require('../lib/http-server/http-server')
+  const AuthRestApi = require('../lib/auth/auth-rest-api')
+  const XPubRestApi = require('./xpub-rest-api')
+  const FeesRestApi = require('./fees-rest-api')
+  const HeadersRestApi = require('./headers-rest-api')
+  const TransactionsRestApi = require('./transactions-rest-api')
+  const StatusRestApi = require('./status-rest-api')
+  const notifServer = require('./notifications-server')
+  const MultiaddrRestApi = require('./multiaddr-rest-api')
+  const UnspentRestApi = require('./unspent-rest-api')
+  const SupportRestApi = require('./support-rest-api')
+
+
+  /**
+   * Samourai REST API
+   */
+  Logger.info('Process ID: ' + process.pid)
+  Logger.info('Preparing the REST API')
+  
+  // Wait for Bitcoind RPC API
+  // being ready to process requests
+  await RpcClient.waitForBitcoindRpcApi()
+
+  // Initialize the db wrapper
+  const dbConfig = {
+    connectionLimit: keys.db.connectionLimitApi,
+    acquireTimeout: keys.db.acquireTimeout,
+    host: keys.db.host,
+    user: keys.db.user,
+    password: keys.db.pass,
+    database: keys.db.database
+  }
+
+  db.connect(dbConfig)
+
+  // Activate addresses derivation
+  // in an external process
+  hdaHelper.activateExternalDerivation()
+
+  // Initialize the http server
+  const port = keys.ports.account
+  const httpsOptions = keys.https.account
+  const httpServer = new HttpServer(port, httpsOptions)
+
+  // Initialize the rest api endpoints
+  const authRestApi = new AuthRestApi(httpServer)
+  const xpubRestApi = new XPubRestApi(httpServer)
+  const feesRestApi = new FeesRestApi(httpServer)
+  const headersRestApi = new HeadersRestApi(httpServer)
+  const transactionsRestApi = new TransactionsRestApi(httpServer)
+  const statusRestApi = new StatusRestApi(httpServer)
+  const multiaddrRestApi = new MultiaddrRestApi(httpServer)
+  const unspentRestApi = new UnspentRestApi(httpServer)
+  const supportRestApi = new SupportRestApi(httpServer)
+
+  // Start the http server
+  httpServer.start()
+
+  // Attach the web sockets server to the web server
+  notifServer.attach(httpServer)
+
+})()

accounts/multiaddr-rest-api.js

+/*!
+ * accounts/multiaddr-rest-api.js
+ * Copyright © 2019 – Katana Cryptographic Ltd. All Rights Reserved.
+ */
+'use strict'
+
+const bodyParser = require('body-parser')
+const Logger = require('../lib/logger')
+const errors = require('../lib/errors')
+const walletService = require('../lib/wallet/wallet-service')
+const authMgr = require('../lib/auth/authorizations-manager')
+const HttpServer = require('../lib/http-server/http-server')
+const apiHelper = require('./api-helper')
+
+const debugApi = !!(process.argv.indexOf('api-debug') > -1)
+
+
+/**
+ * Multiaddr API endpoints
+ */
+class MultiaddrRestApi {
+
+  /**
+   * Constructor
+   * @param {pushtx.HttpServer} httpServer - HTTP server
+   */
+  constructor(httpServer) {
+    this.httpServer = httpServer
+
+    // Establish routes
+    const urlencodedParser = bodyParser.urlencoded({ extended: true })
+
+    this.httpServer.app.get(
+      '/multiaddr',
+      authMgr.checkAuthentication.bind(authMgr),
+      apiHelper.validateEntitiesParams.bind(apiHelper),
+      this.getMultiaddr.bind(this),
+      HttpServer.sendAuthError
+    )
+
+    this.httpServer.app.post(
+      '/multiaddr',
+      urlencodedParser,
+      authMgr.checkAuthentication.bind(authMgr),
+      apiHelper.validateEntitiesParams.bind(apiHelper),
+      this.postMultiaddr.bind(this),
+      HttpServer.sendAuthError
+    )
+  }
+
+  /**
+   * Handle multiaddr GET request
+   * @param {object} req - http request object
+   * @param {object} res - http response object
+   */
+  async getMultiaddr(req, res) {
+    try {
+      // Check request params
+      if (!apiHelper.checkEntitiesParams(req.query))
+        return HttpServer.sendError(res, errors.multiaddr.NOACT)
+
+      // Parse params
+      const entities = apiHelper.parseEntitiesParams(req.query)
+
+      const result = await walletService.getWalletInfo(
+        entities.active,
+        entities.legacy,
+        entities.bip49,
+        entities.bip84,
+        entities.pubkey
+      )
+
+      const ret = JSON.stringify(result, null, 2)
+      HttpServer.sendRawData(res, ret)
+
+    } catch(e) {
+      HttpServer.sendError(res, e)
+
+    } finally {
+      if (debugApi) {
+        const strParams = 
+          `${req.query.active ? req.query.active : ''} \
+          ${req.query.new ? req.query.new : ''} \
+          ${req.query.pubkey ? req.query.pubkey : ''} \
+          ${req.query.bip49 ? req.query.bip49 : ''} \
+          ${req.query.bip84 ? req.query.bip84 : ''}`
+
+        Logger.info(`Completed GET /multiaddr ${strParams}`)
+      }
+    }
+  }
+
+  /**
+   * Handle multiaddr POST request
+   * @param {object} req - http request object
+   * @param {object} res - http response object
+   */
+  async postMultiaddr(req, res) {
+    try {
+      // Check request params
+      if (!apiHelper.checkEntitiesParams(req.body))
+        return HttpServer.sendError(res, errors.multiaddr.NOACT)
+
+      // Parse params
+      const entities = apiHelper.parseEntitiesParams(req.body)
+
+      const result = await walletService.getWalletInfo(
+        entities.active,