Commit b4cd1563 authored by kenshin-samourai's avatar kenshin-samourai
Browse files

Initial commit

parents
node_modules
.git
private-tests
\ No newline at end of file
db-scripts/updates/
keys/index.js
keys/sslcert/
node_modules/
private-tests/
*.log
package-lock.json
This diff is collapsed.
# Samourai Dojo
Samourai Dojo is the backing server for Samourai Wallet. Provides HD account & loose addresses (BIP47) balances & transactions lists. Provides unspent output lists to the wallet. PushTX endpoint broadcasts transactions through the backing bitcoind node.
[View API documentation](../master/doc/README.md)
## Installation ##
### MyDojo (installation with Docker and Docker Compose)
This setup is recommended to Samourai users who feel comfortable with a few command lines.
It provides in a single command the setup of a full Samourai backend composed of:
* a bitcoin full node only accessible as an ephemeral Tor hidden service,
* the backend database,
* the backend modules with an API accessible as a static Tor hidden service,
* a maintenance tool accessible through a Tor web browser.
See [the documentation](./doc/DOCKER_setup.md) for detailed setup instructions.
### Manual installation (developers only)
A full manual setup isn't recommended if you don't intend to install a local development environment.
## Theory of Operation
Tracking wallet balances via `xpub` requires conforming to [BIP44](https://github.com/bitcoin/bips/blob/master/bip-0044.mediawiki), [BIP49](https://github.com/bitcoin/bips/blob/master/bip-0049.mediawiki) or [BIP84](https://github.com/bitcoin/bips/blob/master/bip-0084.mediawiki) address derivation scheme. Public keys received by Dojo correspond to single accounts and derive all addresses in the account and change chains. These addresses are at `M/0/x` and `M/1/y`, respectively.
Dojo relies on the backing bitcoind node to maintain privacy.
### Architecture
Dojo is composed of 3 modules:
* API (/account): web server providing a REST API and web sockets used by Samourai Wallet and Sentinel.
* PushTx (/pushtx): web server providing a REST API used to push transactions on the Bitcoin P2P network.
* Tracker (/tracker): process listening to the bitcoind node and indexing transactions of interest.
API and PushTx modules are able to operate behind a web server (e.g. nginx) or as frontend http servers (not recommended). Both support HTTP or HTTPS (if SSL has been properly configured in /keys/index.js). These modules can also operate as a Tor hidden service (recommended).
Authentication is enforced by an API key and Json Web Tokens.
### Implementation Notes
**Tracker**
* ZMQ notifications send raw transactions and block hashes. Keep track of txids with timestamps, clearing out old txids after a timeout
* On realtime transaction:
* Query database with all output addresses to see if an account has received a transaction. Notify client via WebSocket.
* Query database with all input txids to see if an account has sent coins. Make proper database entries and notify via WebSocket.
* On a block notification, query database for txids included and update confirmed height
* On a blockchain reorg (orphan block), previous block hash will not match last known block hash in the app. Need to mark transactions as unconfirmed and rescan blocks from new chain tip to last known hash. Note that many of the transactions from the orphaned block may be included in the new chain.
* When an input spending a known output is confirmed in a block, delete any other inputs referencing that output, since this would be a double-spend.
**Import of HD Accounts and data sources**
* First import of an unknown HD account relies on a data source (local bitcoind or OXT). After that, the tracker will keep everything current.
* Default option relies on the local bitcoind and makes you 100% independent of Samourai Wallet's infrastructure. This option is recommended for better privacy.
* Activation of bitcoind as the data source:
* Edit /keys/index.js and set "explorers.bitcoind" to "active". OXT API will be ignored.
* Activation of OXT as the data source (through socks5):
* Edit /keys/index.js and set "explorers.bitcoind" to "inactive".
* Main drawbacks of using your local bitcoind for these imports:
* It doesn't return the full transactional history associated to the HD account but only transactions having an unspent output controlled by the HD account.
* It's slightly slower than using the option relying on the OXT API.
* In some specific cases, the importer might miss the most recent unspent outputs. Higher values of gap.external and gap.internal in /keys/index.js should help to mitigate this issue. Another workaround is to request the endpoint /support/xpub/.../rescan provided by the REST API with the optional gap parameter.
* This option is considered as experimental.
/*!
* accounts/api-helper.js
* Copyright © 2019 – Katana Cryptographic Ltd. All Rights Reserved.
*/
'use strict'
const bitcoin = require('bitcoinjs-lib')
const validator = require('validator')
const Logger = require('../lib/logger')
const errors = require('../lib/errors')
const WalletEntities = require('../lib/wallet/wallet-entities')
const network = require('../lib/bitcoin/network')
const activeNet = network.network
const hdaHelper = require('../lib/bitcoin/hd-accounts-helper')
const addrHelper = require('../lib/bitcoin/addresses-helper')
const HttpServer = require('../lib/http-server/http-server')
/**
* A singleton providing util methods used by the API
*/
class ApiHelper {
/**
* Parse a string and extract (x|y|z|t|u|v)pubs, addresses and pubkeys
* @param {string} str - list of entities separated by '|'
* @returns {object} returns a WalletEntities object
*/
parseEntities(str) {
const ret = new WalletEntities()
if (typeof str !== 'string')
return ret
for (let item of str.split('|')) {
try {
if (hdaHelper.isValid(item) && !ret.hasXPub(item)) {
const xpub = hdaHelper.xlatXPUB(item)
if (hdaHelper.isYpub(item))
ret.addHdAccount(xpub, item, false)
else if (hdaHelper.isZpub(item))
ret.addHdAccount(xpub, false, item)
else
ret.addHdAccount(item, false, false)
} else if (addrHelper.isSupportedPubKey(item) && !ret.hasPubKey(item)) {
// Derive pubkey as 3 addresses (P1PKH, P2WPKH/P2SH, BECH32)
const bufItem = new Buffer(item, 'hex')
const funcs = [
addrHelper.p2pkhAddress,
addrHelper.p2wpkhP2shAddress,
addrHelper.p2wpkhAddress
]
for (let f of funcs) {
const addr = f(bufItem)
if (ret.hasAddress(addr))
ret.updatePubKey(addr, item)
else
ret.addAddress(addr, item)
}
} else if (bitcoin.address.toOutputScript(item, activeNet) && !ret.hasAddress(item)) {
// Bech32 addresses are managed in lower case
if (addrHelper.isBech32(item))
item = item.toLowerCase()
ret.addAddress(item, false)
}
} catch(e) {}
}
return ret
}
/**
* Check entities passed as url params
* @param {object} params - request query or body object
* @returns {boolean} return true if conditions are met, false otherwise
*/
checkEntitiesParams(params) {
return params.active
|| params.new
|| params.pubkey
|| params.bip49
|| params.bip84
}
/**
* Parse the entities passed as arguments of an url
* @param {object} params - request query or body object
* @returns {object} return a mapping object
* {active:..., legacy:..., pubkey:..., bip49:..., bip84:...}
*/
parseEntitiesParams(params) {
return {
active: this.parseEntities(params.active),
legacy: this.parseEntities(params.new),
pubkey: this.parseEntities(params.pubkey),
bip49: this.parseEntities(params.bip49),
bip84: this.parseEntities(params.bip84)
}
}
/**
* Express middleware validating if entities params are well formed
* @param {object} req - http request object
* @param {object} res - http response object
* @param {function} next - next express middleware
*/
validateEntitiesParams(req, res, next) {
const params = this.checkEntitiesParams(req.query) ? req.query : req.body
let isValid = true
if (params.active && !this.subValidateEntitiesParams(params.active))
isValid &= false
if (params.new && !this.subValidateEntitiesParams(params.new))
isValid &= false
if (params.pubkey && !this.subValidateEntitiesParams(params.pubkey))
isValid &= false
if (params.bip49 && !this.subValidateEntitiesParams(params.bip49))
isValid &= false
if (params.bip84 && !this.subValidateEntitiesParams(params.bip84))
isValid &= false
if (isValid) {
next()
} else {
HttpServer.sendError(res, errors.body.INVDATA)
Logger.error(
params,
`ApiHelper.validateEntitiesParams() : Invalid arguments`
)
}
}
/**
* Validate a request argument
* @param {string} arg - request argument
*/
subValidateEntitiesParams(arg) {
for (let item of arg.split('|')) {
const isValid = validator.isAlphanumeric(item)
if (!isValid)
return false
}
return true
}
}
module.exports = new ApiHelper()
/*!
* accounts/get-fees-rest-api.js
* Copyright © 2019 – Katana Cryptographic Ltd. All Rights Reserved.
*/
'use strict'
const Logger = require('../lib/logger')
const rpcFees = require('../lib/bitcoind-rpc/fees')
const authMgr = require('../lib/auth/authorizations-manager')
const HttpServer = require('../lib/http-server/http-server')
const debugApi = !!(process.argv.indexOf('api-debug') > -1)
/**
* A singleton providing util methods used by the API
*/
class FeesRestApi {
/**
* Constructor
* @param {pushtx.HttpServer} httpServer - HTTP server
*/
constructor(httpServer) {
this.httpServer = httpServer
// Establish routes
this.httpServer.app.get(
'/fees',
authMgr.checkAuthentication.bind(authMgr),
this.getFees.bind(this),
HttpServer.sendAuthError
)
// Refresh the network fees
rpcFees.refresh()
}
/**
* Refresh and return the current fees
* @param {object} req - http request object
* @param {object} res - http response object
*/
async getFees(req, res) {
try {
const fees = await rpcFees.getFees()
HttpServer.sendOkDataOnly(res, fees)
} catch (e) {
HttpServer.sendError(res, e)
} finally {
debugApi && Logger.info(`Completed GET /fees`)
}
}
}
module.exports = FeesRestApi
/*!
* accounts/headers-fees-rest-api.js
* Copyright © 2019 – Katana Cryptographic Ltd. All Rights Reserved.
*/
'use strict'
const validator = require('validator')
const Logger = require('../lib/logger')
const errors = require('../lib/errors')
const rpcHeaders = require('../lib/bitcoind-rpc/headers')
const authMgr = require('../lib/auth/authorizations-manager')
const HttpServer = require('../lib/http-server/http-server')
const apiHelper = require('./api-helper')
const debugApi = !!(process.argv.indexOf('api-debug') > -1)
/**
* Headers API endpoints
*/
class HeadersRestApi {
/**
* Constructor
* @param {pushtx.HttpServer} httpServer - HTTP server
*/
constructor(httpServer) {
this.httpServer = httpServer
// Establish routes
this.httpServer.app.get(
'/header/:hash',
authMgr.checkAuthentication.bind(authMgr),
this.validateArgsGetHeader.bind(this),
this.getHeader.bind(this),
HttpServer.sendAuthError
)
}
/**
* Retrieve the block header for a given hash
* @param {object} req - http request object
* @param {object} res - http response object
*/
async getHeader(req, res) {
try {
const header = await rpcHeaders.getHeader(req.params.hash)
HttpServer.sendRawData(res, header)
} catch(e) {
HttpServer.sendError(res, e)
} finally {
debugApi && Logger.info(`Completed GET /header/${req.params.hash}`)
}
}
/**
* Validate request arguments
* @param {object} req - http request object
* @param {object} res - http response object
* @param {function} next - next express middleware
*/
validateArgsGetHeader(req, res, next) {
const isValidHash = validator.isHash(req.params.hash, 'sha256')
if (!isValidHash) {
HttpServer.sendError(res, errors.body.INVDATA)
Logger.error(
req.params.hash,
'HeadersRestApi.validateArgsGetHeader() : Invalid hash'
)
} else {
next()
}
}
}
module.exports = HeadersRestApi
/*!
* accounts/index-cluster.js
* Copyright © 2019 – Katana Cryptographic Ltd. All Rights Reserved.
*/
'use strict'
const os = require('os')
const cluster = require('cluster')
const Logger = require('../lib/logger')
/**
* Launch a cluster of Samourai API
*/
const nbCPUS = os.cpus()
if (cluster.isMaster) {
nbCPUS.forEach(function() {
cluster.fork()
})
cluster.on('listening', function(worker) {
Logger.info(`Cluster ${worker.process.pid} connected`)
})
cluster.on('disconnect', function(worker) {
Logger.info(`Cluster ${worker.process.pid} disconnected`)
})
cluster.on('exit', function(worker) {
Logger.info(`Cluster ${worker.process.pid} is dead`)
// Ensuring a new cluster will start if an old one dies
cluster.fork()
})
} else {
require('./index.js')
}
/*!
* accounts/index.js
* Copyright © 2019 – Katana Cryptographic Ltd. All Rights Reserved.
*/
(async () => {
'use strict'
const Logger = require('../lib/logger')
const RpcClient = require('../lib/bitcoind-rpc/rpc-client')
const network = require('../lib/bitcoin/network')
const keys = require('../keys')[network.key]
const db = require('../lib/db/mysql-db-wrapper')
const hdaHelper = require('../lib/bitcoin/hd-accounts-helper')
const HttpServer = require('../lib/http-server/http-server')
const AuthRestApi = require('../lib/auth/auth-rest-api')
const XPubRestApi = require('./xpub-rest-api')
const FeesRestApi = require('./fees-rest-api')
const HeadersRestApi = require('./headers-rest-api')
const TransactionsRestApi = require('./transactions-rest-api')
const StatusRestApi = require('./status-rest-api')
const notifServer = require('./notifications-server')
const MultiaddrRestApi = require('./multiaddr-rest-api')
const UnspentRestApi = require('./unspent-rest-api')
const SupportRestApi = require('./support-rest-api')
/**
* Samourai REST API
*/
Logger.info('Process ID: ' + process.pid)
Logger.info('Preparing the REST API')
// Wait for Bitcoind RPC API
// being ready to process requests
await RpcClient.waitForBitcoindRpcApi()
// Initialize the db wrapper
const dbConfig = {
connectionLimit: keys.db.connectionLimitApi,
acquireTimeout: keys.db.acquireTimeout,
host: keys.db.host,
user: keys.db.user,
password: keys.db.pass,
database: keys.db.database
}
db.connect(dbConfig)
// Activate addresses derivation
// in an external process
hdaHelper.activateExternalDerivation()
// Initialize the http server
const port = keys.ports.account
const httpsOptions = keys.https.account
const httpServer = new HttpServer(port, httpsOptions)
// Initialize the rest api endpoints
const authRestApi = new AuthRestApi(httpServer)
const xpubRestApi = new XPubRestApi(httpServer)
const feesRestApi = new FeesRestApi(httpServer)
const headersRestApi = new HeadersRestApi(httpServer)
const transactionsRestApi = new TransactionsRestApi(httpServer)
const statusRestApi = new StatusRestApi(httpServer)
const multiaddrRestApi = new MultiaddrRestApi(httpServer)
const unspentRestApi = new UnspentRestApi(httpServer)
const supportRestApi = new SupportRestApi(httpServer)
// Start the http server
httpServer.start()
// Attach the web sockets server to the web server
notifServer.attach(httpServer)
})()
/*!
* accounts/multiaddr-rest-api.js
* Copyright © 2019 – Katana Cryptographic Ltd. All Rights Reserved.
*/
'use strict'
const bodyParser = require('body-parser')
const Logger = require('../lib/logger')
const errors = require('../lib/errors')
const walletService = require('../lib/wallet/wallet-service')
const authMgr = require('../lib/auth/authorizations-manager')
const HttpServer = require('../lib/http-server/http-server')
const apiHelper = require('./api-helper')
const debugApi = !!(process.argv.indexOf('api-debug') > -1)
/**
* Multiaddr API endpoints
*/
class MultiaddrRestApi {
/**
* Constructor
* @param {pushtx.HttpServer} httpServer - HTTP server
*/
constructor(httpServer) {
this.httpServer = httpServer
// Establish routes
const urlencodedParser = bodyParser.urlencoded({ extended: true })
this.httpServer.app.get(
'/multiaddr',
authMgr.checkAuthentication.bind(authMgr),
apiHelper.validateEntitiesParams.bind(apiHelper),
this.getMultiaddr.bind(this),
HttpServer.sendAuthError
)
this.httpServer.app.post(
'/multiaddr',
urlencodedParser,
authMgr.checkAuthentication.bind(authMgr),
apiHelper.validateEntitiesParams.bind(apiHelper),
this.postMultiaddr.bind(this),
HttpServer.sendAuthError
)
}
/**
* Handle multiaddr GET request
* @param {object} req - http request object
* @param {object} res - http response object
*/
async getMultiaddr(req, res) {
try {
// Check request params
if (!apiHelper.checkEntitiesParams(req.query))
return HttpServer.sendError(res, errors.multiaddr.NOACT)
// Parse params
const entities = apiHelper.parseEntitiesParams(req.query)
const result = await walletService.getWalletInfo(
entities.active,
entities.legacy,
entities.bip49,
entities.bip84,
entities.pubkey
)
const ret = JSON.stringify(result, null, 2)
HttpServer.sendRawData(res, ret)
} catch(e) {
HttpServer.sendError(res, e)
} finally {
if (debugApi) {
const strParams =
`${req.query.active ? req.query.active : ''} \
${req.query.new ? req.query.new : ''} \
${req.query.pubkey ? req.query.pubkey : ''} \
${req.query.bip49 ? req.query.bip49 : ''} \
${req.query.bip84 ? req.query.bip84 : ''}`
Logger.info(`Completed GET /multiaddr ${strParams}`)
}
}
}
/**
* Handle multiaddr POST request
* @param {object} req - http request object
* @param {object} res - http response object
*/
async postMultiaddr(req, res) {
try {
// Check request params
if (!apiHelper.checkEntitiesParams(req.body))
return HttpServer.sendError(res, errors.multiaddr.NOACT)
// Parse params
const entities = apiHelper.parseEntitiesParams(req.body)
const result = await walletService.getWalletInfo(
entities.active,