I'm still working on my Lorenz curve project where I'm trying to learn Node.js and the Steem APIs. In order to get information about rewards coming from things like curation and beneficiaries it seemed that I would need to switch to scanning the blocks to find the virtual operations that actually deliver the different types of rewards. The getOperations method seems to be able to do that, so I was initially optimistic.

function scanForOps(start) {
  let pList = [];
  for (let bn=start; bn>=start-1; bn--) {
    console.log("Issue getOperations for block number", bn);
    pList.push(client.database.getOperations(bn, true).then(ops => {
      ops.forEach(op => {
        console.log("op occured", op);
        if (typeof(opHash[op.op[0]]) === 'undefined') {
          opHash[op.op[0]] = {count: 1, example: op};

          let reward = {};
          switch (op.op[0]) {
            case 'producer_reward':
              reward[op.op[1].producer] = {
                type: op.op[0],
                vests: parseVests(op.op[1].vesting_shares)
              };
              break;

            case 'curation_reward':
              reward[op.op[1].curator] = {
                type: op.op[0],
                vests: parseVests(op.op[1].reward)
              };
              break;

            case 'comment_benefactor_reward':
              reward[op.op[1].benefactor] = {
                type: op.op[0],
                vests: parseVests(op.op[1].vesting_payout),
                steem: parseSteem(op.op[1].steem_payout),
                sbd:   parseSBD(op.op[1].sbd_payout)
              };
              break;

            case 'author_reward':
              reward[op.op[1].author] = {
                type: op.op[0],
                vests: parseVests(op.op[1].vesting_payout),
                steem: parseSteem(op.op[1].steem_payout),
                sbd:   parseSBD(op.op[1].sbd_payout)
              };
              break;

            case 'return_vesting_delegation':
            case 'fill_vesting_withdraw':
            case 'fill_order':
            case 'fill_convert_request':
              // not involved in rewards
              break;

            default:
              console.error("Don't know what to do with", op.op[0]);
              break;
          }
          console.log("Reward:", reward);

        } else {
          opHash[op.op[0]].count++;
        }
      });
    }).catch(err => {
      console.error(err);
    }));
  }
  return Promise.all(pList);
}

Each operation is a little bit different in terms of what parameters it uses, but I plan to just expand my parsing function to get them all into a common "$X going to Y person" data structure. However, when I started trying to scale it up I started running into error messages about not being able to acquire a lock on the database. I'm not 100% sure, but I think it's likely due to rate limiting from the API servers. I did some searches to see and found some threads that talked about rate limiting but didn't find any concrete information. But it makes sense that the API servers wouldn't want to be swamped by tons of requests for data. So my plan is to have my app create a local DB to serve as a cache for the information pulled from the blockchain so I can filter out duplicate requests, and I'll build a rate-limiter into my blockchain querying code.

That works fine in theory, but in practice I've been banging my head against the wall trying to get Node.js to do what I want. I think I've got about a 75% handle on Node.js's asynchronous programming model, so I can get a lot of the more straightforward stuff to work how I want, but I'm still stumbling with more complex structures like a loop that issues a chunk of requests that can operate in parallel. Intellectually I recognize that it makes sense to be struggling with things while I'm still learning, but the speed bumps and slow progress have been a bit of a drain emotionally.

CoinGecko API

To give myself a vacation from that so I could come back fresh I started looking at the CoinGecko API to see if there were easy ways to get things like market price information. It was a little aggravating at first to figure out how to construct the requests and parse the data structures of what you get back, but eventually I figured out what was going on enough to get my little test script working.

const CoinGecko = require('coingecko-api');
const CoinGeckoClient = new CoinGecko();

let marketPrices = {};

// id | symbol | name
// bitcoin | btc | bitcoin
// hive | hive | Hive
// hive_dollar | HBD | Hive Dollar
// steem | steem | Steem
// steem-dollars | sbd | Steem Dollars

async function getPricesFromCoinGecko() {
  await CoinGeckoClient.ping().then((data) => {
    console.log("Resolved ping with", data);
  }).then(() => {
    return CoinGeckoClient.coins.markets({ids:"bitcoin,hive,hive_dollar,steem,steem-dollars"});
  }).then((response) => {
    response.data.forEach((coin) => {
      marketPrices[coin.symbol.toUpperCase()] = coin.current_price;
      //console.log(coin.symbol.toUpperCase(), "price is", coin.current_price);
    });
  });
  console.log("Got info from CoinGecko");
  return marketPrices;
}

getPricesFromCoinGecko().then((result) => {
    console.log(result);
});