BlockChain.cpp 48 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554
  1. /*
  2. This file is part of cpp-ethereum.
  3. cpp-ethereum is free software: you can redistribute it and/or modify
  4. it under the terms of the GNU General Public License as published by
  5. the Free Software Foundation, either version 3 of the License, or
  6. (at your option) any later version.
  7. cpp-ethereum is distributed in the hope that it will be useful,
  8. but WITHOUT ANY WARRANTY; without even the implied warranty of
  9. MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  10. GNU General Public License for more details.
  11. You should have received a copy of the GNU General Public License
  12. along with cpp-ethereum. If not, see <http://www.gnu.org/licenses/>.
  13. */
  14. /** @file BlockChain.cpp
  15. * @author Gav Wood <i@gavwood.com>
  16. * @date 2014
  17. */
  18. #include "BlockChain.h"
  19. #if ETH_PROFILING_GPERF
  20. #include <gperftools/profiler.h>
  21. #endif
  22. #include <boost/timer.hpp>
  23. #include <boost/filesystem.hpp>
  24. #include <json_spirit/JsonSpiritHeaders.h>
  25. #include <libdevcore/Common.h>
  26. #include <libdevcore/Assertions.h>
  27. #include <libdevcore/RLP.h>
  28. #include <libdevcore/TrieHash.h>
  29. #include <libdevcore/FileSystem.h>
  30. #include <libethcore/Exceptions.h>
  31. #include <libethcore/BlockHeader.h>
  32. #include "GenesisInfo.h"
  33. #include "State.h"
  34. #include "Block.h"
  35. #include "Utility.h"
  36. #include "Defaults.h"
  37. using namespace std;
  38. using namespace dev;
  39. using namespace dev::eth;
  40. namespace js = json_spirit;
  41. namespace fs = boost::filesystem;
  42. #define ETH_CATCH 1
  43. #define ETH_TIMED_IMPORTS 1
  44. #if defined(_WIN32)
  45. const char* BlockChainDebug::name() { return EthBlue "8" EthWhite " <>"; }
  46. const char* BlockChainWarn::name() { return EthBlue "8" EthOnRed EthBlackBold " X"; }
  47. const char* BlockChainNote::name() { return EthBlue "8" EthBlue " i"; }
  48. const char* BlockChainChat::name() { return EthBlue "8" EthWhite " o"; }
  49. #else
  50. const char* BlockChainDebug::name() { return EthBlue "☍" EthWhite " ◇"; }
  51. const char* BlockChainWarn::name() { return EthBlue "☍" EthOnRed EthBlackBold " ✘"; }
  52. const char* BlockChainNote::name() { return EthBlue "☍" EthBlue " ℹ"; }
  53. const char* BlockChainChat::name() { return EthBlue "☍" EthWhite " ◌"; }
  54. #endif
  55. std::ostream& dev::eth::operator<<(std::ostream& _out, BlockChain const& _bc)
  56. {
  57. string cmp = toBigEndianString(_bc.currentHash());
  58. auto it = _bc.m_blocksDB->NewIterator(_bc.m_readOptions);
  59. for (it->SeekToFirst(); it->Valid(); it->Next())
  60. if (it->key().ToString() != "best")
  61. {
  62. try {
  63. BlockHeader d(bytesConstRef(it->value()));
  64. _out << toHex(it->key().ToString()) << ": " << d.number() << " @ " << d.parentHash() << (cmp == it->key().ToString() ? " BEST" : "") << std::endl;
  65. }
  66. catch (...) {
  67. cwarn << "Invalid DB entry:" << toHex(it->key().ToString()) << " -> " << toHex(bytesConstRef(it->value()));
  68. }
  69. }
  70. delete it;
  71. return _out;
  72. }
  73. ldb::Slice dev::eth::toSlice(h256 const& _h, unsigned _sub)
  74. {
  75. #if ALL_COMPILERS_ARE_CPP11_COMPLIANT
  76. static thread_local FixedHash<33> h = _h;
  77. h[32] = (uint8_t)_sub;
  78. return (ldb::Slice)h.ref();
  79. #else
  80. static boost::thread_specific_ptr<FixedHash<33>> t_h;
  81. if (!t_h.get())
  82. t_h.reset(new FixedHash<33>);
  83. *t_h = FixedHash<33>(_h);
  84. (*t_h)[32] = (uint8_t)_sub;
  85. return (ldb::Slice)t_h->ref();
  86. #endif //ALL_COMPILERS_ARE_CPP11_COMPLIANT
  87. }
  88. ldb::Slice dev::eth::toSlice(uint64_t _n, unsigned _sub)
  89. {
  90. #if ALL_COMPILERS_ARE_CPP11_COMPLIANT
  91. static thread_local FixedHash<33> h;
  92. toBigEndian(_n, bytesRef(h.data() + 24, 8));
  93. h[32] = (uint8_t)_sub;
  94. return (ldb::Slice)h.ref();
  95. #else
  96. static boost::thread_specific_ptr<FixedHash<33>> t_h;
  97. if (!t_h.get())
  98. t_h.reset(new FixedHash<33>);
  99. bytesRef ref(t_h->data() + 24, 8);
  100. toBigEndian(_n, ref);
  101. (*t_h)[32] = (uint8_t)_sub;
  102. return (ldb::Slice)t_h->ref();
  103. #endif
  104. }
  105. namespace dev
  106. {
  107. class WriteBatchNoter: public ldb::WriteBatch::Handler
  108. {
  109. virtual void Put(ldb::Slice const& _key, ldb::Slice const& _value) { cnote << "Put" << toHex(bytesConstRef(_key)) << "=>" << toHex(bytesConstRef(_value)); }
  110. virtual void Delete(ldb::Slice const& _key) { cnote << "Delete" << toHex(bytesConstRef(_key)); }
  111. };
  112. }
  113. #if ETH_DEBUG&&0
  114. static const chrono::system_clock::duration c_collectionDuration = chrono::seconds(15);
  115. static const unsigned c_collectionQueueSize = 2;
  116. static const unsigned c_maxCacheSize = 1024 * 1024 * 1;
  117. static const unsigned c_minCacheSize = 1;
  118. #else
  119. /// Duration between flushes.
  120. static const chrono::system_clock::duration c_collectionDuration = chrono::seconds(60);
  121. /// Length of death row (total time in cache is multiple of this and collection duration).
  122. static const unsigned c_collectionQueueSize = 20;
  123. /// Max size, above which we start forcing cache reduction.
  124. static const unsigned c_maxCacheSize = 1024 * 1024 * 64;
  125. /// Min size, below which we don't bother flushing it.
  126. static const unsigned c_minCacheSize = 1024 * 1024 * 32;
  127. #endif
  128. BlockChain::BlockChain(ChainParams const& _p, std::string const& _dbPath, WithExisting _we, ProgressCallback const& _pc):
  129. m_dbPath(_dbPath)
  130. {
  131. init(_p, _dbPath);
  132. open(_dbPath, _we, _pc);
  133. }
  134. BlockChain::~BlockChain()
  135. {
  136. close();
  137. }
  138. BlockHeader const& BlockChain::genesis() const
  139. {
  140. UpgradableGuard l(x_genesis);
  141. if (!m_genesis)
  142. {
  143. auto gb = m_params.genesisBlock();
  144. UpgradeGuard ul(l);
  145. m_genesis = BlockHeader(gb);
  146. m_genesisHeaderBytes = BlockHeader::extractHeader(&gb).data().toBytes();
  147. m_genesisHash = m_genesis.hash();
  148. }
  149. return m_genesis;
  150. }
  151. void BlockChain::init(ChainParams const& _p, std::string const& _path)
  152. {
  153. // initialise deathrow.
  154. m_cacheUsage.resize(c_collectionQueueSize);
  155. m_lastCollection = chrono::system_clock::now();
  156. // Initialise with the genesis as the last block on the longest chain.
  157. m_params = _p;
  158. m_sealEngine.reset(m_params.createSealEngine());
  159. m_genesis.clear();
  160. genesis();
  161. // remove the next line real soon. we don't need to be supporting this forever.
  162. upgradeDatabase(_path, genesisHash());
  163. }
  164. unsigned BlockChain::open(std::string const& _path, WithExisting _we)
  165. {
  166. string path = _path.empty() ? Defaults::get()->m_dbPath : _path;
  167. string chainPath = path + "/" + toHex(m_genesisHash.ref().cropped(0, 4));
  168. string extrasPath = chainPath + "/" + toString(c_databaseVersion);
  169. fs::create_directories(extrasPath);
  170. DEV_IGNORE_EXCEPTIONS(fs::permissions(extrasPath, fs::owner_all));
  171. bytes status = contents(extrasPath + "/minor");
  172. unsigned lastMinor = c_minorProtocolVersion;
  173. if (!status.empty())
  174. DEV_IGNORE_EXCEPTIONS(lastMinor = (unsigned)RLP(status));
  175. if (c_minorProtocolVersion != lastMinor)
  176. {
  177. cnote << "Killing extras database (DB minor version:" << lastMinor << " != our miner version: " << c_minorProtocolVersion << ").";
  178. DEV_IGNORE_EXCEPTIONS(boost::filesystem::remove_all(extrasPath + "/details.old"));
  179. boost::filesystem::rename(extrasPath + "/extras", extrasPath + "/extras.old");
  180. boost::filesystem::remove_all(extrasPath + "/state");
  181. writeFile(extrasPath + "/minor", rlp(c_minorProtocolVersion));
  182. lastMinor = (unsigned)RLP(status);
  183. }
  184. if (_we == WithExisting::Kill)
  185. {
  186. cnote << "Killing blockchain & extras database (WithExisting::Kill).";
  187. boost::filesystem::remove_all(chainPath + "/blocks");
  188. boost::filesystem::remove_all(extrasPath + "/extras");
  189. }
  190. ldb::Options o;
  191. o.create_if_missing = true;
  192. o.max_open_files = 256;
  193. ldb::DB::Open(o, chainPath + "/blocks", &m_blocksDB);
  194. ldb::DB::Open(o, extrasPath + "/extras", &m_extrasDB);
  195. if (!m_blocksDB || !m_extrasDB)
  196. {
  197. if (boost::filesystem::space(chainPath + "/blocks").available < 1024)
  198. {
  199. cwarn << "Not enough available space found on hard drive. Please free some up and then re-run. Bailing.";
  200. BOOST_THROW_EXCEPTION(NotEnoughAvailableSpace());
  201. }
  202. else
  203. {
  204. cwarn <<
  205. "Database " <<
  206. (chainPath + "/blocks") <<
  207. "or " <<
  208. (extrasPath + "/extras") <<
  209. "already open. You appear to have another instance of ethereum running. Bailing.";
  210. BOOST_THROW_EXCEPTION(DatabaseAlreadyOpen());
  211. }
  212. }
  213. // m_writeOptions.sync = true;
  214. if (_we != WithExisting::Verify && !details(m_genesisHash))
  215. {
  216. BlockHeader gb(m_params.genesisBlock());
  217. // Insert details of genesis block.
  218. m_details[m_genesisHash] = BlockDetails(0, gb.difficulty(), h256(), {});
  219. auto r = m_details[m_genesisHash].rlp();
  220. m_extrasDB->Put(m_writeOptions, toSlice(m_genesisHash, ExtraDetails), (ldb::Slice)dev::ref(r));
  221. assert(isKnown(gb.hash()));
  222. }
  223. #if ETH_PARANOIA
  224. checkConsistency();
  225. #endif
  226. // TODO: Implement ability to rebuild details map from DB.
  227. std::string l;
  228. m_extrasDB->Get(m_readOptions, ldb::Slice("best"), &l);
  229. m_lastBlockHash = l.empty() ? m_genesisHash : *(h256*)l.data();
  230. m_lastBlockNumber = number(m_lastBlockHash);
  231. ctrace << "Opened blockchain DB. Latest: " << currentHash() << (lastMinor == c_minorProtocolVersion ? "(rebuild not needed)" : "*** REBUILD NEEDED ***");
  232. return lastMinor;
  233. }
  234. void BlockChain::open(std::string const& _path, WithExisting _we, ProgressCallback const& _pc)
  235. {
  236. if (open(_path, _we) != c_minorProtocolVersion || _we == WithExisting::Verify)
  237. rebuild(_path, _pc);
  238. }
  239. void BlockChain::reopen(ChainParams const& _p, WithExisting _we, ProgressCallback const& _pc)
  240. {
  241. close();
  242. init(_p, m_dbPath);
  243. open(m_dbPath, _we, _pc);
  244. }
  245. void BlockChain::close()
  246. {
  247. ctrace << "Closing blockchain DB";
  248. // Not thread safe...
  249. delete m_extrasDB;
  250. delete m_blocksDB;
  251. m_lastBlockHash = m_genesisHash;
  252. m_lastBlockNumber = 0;
  253. m_details.clear();
  254. m_blocks.clear();
  255. m_logBlooms.clear();
  256. m_receipts.clear();
  257. m_transactionAddresses.clear();
  258. m_blockHashes.clear();
  259. m_blocksBlooms.clear();
  260. m_cacheUsage.clear();
  261. m_inUse.clear();
  262. m_lastLastHashes.clear();
  263. }
  264. void BlockChain::rebuild(std::string const& _path, std::function<void(unsigned, unsigned)> const& _progress)
  265. {
  266. string path = _path.empty() ? Defaults::get()->m_dbPath : _path;
  267. string chainPath = path + "/" + toHex(m_genesisHash.ref().cropped(0, 4));
  268. string extrasPath = chainPath + "/" + toString(c_databaseVersion);
  269. #if ETH_PROFILING_GPERF
  270. ProfilerStart("BlockChain_rebuild.log");
  271. #endif
  272. unsigned originalNumber = m_lastBlockNumber;
  273. ///////////////////////////////
  274. // TODO
  275. // - KILL ALL STATE/CHAIN
  276. // - REINSERT ALL BLOCKS
  277. ///////////////////////////////
  278. // Keep extras DB around, but under a temp name
  279. delete m_extrasDB;
  280. m_extrasDB = nullptr;
  281. boost::filesystem::rename(extrasPath + "/extras", extrasPath + "/extras.old");
  282. ldb::DB* oldExtrasDB;
  283. ldb::Options o;
  284. o.create_if_missing = true;
  285. ldb::DB::Open(o, extrasPath + "/extras.old", &oldExtrasDB);
  286. ldb::DB::Open(o, extrasPath + "/extras", &m_extrasDB);
  287. // Open a fresh state DB
  288. Block s = genesisBlock(State::openDB(path, m_genesisHash, WithExisting::Kill));
  289. // Clear all memos ready for replay.
  290. m_details.clear();
  291. m_logBlooms.clear();
  292. m_receipts.clear();
  293. m_transactionAddresses.clear();
  294. m_blockHashes.clear();
  295. m_blocksBlooms.clear();
  296. m_lastLastHashes.clear();
  297. m_lastBlockHash = genesisHash();
  298. m_lastBlockNumber = 0;
  299. m_details[m_lastBlockHash].totalDifficulty = s.info().difficulty();
  300. m_extrasDB->Put(m_writeOptions, toSlice(m_lastBlockHash, ExtraDetails), (ldb::Slice)dev::ref(m_details[m_lastBlockHash].rlp()));
  301. h256 lastHash = m_lastBlockHash;
  302. Timer t;
  303. for (unsigned d = 1; d <= originalNumber; ++d)
  304. {
  305. if (!(d % 1000))
  306. {
  307. cerr << "\n1000 blocks in " << t.elapsed() << "s = " << (1000.0 / t.elapsed()) << "b/s" << endl;
  308. t.restart();
  309. }
  310. try
  311. {
  312. bytes b = block(queryExtras<BlockHash, uint64_t, ExtraBlockHash>(d, m_blockHashes, x_blockHashes, NullBlockHash, oldExtrasDB).value);
  313. BlockHeader bi(&b);
  314. if (bi.parentHash() != lastHash)
  315. {
  316. cwarn << "DISJOINT CHAIN DETECTED; " << bi.hash() << "#" << d << " -> parent is" << bi.parentHash() << "; expected" << lastHash << "#" << (d - 1);
  317. return;
  318. }
  319. lastHash = bi.hash();
  320. import(b, s.db(), 0);
  321. }
  322. catch (...)
  323. {
  324. // Failed to import - stop here.
  325. break;
  326. }
  327. if (_progress)
  328. _progress(d, originalNumber);
  329. }
  330. #if ETH_PROFILING_GPERF
  331. ProfilerStop();
  332. #endif
  333. delete oldExtrasDB;
  334. boost::filesystem::remove_all(path + "/extras.old");
  335. }
  336. string BlockChain::dumpDatabase() const
  337. {
  338. stringstream ss;
  339. ss << m_lastBlockHash << endl;
  340. ldb::Iterator* i = m_extrasDB->NewIterator(m_readOptions);
  341. for (i->SeekToFirst(); i->Valid(); i->Next())
  342. ss << toHex(bytesConstRef(i->key())) << "/" << toHex(bytesConstRef(i->value())) << endl;
  343. return ss.str();
  344. }
  345. LastHashes BlockChain::lastHashes(h256 const& _parent) const
  346. {
  347. Guard l(x_lastLastHashes);
  348. if (m_lastLastHashes.empty() || m_lastLastHashes.back() != _parent)
  349. {
  350. m_lastLastHashes.resize(256);
  351. m_lastLastHashes[0] = _parent;
  352. for (unsigned i = 0; i < 255; ++i)
  353. m_lastLastHashes[i + 1] = m_lastLastHashes[i] ? info(m_lastLastHashes[i]).parentHash() : h256();
  354. }
  355. return m_lastLastHashes;
  356. }
  357. tuple<ImportRoute, bool, unsigned> BlockChain::sync(BlockQueue& _bq, OverlayDB const& _stateDB, unsigned _max)
  358. {
  359. // _bq.tick(*this);
  360. VerifiedBlocks blocks;
  361. _bq.drain(blocks, _max);
  362. h256s fresh;
  363. h256s dead;
  364. h256s badBlocks;
  365. Transactions goodTransactions;
  366. unsigned count = 0;
  367. for (VerifiedBlock const& block: blocks)
  368. {
  369. do {
  370. try
  371. {
  372. // Nonce & uncle nonces already verified in verification thread at this point.
  373. ImportRoute r;
  374. DEV_TIMED_ABOVE("Block import " + toString(block.verified.info.number()), 500)
  375. r = import(block.verified, _stateDB, (ImportRequirements::Everything & ~ImportRequirements::ValidSeal & ~ImportRequirements::CheckUncles) != 0);
  376. fresh += r.liveBlocks;
  377. dead += r.deadBlocks;
  378. goodTransactions.reserve(goodTransactions.size() + r.goodTranactions.size());
  379. std::move(std::begin(r.goodTranactions), std::end(r.goodTranactions), std::back_inserter(goodTransactions));
  380. ++count;
  381. }
  382. catch (dev::eth::UnknownParent& e)
  383. {
  384. cwarn << "ODD: Import queue contains block with unknown parent.";// << LogTag::Error << boost::current_exception_diagnostic_information();
  385. // NOTE: don't reimport since the queue should guarantee everything in the right order.
  386. // Can't continue - chain bad.
  387. badBlocks.push_back(block.verified.info.hash());
  388. }
  389. catch (dev::eth::FutureTime& e)
  390. {
  391. cwarn << "ODD: Import queue contains a block with future time.";
  392. this_thread::sleep_for(chrono::seconds(1));
  393. continue;
  394. }
  395. catch (dev::eth::TransientError& e)
  396. {
  397. this_thread::sleep_for(chrono::milliseconds(100));
  398. continue;
  399. }
  400. catch (Exception& ex)
  401. {
  402. // cnote << "Exception while importing block. Someone (Jeff? That you?) seems to be giving us dodgy blocks!";// << LogTag::Error << diagnostic_information(ex);
  403. if (m_onBad)
  404. m_onBad(ex);
  405. // NOTE: don't reimport since the queue should guarantee everything in the right order.
  406. // Can't continue - chain bad.
  407. badBlocks.push_back(block.verified.info.hash());
  408. }
  409. } while (false);
  410. }
  411. return make_tuple(ImportRoute{dead, fresh, goodTransactions}, _bq.doneDrain(badBlocks), count);
  412. }
  413. pair<ImportResult, ImportRoute> BlockChain::attemptImport(bytes const& _block, OverlayDB const& _stateDB, bool _mustBeNew) noexcept
  414. {
  415. try
  416. {
  417. return make_pair(ImportResult::Success, import(verifyBlock(&_block, m_onBad, ImportRequirements::OutOfOrderChecks), _stateDB, _mustBeNew));
  418. }
  419. catch (UnknownParent&)
  420. {
  421. return make_pair(ImportResult::UnknownParent, ImportRoute());
  422. }
  423. catch (AlreadyHaveBlock&)
  424. {
  425. return make_pair(ImportResult::AlreadyKnown, ImportRoute());
  426. }
  427. catch (FutureTime&)
  428. {
  429. return make_pair(ImportResult::FutureTimeKnown, ImportRoute());
  430. }
  431. catch (Exception& ex)
  432. {
  433. if (m_onBad)
  434. m_onBad(ex);
  435. return make_pair(ImportResult::Malformed, ImportRoute());
  436. }
  437. }
  438. ImportRoute BlockChain::import(bytes const& _block, OverlayDB const& _db, bool _mustBeNew)
  439. {
  440. // VERIFY: populates from the block and checks the block is internally coherent.
  441. VerifiedBlockRef block;
  442. #if ETH_CATCH
  443. try
  444. #endif
  445. {
  446. block = verifyBlock(&_block, m_onBad, ImportRequirements::OutOfOrderChecks);
  447. }
  448. #if ETH_CATCH
  449. catch (Exception& ex)
  450. {
  451. // clog(BlockChainNote) << " Malformed block: " << diagnostic_information(ex);
  452. ex << errinfo_phase(2);
  453. ex << errinfo_now(time(0));
  454. throw;
  455. }
  456. #endif
  457. return import(block, _db, _mustBeNew);
  458. }
  459. void BlockChain::insert(bytes const& _block, bytesConstRef _receipts, bool _mustBeNew)
  460. {
  461. // VERIFY: populates from the block and checks the block is internally coherent.
  462. VerifiedBlockRef block;
  463. #if ETH_CATCH
  464. try
  465. #endif
  466. {
  467. block = verifyBlock(&_block, m_onBad, ImportRequirements::OutOfOrderChecks);
  468. }
  469. #if ETH_CATCH
  470. catch (Exception& ex)
  471. {
  472. // clog(BlockChainNote) << " Malformed block: " << diagnostic_information(ex);
  473. ex << errinfo_phase(2);
  474. ex << errinfo_now(time(0));
  475. throw;
  476. }
  477. #endif
  478. insert(block, _receipts, _mustBeNew);
  479. }
  480. void BlockChain::insert(VerifiedBlockRef _block, bytesConstRef _receipts, bool _mustBeNew)
  481. {
  482. // Check block doesn't already exist first!
  483. if (isKnown(_block.info.hash()) && _mustBeNew)
  484. {
  485. clog(BlockChainNote) << _block.info.hash() << ": Not new.";
  486. BOOST_THROW_EXCEPTION(AlreadyHaveBlock());
  487. }
  488. // Work out its number as the parent's number + 1
  489. if (!isKnown(_block.info.parentHash(), false))
  490. {
  491. clog(BlockChainNote) << _block.info.hash() << ": Unknown parent " << _block.info.parentHash();
  492. // We don't know the parent (yet) - discard for now. It'll get resent to us if we find out about its ancestry later on.
  493. BOOST_THROW_EXCEPTION(UnknownParent());
  494. }
  495. // Check receipts
  496. vector<bytesConstRef> receipts;
  497. for (auto i: RLP(_receipts))
  498. receipts.push_back(i.data());
  499. h256 receiptsRoot = orderedTrieRoot(receipts);
  500. if (_block.info.receiptsRoot() != receiptsRoot)
  501. {
  502. clog(BlockChainNote) << _block.info.hash() << ": Invalid receipts root " << _block.info.receiptsRoot() << " not " << receiptsRoot;
  503. // We don't know the parent (yet) - discard for now. It'll get resent to us if we find out about its ancestry later on.
  504. BOOST_THROW_EXCEPTION(InvalidReceiptsStateRoot());
  505. }
  506. auto pd = details(_block.info.parentHash());
  507. if (!pd)
  508. {
  509. auto pdata = pd.rlp();
  510. clog(BlockChainDebug) << "Details is returning false despite block known:" << RLP(pdata);
  511. auto parentBlock = block(_block.info.parentHash());
  512. clog(BlockChainDebug) << "isKnown:" << isKnown(_block.info.parentHash());
  513. clog(BlockChainDebug) << "last/number:" << m_lastBlockNumber << m_lastBlockHash << _block.info.number();
  514. clog(BlockChainDebug) << "Block:" << BlockHeader(&parentBlock);
  515. clog(BlockChainDebug) << "RLP:" << RLP(parentBlock);
  516. clog(BlockChainDebug) << "DATABASE CORRUPTION: CRITICAL FAILURE";
  517. exit(-1);
  518. }
  519. // Check it's not crazy
  520. if (_block.info.timestamp() > utcTime() && !m_params.otherParams.count("allowFutureBlocks"))
  521. {
  522. clog(BlockChainChat) << _block.info.hash() << ": Future time " << _block.info.timestamp() << " (now at " << utcTime() << ")";
  523. // Block has a timestamp in the future. This is no good.
  524. BOOST_THROW_EXCEPTION(FutureTime());
  525. }
  526. // Verify parent-critical parts
  527. verifyBlock(_block.block, m_onBad, ImportRequirements::InOrderChecks);
  528. // OK - we're happy. Insert into database.
  529. ldb::WriteBatch blocksBatch;
  530. ldb::WriteBatch extrasBatch;
  531. BlockLogBlooms blb;
  532. for (auto i: RLP(_receipts))
  533. blb.blooms.push_back(TransactionReceipt(i.data()).bloom());
  534. // ensure parent is cached for later addition.
  535. // TODO: this is a bit horrible would be better refactored into an enveloping UpgradableGuard
  536. // together with an "ensureCachedWithUpdatableLock(l)" method.
  537. // This is safe in practice since the caches don't get flushed nearly often enough to be
  538. // done here.
  539. details(_block.info.parentHash());
  540. DEV_WRITE_GUARDED(x_details)
  541. {
  542. if (!dev::contains(m_details[_block.info.parentHash()].children, _block.info.hash()))
  543. m_details[_block.info.parentHash()].children.push_back(_block.info.hash());
  544. }
  545. blocksBatch.Put(toSlice(_block.info.hash()), ldb::Slice(_block.block));
  546. DEV_READ_GUARDED(x_details)
  547. extrasBatch.Put(toSlice(_block.info.parentHash(), ExtraDetails), (ldb::Slice)dev::ref(m_details[_block.info.parentHash()].rlp()));
  548. BlockDetails bd((unsigned)pd.number + 1, pd.totalDifficulty + _block.info.difficulty(), _block.info.parentHash(), {});
  549. extrasBatch.Put(toSlice(_block.info.hash(), ExtraDetails), (ldb::Slice)dev::ref(bd.rlp()));
  550. extrasBatch.Put(toSlice(_block.info.hash(), ExtraLogBlooms), (ldb::Slice)dev::ref(blb.rlp()));
  551. extrasBatch.Put(toSlice(_block.info.hash(), ExtraReceipts), (ldb::Slice)_receipts);
  552. ldb::Status o = m_blocksDB->Write(m_writeOptions, &blocksBatch);
  553. if (!o.ok())
  554. {
  555. cwarn << "Error writing to blockchain database: " << o.ToString();
  556. WriteBatchNoter n;
  557. blocksBatch.Iterate(&n);
  558. cwarn << "Fail writing to blockchain database. Bombing out.";
  559. exit(-1);
  560. }
  561. o = m_extrasDB->Write(m_writeOptions, &extrasBatch);
  562. if (!o.ok())
  563. {
  564. cwarn << "Error writing to extras database: " << o.ToString();
  565. WriteBatchNoter n;
  566. extrasBatch.Iterate(&n);
  567. cwarn << "Fail writing to extras database. Bombing out.";
  568. exit(-1);
  569. }
  570. }
  571. ImportRoute BlockChain::import(VerifiedBlockRef const& _block, OverlayDB const& _db, bool _mustBeNew)
  572. {
  573. //@tidy This is a behemoth of a method - could do to be split into a few smaller ones.
  574. #if ETH_TIMED_IMPORTS
  575. Timer total;
  576. double preliminaryChecks;
  577. double enactment;
  578. double collation;
  579. double writing;
  580. double checkBest;
  581. Timer t;
  582. #endif
  583. // Check block doesn't already exist first!
  584. if (isKnown(_block.info.hash()) && _mustBeNew)
  585. {
  586. clog(BlockChainNote) << _block.info.hash() << ": Not new.";
  587. BOOST_THROW_EXCEPTION(AlreadyHaveBlock());
  588. }
  589. // Work out its number as the parent's number + 1
  590. if (!isKnown(_block.info.parentHash(), false)) // doesn't have to be current.
  591. {
  592. clog(BlockChainNote) << _block.info.hash() << ": Unknown parent " << _block.info.parentHash();
  593. // We don't know the parent (yet) - discard for now. It'll get resent to us if we find out about its ancestry later on.
  594. BOOST_THROW_EXCEPTION(UnknownParent() << errinfo_hash256(_block.info.parentHash()));
  595. }
  596. auto pd = details(_block.info.parentHash());
  597. if (!pd)
  598. {
  599. auto pdata = pd.rlp();
  600. clog(BlockChainDebug) << "Details is returning false despite block known:" << RLP(pdata);
  601. auto parentBlock = block(_block.info.parentHash());
  602. clog(BlockChainDebug) << "isKnown:" << isKnown(_block.info.parentHash());
  603. clog(BlockChainDebug) << "last/number:" << m_lastBlockNumber << m_lastBlockHash << _block.info.number();
  604. clog(BlockChainDebug) << "Block:" << BlockHeader(&parentBlock);
  605. clog(BlockChainDebug) << "RLP:" << RLP(parentBlock);
  606. clog(BlockChainDebug) << "DATABASE CORRUPTION: CRITICAL FAILURE";
  607. exit(-1);
  608. }
  609. // Check it's not crazy
  610. if (_block.info.timestamp() > utcTime() && !m_params.otherParams.count("allowFutureBlocks"))
  611. {
  612. clog(BlockChainChat) << _block.info.hash() << ": Future time " << _block.info.timestamp() << " (now at " << utcTime() << ")";
  613. // Block has a timestamp in the future. This is no good.
  614. BOOST_THROW_EXCEPTION(FutureTime());
  615. }
  616. // Verify parent-critical parts
  617. verifyBlock(_block.block, m_onBad, ImportRequirements::InOrderChecks);
  618. clog(BlockChainChat) << "Attempting import of " << _block.info.hash() << "...";
  619. #if ETH_TIMED_IMPORTS
  620. preliminaryChecks = t.elapsed();
  621. t.restart();
  622. #endif
  623. ldb::WriteBatch blocksBatch;
  624. ldb::WriteBatch extrasBatch;
  625. h256 newLastBlockHash = currentHash();
  626. unsigned newLastBlockNumber = number();
  627. BlockLogBlooms blb;
  628. BlockReceipts br;
  629. u256 td;
  630. Transactions goodTransactions;
  631. #if ETH_CATCH
  632. try
  633. #endif
  634. {
  635. // Check transactions are valid and that they result in a state equivalent to our state_root.
  636. // Get total difficulty increase and update state, checking it.
  637. Block s(*this, _db);
  638. auto tdIncrease = s.enactOn(_block, *this);
  639. for (unsigned i = 0; i < s.pending().size(); ++i)
  640. {
  641. blb.blooms.push_back(s.receipt(i).bloom());
  642. br.receipts.push_back(s.receipt(i));
  643. goodTransactions.push_back(s.pending()[i]);
  644. }
  645. s.cleanup(true);
  646. td = pd.totalDifficulty + tdIncrease;
  647. #if ETH_TIMED_IMPORTS
  648. enactment = t.elapsed();
  649. t.restart();
  650. #endif // ETH_TIMED_IMPORTS
  651. #if ETH_PARANOIA
  652. checkConsistency();
  653. #endif // ETH_PARANOIA
  654. // All ok - insert into DB
  655. // ensure parent is cached for later addition.
  656. // TODO: this is a bit horrible would be better refactored into an enveloping UpgradableGuard
  657. // together with an "ensureCachedWithUpdatableLock(l)" method.
  658. // This is safe in practice since the caches don't get flushed nearly often enough to be
  659. // done here.
  660. details(_block.info.parentHash());
  661. DEV_WRITE_GUARDED(x_details)
  662. m_details[_block.info.parentHash()].children.push_back(_block.info.hash());
  663. #if ETH_TIMED_IMPORTS
  664. collation = t.elapsed();
  665. t.restart();
  666. #endif // ETH_TIMED_IMPORTS
  667. blocksBatch.Put(toSlice(_block.info.hash()), ldb::Slice(_block.block));
  668. DEV_READ_GUARDED(x_details)
  669. extrasBatch.Put(toSlice(_block.info.parentHash(), ExtraDetails), (ldb::Slice)dev::ref(m_details[_block.info.parentHash()].rlp()));
  670. extrasBatch.Put(toSlice(_block.info.hash(), ExtraDetails), (ldb::Slice)dev::ref(BlockDetails((unsigned)pd.number + 1, td, _block.info.parentHash(), {}).rlp()));
  671. extrasBatch.Put(toSlice(_block.info.hash(), ExtraLogBlooms), (ldb::Slice)dev::ref(blb.rlp()));
  672. extrasBatch.Put(toSlice(_block.info.hash(), ExtraReceipts), (ldb::Slice)dev::ref(br.rlp()));
  673. #if ETH_TIMED_IMPORTS
  674. writing = t.elapsed();
  675. t.restart();
  676. #endif // ETH_TIMED_IMPORTS
  677. }
  678. #if ETH_CATCH
  679. catch (BadRoot& ex)
  680. {
  681. cwarn << "*** BadRoot error! Trying to import" << _block.info.hash() << "needed root" << ex.root;
  682. cwarn << _block.info;
  683. // Attempt in import later.
  684. BOOST_THROW_EXCEPTION(TransientError());
  685. }
  686. catch (Exception& ex)
  687. {
  688. ex << errinfo_now(time(0));
  689. ex << errinfo_block(_block.block.toBytes());
  690. // only populate extraData if we actually managed to extract it. otherwise,
  691. // we might be clobbering the existing one.
  692. if (!_block.info.extraData().empty())
  693. ex << errinfo_extraData(_block.info.extraData());
  694. throw;
  695. }
  696. #endif // ETH_CATCH
  697. h256s route;
  698. h256 common;
  699. // This might be the new best block...
  700. h256 last = currentHash();
  701. if (td > details(last).totalDifficulty || (m_sealEngine->chainParams().tieBreakingGas && td == details(last).totalDifficulty && _block.info.gasUsed() > info(last).gasUsed()))
  702. {
  703. // don't include bi.hash() in treeRoute, since it's not yet in details DB...
  704. // just tack it on afterwards.
  705. unsigned commonIndex;
  706. tie(route, common, commonIndex) = treeRoute(last, _block.info.parentHash());
  707. route.push_back(_block.info.hash());
  708. // Most of the time these two will be equal - only when we're doing a chain revert will they not be
  709. if (common != last)
  710. clearCachesDuringChainReversion(number(common) + 1);
  711. // Go through ret backwards (i.e. from new head to common) until hash != last.parent and
  712. // update m_transactionAddresses, m_blockHashes
  713. for (auto i = route.rbegin(); i != route.rend() && *i != common; ++i)
  714. {
  715. BlockHeader tbi;
  716. if (*i == _block.info.hash())
  717. tbi = _block.info;
  718. else
  719. tbi = BlockHeader(block(*i));
  720. // Collate logs into blooms.
  721. h256s alteredBlooms;
  722. {
  723. LogBloom blockBloom = tbi.logBloom();
  724. blockBloom.shiftBloom<3>(sha3(tbi.author().ref()));
  725. // Pre-memoize everything we need before locking x_blocksBlooms
  726. for (unsigned level = 0, index = (unsigned)tbi.number(); level < c_bloomIndexLevels; level++, index /= c_bloomIndexSize)
  727. blocksBlooms(chunkId(level, index / c_bloomIndexSize));
  728. WriteGuard l(x_blocksBlooms);
  729. for (unsigned level = 0, index = (unsigned)tbi.number(); level < c_bloomIndexLevels; level++, index /= c_bloomIndexSize)
  730. {
  731. unsigned i = index / c_bloomIndexSize;
  732. unsigned o = index % c_bloomIndexSize;
  733. alteredBlooms.push_back(chunkId(level, i));
  734. m_blocksBlooms[alteredBlooms.back()].blooms[o] |= blockBloom;
  735. }
  736. }
  737. // Collate transaction hashes and remember who they were.
  738. //h256s newTransactionAddresses;
  739. {
  740. bytes blockBytes;
  741. RLP blockRLP(*i == _block.info.hash() ? _block.block : &(blockBytes = block(*i)));
  742. TransactionAddress ta;
  743. ta.blockHash = tbi.hash();
  744. for (ta.index = 0; ta.index < blockRLP[1].itemCount(); ++ta.index)
  745. extrasBatch.Put(toSlice(sha3(blockRLP[1][ta.index].data()), ExtraTransactionAddress), (ldb::Slice)dev::ref(ta.rlp()));
  746. }
  747. // Update database with them.
  748. ReadGuard l1(x_blocksBlooms);
  749. for (auto const& h: alteredBlooms)
  750. extrasBatch.Put(toSlice(h, ExtraBlocksBlooms), (ldb::Slice)dev::ref(m_blocksBlooms[h].rlp()));
  751. extrasBatch.Put(toSlice(h256(tbi.number()), ExtraBlockHash), (ldb::Slice)dev::ref(BlockHash(tbi.hash()).rlp()));
  752. }
  753. // FINALLY! change our best hash.
  754. {
  755. newLastBlockHash = _block.info.hash();
  756. newLastBlockNumber = (unsigned)_block.info.number();
  757. }
  758. clog(BlockChainNote) << " Imported and best" << td << " (#" << _block.info.number() << "). Has" << (details(_block.info.parentHash()).children.size() - 1) << "siblings. Route:" << route;
  759. }
  760. else
  761. {
  762. clog(BlockChainChat) << " Imported but not best (oTD:" << details(last).totalDifficulty << " > TD:" << td << "; " << details(last).number << ".." << _block.info.number() << ")";
  763. }
  764. ldb::Status o = m_blocksDB->Write(m_writeOptions, &blocksBatch);
  765. if (!o.ok())
  766. {
  767. cwarn << "Error writing to blockchain database: " << o.ToString();
  768. WriteBatchNoter n;
  769. blocksBatch.Iterate(&n);
  770. cwarn << "Fail writing to blockchain database. Bombing out.";
  771. exit(-1);
  772. }
  773. o = m_extrasDB->Write(m_writeOptions, &extrasBatch);
  774. if (!o.ok())
  775. {
  776. cwarn << "Error writing to extras database: " << o.ToString();
  777. WriteBatchNoter n;
  778. extrasBatch.Iterate(&n);
  779. cwarn << "Fail writing to extras database. Bombing out.";
  780. exit(-1);
  781. }
  782. #if ETH_PARANOIA
  783. if (isKnown(_block.info.hash()) && !details(_block.info.hash()))
  784. {
  785. clog(BlockChainDebug) << "Known block just inserted has no details.";
  786. clog(BlockChainDebug) << "Block:" << _block.info;
  787. clog(BlockChainDebug) << "DATABASE CORRUPTION: CRITICAL FAILURE";
  788. exit(-1);
  789. }
  790. try
  791. {
  792. State canary(_db, BaseState::Empty);
  793. canary.populateFromChain(*this, _block.info.hash());
  794. }
  795. catch (...)
  796. {
  797. clog(BlockChainDebug) << "Failed to initialise State object form imported block.";
  798. clog(BlockChainDebug) << "Block:" << _block.info;
  799. clog(BlockChainDebug) << "DATABASE CORRUPTION: CRITICAL FAILURE";
  800. exit(-1);
  801. }
  802. #endif // ETH_PARANOIA
  803. if (m_lastBlockHash != newLastBlockHash)
  804. DEV_WRITE_GUARDED(x_lastBlockHash)
  805. {
  806. m_lastBlockHash = newLastBlockHash;
  807. m_lastBlockNumber = newLastBlockNumber;
  808. o = m_extrasDB->Put(m_writeOptions, ldb::Slice("best"), ldb::Slice((char const*)&m_lastBlockHash, 32));
  809. if (!o.ok())
  810. {
  811. cwarn << "Error writing to extras database: " << o.ToString();
  812. cout << "Put" << toHex(bytesConstRef(ldb::Slice("best"))) << "=>" << toHex(bytesConstRef(ldb::Slice((char const*)&m_lastBlockHash, 32)));
  813. cwarn << "Fail writing to extras database. Bombing out.";
  814. exit(-1);
  815. }
  816. }
  817. #if ETH_PARANOIA
  818. checkConsistency();
  819. #endif // ETH_PARANOIA
  820. #if ETH_TIMED_IMPORTS
  821. checkBest = t.elapsed();
  822. if (total.elapsed() > 0.5)
  823. {
  824. unsigned const gasPerSecond = static_cast<double>(_block.info.gasUsed()) / enactment;
  825. cnote << "SLOW IMPORT: "
  826. << "{ \"blockHash\": \"" << _block.info.hash() << "\", "
  827. << "\"blockNumber\": " << _block.info.number() << ", "
  828. << "\"importTime\": " << total.elapsed() << ", "
  829. << "\"gasPerSecond\": " << gasPerSecond << ", "
  830. << "\"preliminaryChecks\":" << preliminaryChecks << ", "
  831. << "\"enactment\":" << enactment << ", "
  832. << "\"collation\":" << collation << ", "
  833. << "\"writing\":" << writing << ", "
  834. << "\"checkBest\":" << checkBest << ", "
  835. << "\"transactions\":" << _block.transactions.size() << ", "
  836. << "\"gasUsed\":" << _block.info.gasUsed() << " }";
  837. }
  838. #endif // ETH_TIMED_IMPORTS
  839. if (!route.empty())
  840. noteCanonChanged();
  841. h256s fresh;
  842. h256s dead;
  843. bool isOld = true;
  844. for (auto const& h: route)
  845. if (h == common)
  846. isOld = false;
  847. else if (isOld)
  848. dead.push_back(h);
  849. else
  850. fresh.push_back(h);
  851. return ImportRoute{dead, fresh, move(goodTransactions)};
  852. }
  853. void BlockChain::clearBlockBlooms(unsigned _begin, unsigned _end)
  854. {
  855. // ... c c c c c c c c c c C o o o o o o
  856. // ... /=15 /=21
  857. // L0...| ' | ' | ' | ' | ' | ' | ' | 'b|x'x|x'x|x'e| /=11
  858. // L1...| ' | ' | ' | ' b | x ' x | x ' e | /=6
  859. // L2...| ' | ' b | x ' x | e /=3
  860. // L3...| ' b | x ' e
  861. // model: c_bloomIndexLevels = 4, c_bloomIndexSize = 2
  862. // ... /=15 /=21
  863. // L0...| ' ' ' | ' ' ' | ' ' ' | ' ' 'b|x'x'x'x|x'e' ' |
  864. // L1...| ' ' ' b | x ' x ' e ' |
  865. // L2...| b ' x ' e ' |
  866. // model: c_bloomIndexLevels = 2, c_bloomIndexSize = 4
  867. // algorithm doesn't have the best memoisation coherence, but eh well...
  868. unsigned beginDirty = _begin;
  869. unsigned endDirty = _end;
  870. for (unsigned level = 0; level < c_bloomIndexLevels; level++, beginDirty /= c_bloomIndexSize, endDirty = (endDirty - 1) / c_bloomIndexSize + 1)
  871. {
  872. // compute earliest & latest index for each level, rebuild from previous levels.
  873. for (unsigned item = beginDirty; item != endDirty; ++item)
  874. {
  875. unsigned bunch = item / c_bloomIndexSize;
  876. unsigned offset = item % c_bloomIndexSize;
  877. auto id = chunkId(level, bunch);
  878. LogBloom acc;
  879. if (!!level)
  880. {
  881. // rebuild the bloom from the previous (lower) level (if there is one).
  882. auto lowerChunkId = chunkId(level - 1, item);
  883. for (auto const& bloom: blocksBlooms(lowerChunkId).blooms)
  884. acc |= bloom;
  885. }
  886. blocksBlooms(id); // make sure it has been memoized.
  887. m_blocksBlooms[id].blooms[offset] = acc;
  888. }
  889. }
  890. }
  891. void BlockChain::rescue(OverlayDB const& _db)
  892. {
  893. cout << "Rescuing database..." << endl;
  894. unsigned u = 1;
  895. while (true)
  896. {
  897. try {
  898. if (isKnown(numberHash(u)))
  899. u *= 2;
  900. else
  901. break;
  902. }
  903. catch (...)
  904. {
  905. break;
  906. }
  907. }
  908. unsigned l = u / 2;
  909. cout << "Finding last likely block number..." << endl;
  910. while (u - l > 1)
  911. {
  912. unsigned m = (u + l) / 2;
  913. cout << " " << m << flush;
  914. if (isKnown(numberHash(m)))
  915. l = m;
  916. else
  917. u = m;
  918. }
  919. cout << " lowest is " << l << endl;
  920. for (; l > 0; --l)
  921. {
  922. h256 h = numberHash(l);
  923. cout << "Checking validity of " << l << " (" << h << ")..." << flush;
  924. try
  925. {
  926. cout << "block..." << flush;
  927. BlockHeader bi(block(h));
  928. cout << "extras..." << flush;
  929. details(h);
  930. cout << "state..." << flush;
  931. if (_db.exists(bi.stateRoot()))
  932. break;
  933. }
  934. catch (...) {}
  935. }
  936. cout << "OK." << endl;
  937. rewind(l);
  938. }
  939. void BlockChain::rewind(unsigned _newHead)
  940. {
  941. DEV_WRITE_GUARDED(x_lastBlockHash)
  942. {
  943. if (_newHead >= m_lastBlockNumber)
  944. return;
  945. clearCachesDuringChainReversion(_newHead + 1);
  946. m_lastBlockHash = numberHash(_newHead);
  947. m_lastBlockNumber = _newHead;
  948. auto o = m_extrasDB->Put(m_writeOptions, ldb::Slice("best"), ldb::Slice((char const*)&m_lastBlockHash, 32));
  949. if (!o.ok())
  950. {
  951. cwarn << "Error writing to extras database: " << o.ToString();
  952. cout << "Put" << toHex(bytesConstRef(ldb::Slice("best"))) << "=>" << toHex(bytesConstRef(ldb::Slice((char const*)&m_lastBlockHash, 32)));
  953. cwarn << "Fail writing to extras database. Bombing out.";
  954. exit(-1);
  955. }
  956. noteCanonChanged();
  957. }
  958. }
  959. tuple<h256s, h256, unsigned> BlockChain::treeRoute(h256 const& _from, h256 const& _to, bool _common, bool _pre, bool _post) const
  960. {
  961. // cdebug << "treeRoute" << _from << "..." << _to;
  962. if (!_from || !_to)
  963. return make_tuple(h256s(), h256(), 0);
  964. h256s ret;
  965. h256s back;
  966. unsigned fn = details(_from).number;
  967. unsigned tn = details(_to).number;
  968. // cdebug << "treeRoute" << fn << "..." << tn;
  969. h256 from = _from;
  970. while (fn > tn)
  971. {
  972. if (_pre)
  973. ret.push_back(from);
  974. from = details(from).parent;
  975. fn--;
  976. // cdebug << "from:" << fn << _from;
  977. }
  978. h256 to = _to;
  979. while (fn < tn)
  980. {
  981. if (_post)
  982. back.push_back(to);
  983. to = details(to).parent;
  984. tn--;
  985. // cdebug << "to:" << tn << _to;
  986. }
  987. for (;; from = details(from).parent, to = details(to).parent)
  988. {
  989. if (_pre && (from != to || _common))
  990. ret.push_back(from);
  991. if (_post && (from != to || (!_pre && _common)))
  992. back.push_back(to);
  993. fn--;
  994. tn--;
  995. // cdebug << "from:" << fn << _from << "; to:" << tn << _to;
  996. if (from == to)
  997. break;
  998. if (!from)
  999. assert(from);
  1000. if (!to)
  1001. assert(to);
  1002. }
  1003. ret.reserve(ret.size() + back.size());
  1004. unsigned i = ret.size() - (int)(_common && !ret.empty() && !back.empty());
  1005. for (auto it = back.rbegin(); it != back.rend(); ++it)
  1006. ret.push_back(*it);
  1007. return make_tuple(ret, from, i);
  1008. }
  1009. void BlockChain::noteUsed(h256 const& _h, unsigned _extra) const
  1010. {
  1011. auto id = CacheID(_h, _extra);
  1012. Guard l(x_cacheUsage);
  1013. m_cacheUsage[0].insert(id);
  1014. if (m_cacheUsage[1].count(id))
  1015. m_cacheUsage[1].erase(id);
  1016. else
  1017. m_inUse.insert(id);
  1018. }
  1019. template <class K, class T> static unsigned getHashSize(unordered_map<K, T> const& _map)
  1020. {
  1021. unsigned ret = 0;
  1022. for (auto const& i: _map)
  1023. ret += i.second.size + 64;
  1024. return ret;
  1025. }
  1026. void BlockChain::updateStats() const
  1027. {
  1028. m_lastStats.memBlocks = 0;
  1029. DEV_READ_GUARDED(x_blocks)
  1030. for (auto const& i: m_blocks)
  1031. m_lastStats.memBlocks += i.second.size() + 64;
  1032. DEV_READ_GUARDED(x_details)
  1033. m_lastStats.memDetails = getHashSize(m_details);
  1034. DEV_READ_GUARDED(x_logBlooms)
  1035. DEV_READ_GUARDED(x_blocksBlooms)
  1036. m_lastStats.memLogBlooms = getHashSize(m_logBlooms) + getHashSize(m_blocksBlooms);
  1037. DEV_READ_GUARDED(x_receipts)
  1038. m_lastStats.memReceipts = getHashSize(m_receipts);
  1039. DEV_READ_GUARDED(x_blockHashes)
  1040. m_lastStats.memBlockHashes = getHashSize(m_blockHashes);
  1041. DEV_READ_GUARDED(x_transactionAddresses)
  1042. m_lastStats.memTransactionAddresses = getHashSize(m_transactionAddresses);
  1043. }
  1044. void BlockChain::garbageCollect(bool _force)
  1045. {
  1046. updateStats();
  1047. if (!_force && chrono::system_clock::now() < m_lastCollection + c_collectionDuration && m_lastStats.memTotal() < c_maxCacheSize)
  1048. return;
  1049. if (m_lastStats.memTotal() < c_minCacheSize)
  1050. return;
  1051. m_lastCollection = chrono::system_clock::now();
  1052. Guard l(x_cacheUsage);
  1053. WriteGuard l1(x_blocks);
  1054. WriteGuard l2(x_details);
  1055. WriteGuard l3(x_blockHashes);
  1056. WriteGuard l4(x_receipts);
  1057. WriteGuard l5(x_logBlooms);
  1058. WriteGuard l6(x_transactionAddresses);
  1059. WriteGuard l7(x_blocksBlooms);
  1060. for (CacheID const& id: m_cacheUsage.back())
  1061. {
  1062. m_inUse.erase(id);
  1063. // kill i from cache.
  1064. switch (id.second)
  1065. {
  1066. case (unsigned)-1:
  1067. m_blocks.erase(id.first);
  1068. break;
  1069. case ExtraDetails:
  1070. m_details.erase(id.first);
  1071. break;
  1072. case ExtraReceipts:
  1073. m_receipts.erase(id.first);
  1074. break;
  1075. case ExtraLogBlooms:
  1076. m_logBlooms.erase(id.first);
  1077. break;
  1078. case ExtraTransactionAddress:
  1079. m_transactionAddresses.erase(id.first);
  1080. break;
  1081. case ExtraBlocksBlooms:
  1082. m_blocksBlooms.erase(id.first);
  1083. break;
  1084. }
  1085. }
  1086. m_cacheUsage.pop_back();
  1087. m_cacheUsage.push_front(std::unordered_set<CacheID>{});
  1088. }
  1089. void BlockChain::checkConsistency()
  1090. {
  1091. DEV_WRITE_GUARDED(x_details)
  1092. m_details.clear();
  1093. ldb::Iterator* it = m_blocksDB->NewIterator(m_readOptions);
  1094. for (it->SeekToFirst(); it->Valid(); it->Next())
  1095. if (it->key().size() == 32)
  1096. {
  1097. h256 h((byte const*)it->key().data(), h256::ConstructFromPointer);
  1098. auto dh = details(h);
  1099. auto p = dh.parent;
  1100. if (p != h256() && p != m_genesisHash) // TODO: for some reason the genesis details with the children get squished. not sure why.
  1101. {
  1102. auto dp = details(p);
  1103. if (asserts(contains(dp.children, h)))
  1104. cnote << "Apparently the database is corrupt. Not much we can do at this stage...";
  1105. if (assertsEqual(dp.number, dh.number - 1))
  1106. cnote << "Apparently the database is corrupt. Not much we can do at this stage...";
  1107. }
  1108. }
  1109. delete it;
  1110. }
  1111. void BlockChain::clearCachesDuringChainReversion(unsigned _firstInvalid)
  1112. {
  1113. unsigned end = number() + 1;
  1114. DEV_WRITE_GUARDED(x_blockHashes)
  1115. for (auto i = _firstInvalid; i < end; ++i)
  1116. m_blockHashes.erase(i);
  1117. DEV_WRITE_GUARDED(x_transactionAddresses)
  1118. m_transactionAddresses.clear(); // TODO: could perhaps delete them individually?
  1119. // If we are reverting previous blocks, we need to clear their blooms (in particular, to
  1120. // rebuild any higher level blooms that they contributed to).
  1121. clearBlockBlooms(_firstInvalid, end);
  1122. }
  1123. static inline unsigned upow(unsigned a, unsigned b) { if (!b) return 1; while (--b > 0) a *= a; return a; }
  1124. static inline unsigned ceilDiv(unsigned n, unsigned d) { return (n + d - 1) / d; }
  1125. //static inline unsigned floorDivPow(unsigned n, unsigned a, unsigned b) { return n / upow(a, b); }
  1126. //static inline unsigned ceilDivPow(unsigned n, unsigned a, unsigned b) { return ceilDiv(n, upow(a, b)); }
  1127. // Level 1
  1128. // [xxx. ]
  1129. // Level 0
  1130. // [.x............F.]
  1131. // [........x.......]
  1132. // [T.............x.]
  1133. // [............ ]
  1134. // F = 14. T = 32
  1135. vector<unsigned> BlockChain::withBlockBloom(LogBloom const& _b, unsigned _earliest, unsigned _latest) const
  1136. {
  1137. vector<unsigned> ret;
  1138. // start from the top-level
  1139. unsigned u = upow(c_bloomIndexSize, c_bloomIndexLevels);
  1140. // run through each of the top-level blockbloom blocks
  1141. for (unsigned index = _earliest / u; index <= ceilDiv(_latest, u); ++index) // 0
  1142. ret += withBlockBloom(_b, _earliest, _latest, c_bloomIndexLevels - 1, index);
  1143. return ret;
  1144. }
  1145. vector<unsigned> BlockChain::withBlockBloom(LogBloom const& _b, unsigned _earliest, unsigned _latest, unsigned _level, unsigned _index) const
  1146. {
  1147. // 14, 32, 1, 0
  1148. // 14, 32, 0, 0
  1149. // 14, 32, 0, 1
  1150. // 14, 32, 0, 2
  1151. vector<unsigned> ret;
  1152. unsigned uCourse = upow(c_bloomIndexSize, _level + 1);
  1153. // 256
  1154. // 16
  1155. unsigned uFine = upow(c_bloomIndexSize, _level);
  1156. // 16
  1157. // 1
  1158. unsigned obegin = _index == _earliest / uCourse ? _earliest / uFine % c_bloomIndexSize : 0;
  1159. // 0
  1160. // 14
  1161. // 0
  1162. // 0
  1163. unsigned oend = _index == _latest / uCourse ? (_latest / uFine) % c_bloomIndexSize + 1 : c_bloomIndexSize;
  1164. // 3
  1165. // 16
  1166. // 16
  1167. // 1
  1168. BlocksBlooms bb = blocksBlooms(_level, _index);
  1169. for (unsigned o = obegin; o < oend; ++o)
  1170. if (bb.blooms[o].contains(_b))
  1171. {
  1172. // This level has something like what we want.
  1173. if (_level > 0)
  1174. ret += withBlockBloom(_b, _earliest, _latest, _level - 1, o + _index * c_bloomIndexSize);
  1175. else
  1176. ret.push_back(o + _index * c_bloomIndexSize);
  1177. }
  1178. return ret;
  1179. }
  1180. h256Hash BlockChain::allKinFrom(h256 const& _parent, unsigned _generations) const
  1181. {
  1182. // Get all uncles cited given a parent (i.e. featured as uncles/main in parent, parent + 1, ... parent + 5).
  1183. h256 p = _parent;
  1184. h256Hash ret = { p };
  1185. // p and (details(p).parent: i == 5) is likely to be overkill, but can't hurt to be cautious.
  1186. for (unsigned i = 0; i < _generations && p != m_genesisHash; ++i, p = details(p).parent)
  1187. {
  1188. ret.insert(details(p).parent);
  1189. auto b = block(p);
  1190. for (auto i: RLP(b)[2])
  1191. ret.insert(sha3(i.data()));
  1192. }
  1193. return ret;
  1194. }
  1195. bool BlockChain::isKnown(h256 const& _hash, bool _isCurrent) const
  1196. {
  1197. if (_hash == m_genesisHash)
  1198. return true;
  1199. DEV_READ_GUARDED(x_blocks)
  1200. if (!m_blocks.count(_hash))
  1201. {
  1202. string d;
  1203. m_blocksDB->Get(m_readOptions, toSlice(_hash), &d);
  1204. if (d.empty())
  1205. return false;
  1206. }
  1207. DEV_READ_GUARDED(x_details)
  1208. if (!m_details.count(_hash))
  1209. {
  1210. string d;
  1211. m_extrasDB->Get(m_readOptions, toSlice(_hash, ExtraDetails), &d);
  1212. if (d.empty())
  1213. return false;
  1214. }
  1215. // return true;
  1216. return !_isCurrent || details(_hash).number <= m_lastBlockNumber; // to allow rewind functionality.
  1217. }
  1218. bytes BlockChain::block(h256 const& _hash) const
  1219. {
  1220. if (_hash == m_genesisHash)
  1221. return m_params.genesisBlock();
  1222. {
  1223. ReadGuard l(x_blocks);
  1224. auto it = m_blocks.find(_hash);
  1225. if (it != m_blocks.end())
  1226. return it->second;
  1227. }
  1228. string d;
  1229. m_blocksDB->Get(m_readOptions, toSlice(_hash), &d);
  1230. if (d.empty())
  1231. {
  1232. cwarn << "Couldn't find requested block:" << _hash;
  1233. return bytes();
  1234. }
  1235. noteUsed(_hash);
  1236. WriteGuard l(x_blocks);
  1237. m_blocks[_hash].resize(d.size());
  1238. memcpy(m_blocks[_hash].data(), d.data(), d.size());
  1239. return m_blocks[_hash];
  1240. }
  1241. bytes BlockChain::headerData(h256 const& _hash) const
  1242. {
  1243. if (_hash == m_genesisHash)
  1244. return m_genesisHeaderBytes;
  1245. {
  1246. ReadGuard l(x_blocks);
  1247. auto it = m_blocks.find(_hash);
  1248. if (it != m_blocks.end())
  1249. return BlockHeader::extractHeader(&it->second).data().toBytes();
  1250. }
  1251. string d;
  1252. m_blocksDB->Get(m_readOptions, toSlice(_hash), &d);
  1253. if (d.empty())
  1254. {
  1255. cwarn << "Couldn't find requested block:" << _hash;
  1256. return bytes();
  1257. }
  1258. noteUsed(_hash);
  1259. WriteGuard l(x_blocks);
  1260. m_blocks[_hash].resize(d.size());
  1261. memcpy(m_blocks[_hash].data(), d.data(), d.size());
  1262. return BlockHeader::extractHeader(&m_blocks[_hash]).data().toBytes();
  1263. }
  1264. Block BlockChain::genesisBlock(OverlayDB const& _db) const
  1265. {
  1266. h256 r = BlockHeader(m_params.genesisBlock()).stateRoot();
  1267. Block ret(*this, _db, BaseState::Empty);
  1268. if (!_db.exists(r))
  1269. {
  1270. ret.noteChain(*this);
  1271. dev::eth::commit(m_params.genesisState, ret.mutableState().m_state); // bit horrible. maybe consider a better way of constructing it?
  1272. ret.mutableState().db().commit(); // have to use this db() since it's the one that has been altered with the above commit.
  1273. if (ret.mutableState().rootHash() != r)
  1274. {
  1275. cwarn << "Hinted genesis block's state root hash is incorrect!";
  1276. cwarn << "Hinted" << r << ", computed" << ret.mutableState().rootHash();
  1277. // TODO: maybe try to fix it by altering the m_params's genesis block?
  1278. exit(-1);
  1279. }
  1280. }
  1281. ret.m_previousBlock = BlockHeader(m_params.genesisBlock());
  1282. ret.resetCurrent();
  1283. return ret;
  1284. }
  1285. VerifiedBlockRef BlockChain::verifyBlock(bytesConstRef _block, std::function<void(Exception&)> const& _onBad, ImportRequirements::value _ir) const
  1286. {
  1287. VerifiedBlockRef res;
  1288. BlockHeader h;
  1289. try
  1290. {
  1291. h = BlockHeader(_block);
  1292. if (!!(_ir & ImportRequirements::PostGenesis) && (!h.parentHash() || h.number() == 0))
  1293. BOOST_THROW_EXCEPTION(InvalidParentHash() << errinfo_required_h256(h.parentHash()) << errinfo_currentNumber(h.number()));
  1294. BlockHeader parent;
  1295. if (!!(_ir & ImportRequirements::Parent))
  1296. {
  1297. bytes parentHeader(headerData(h.parentHash()));
  1298. if (parentHeader.empty())
  1299. BOOST_THROW_EXCEPTION(InvalidParentHash() << errinfo_required_h256(h.parentHash()) << errinfo_currentNumber(h.number()));
  1300. parent = BlockHeader(parentHeader, HeaderData, h.parentHash());
  1301. }
  1302. sealEngine()->verify((_ir & ImportRequirements::ValidSeal) ? Strictness::CheckEverything : Strictness::QuickNonce, h, parent, _block);
  1303. res.info = h;
  1304. }
  1305. catch (Exception& ex)
  1306. {
  1307. ex << errinfo_phase(1);
  1308. ex << errinfo_now(time(0));
  1309. ex << errinfo_block(_block.toBytes());
  1310. // only populate extraData if we actually managed to extract it. otherwise,
  1311. // we might be clobbering the existing one.
  1312. if (!h.extraData().empty())
  1313. ex << errinfo_extraData(h.extraData());
  1314. if (_onBad)
  1315. _onBad(ex);
  1316. throw;
  1317. }
  1318. RLP r(_block);
  1319. unsigned i = 0;
  1320. if (_ir && !!(ImportRequirements::UncleBasic | ImportRequirements::UncleParent | ImportRequirements::UncleSeals))
  1321. for (auto const& uncle: r[2])
  1322. {
  1323. BlockHeader uh(uncle.data(), HeaderData);
  1324. try
  1325. {
  1326. BlockHeader parent;
  1327. if (!!(_ir & ImportRequirements::UncleParent))
  1328. {
  1329. bytes parentHeader(headerData(uh.parentHash()));
  1330. if (parentHeader.empty())
  1331. BOOST_THROW_EXCEPTION(InvalidUncleParentHash() << errinfo_required_h256(uh.parentHash()) << errinfo_currentNumber(h.number()) << errinfo_uncleNumber(uh.number()));
  1332. parent = BlockHeader(parentHeader, HeaderData, uh.parentHash());
  1333. }
  1334. sealEngine()->verify((_ir & ImportRequirements::UncleSeals) ? Strictness::CheckEverything : Strictness::IgnoreSeal, uh, parent);
  1335. }
  1336. catch (Exception& ex)
  1337. {
  1338. ex << errinfo_phase(1);
  1339. ex << errinfo_uncleIndex(i);
  1340. ex << errinfo_now(time(0));
  1341. ex << errinfo_block(_block.toBytes());
  1342. // only populate extraData if we actually managed to extract it. otherwise,
  1343. // we might be clobbering the existing one.
  1344. if (!uh.extraData().empty())
  1345. ex << errinfo_extraData(uh.extraData());
  1346. if (_onBad)
  1347. _onBad(ex);
  1348. throw;
  1349. }
  1350. ++i;
  1351. }
  1352. i = 0;
  1353. if (_ir && !!(ImportRequirements::TransactionBasic | ImportRequirements::TransactionSignatures))
  1354. for (RLP const& tr: r[1])
  1355. {
  1356. bytesConstRef d = tr.data();
  1357. try
  1358. {
  1359. Transaction t(d, (_ir & ImportRequirements::TransactionSignatures) ? CheckTransaction::Everything : CheckTransaction::None);
  1360. m_sealEngine->verifyTransaction(_ir, t, h);
  1361. res.transactions.push_back(t);
  1362. }
  1363. catch (Exception& ex)
  1364. {
  1365. ex << errinfo_phase(1);
  1366. ex << errinfo_transactionIndex(i);
  1367. ex << errinfo_transaction(d.toBytes());
  1368. ex << errinfo_block(_block.toBytes());
  1369. // only populate extraData if we actually managed to extract it. otherwise,
  1370. // we might be clobbering the existing one.
  1371. if (!h.extraData().empty())
  1372. ex << errinfo_extraData(h.extraData());
  1373. if (_onBad)
  1374. _onBad(ex);
  1375. throw;
  1376. }
  1377. ++i;
  1378. }
  1379. res.block = bytesConstRef(_block);
  1380. return res;
  1381. }