framework.ipp 68 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783
  1. // (C) Copyright Gennadiy Rozental 2001.
  2. // Distributed under the Boost Software License, Version 1.0.
  3. // (See accompanying file LICENSE_1_0.txt or copy at
  4. // http://www.boost.org/LICENSE_1_0.txt)
  5. // See http://www.boost.org/libs/test for the library home page.
  6. //
  7. // File : $RCSfile$
  8. //
  9. // Version : $Revision$
  10. //
  11. // Description : implements framework API - main driver for the test
  12. // ***************************************************************************
  13. #ifndef BOOST_TEST_FRAMEWORK_IPP_021005GER
  14. #define BOOST_TEST_FRAMEWORK_IPP_021005GER
  15. // Boost.Test
  16. #include <boost/test/framework.hpp>
  17. #include <boost/test/execution_monitor.hpp>
  18. #include <boost/test/debug.hpp>
  19. #include <boost/test/unit_test_parameters.hpp>
  20. #include <boost/test/unit_test_log.hpp>
  21. #include <boost/test/unit_test_log_formatter.hpp>
  22. #include <boost/test/unit_test_monitor.hpp>
  23. #include <boost/test/results_collector.hpp>
  24. #include <boost/test/progress_monitor.hpp>
  25. #include <boost/test/results_reporter.hpp>
  26. #include <boost/test/test_framework_init_observer.hpp>
  27. #include <boost/test/tree/observer.hpp>
  28. #include <boost/test/tree/test_unit.hpp>
  29. #include <boost/test/tree/visitor.hpp>
  30. #include <boost/test/tree/traverse.hpp>
  31. #include <boost/test/tree/test_case_counter.hpp>
  32. #include <boost/test/tree/global_fixture.hpp>
  33. #if BOOST_TEST_SUPPORT_TOKEN_ITERATOR
  34. #include <boost/test/utils/iterator/token_iterator.hpp>
  35. #endif
  36. #include <boost/test/utils/foreach.hpp>
  37. #include <boost/test/utils/basic_cstring/io.hpp>
  38. #include <boost/test/utils/basic_cstring/compare.hpp>
  39. #include <boost/test/detail/global_typedef.hpp>
  40. #include <boost/test/detail/throw_exception.hpp>
  41. // Boost
  42. #include <boost/test/utils/timer.hpp>
  43. #include <boost/bind.hpp>
  44. // STL
  45. #include <limits>
  46. #include <map>
  47. #include <set>
  48. #include <cstdlib>
  49. #include <ctime>
  50. #include <numeric>
  51. #include <cmath>
  52. #ifdef BOOST_NO_CXX98_RANDOM_SHUFFLE
  53. #include <iterator>
  54. #endif
  55. #ifdef BOOST_NO_STDC_NAMESPACE
  56. namespace std { using ::time; using ::srand; }
  57. #endif
  58. #include <boost/test/detail/suppress_warnings.hpp>
  59. //____________________________________________________________________________//
  60. namespace boost {
  61. namespace unit_test {
  62. namespace framework {
  63. namespace impl {
  64. // ************************************************************************** //
  65. // ************** order detection helpers ************** //
  66. // ************************************************************************** //
  67. struct order_info {
  68. order_info() : depth(-1) {}
  69. int depth;
  70. std::vector<test_unit_id> dependant_siblings;
  71. };
  72. typedef std::set<test_unit_id> tu_id_set;
  73. typedef std::map<test_unit_id,order_info> order_info_per_tu; // !! ?? unordered map
  74. //____________________________________________________________________________//
  75. static test_unit_id
  76. get_tu_parent( test_unit_id tu_id )
  77. {
  78. return framework::get( tu_id, TUT_ANY ).p_parent_id;
  79. }
  80. //____________________________________________________________________________//
  81. static int
  82. tu_depth( test_unit_id tu_id, test_unit_id master_tu_id, order_info_per_tu& tuoi )
  83. {
  84. if( tu_id == master_tu_id )
  85. return 0;
  86. order_info& info = tuoi[tu_id];
  87. if( info.depth == -1 )
  88. info.depth = tu_depth( get_tu_parent( tu_id ), master_tu_id, tuoi ) + 1;
  89. return info.depth;
  90. }
  91. //____________________________________________________________________________//
  92. static void
  93. collect_dependant_siblings( test_unit_id from, test_unit_id to, test_unit_id master_tu_id, order_info_per_tu& tuoi )
  94. {
  95. int from_depth = tu_depth( from, master_tu_id, tuoi );
  96. int to_depth = tu_depth( to, master_tu_id, tuoi );
  97. while(from_depth > to_depth) {
  98. from = get_tu_parent( from );
  99. --from_depth;
  100. }
  101. while(from_depth < to_depth) {
  102. to = get_tu_parent( to );
  103. --to_depth;
  104. }
  105. while(true) {
  106. test_unit_id from_parent = get_tu_parent( from );
  107. test_unit_id to_parent = get_tu_parent( to );
  108. if( from_parent == to_parent )
  109. break;
  110. from = from_parent;
  111. to = to_parent;
  112. }
  113. tuoi[from].dependant_siblings.push_back( to );
  114. }
  115. //____________________________________________________________________________//
  116. static counter_t
  117. assign_sibling_rank( test_unit_id tu_id, order_info_per_tu& tuoi )
  118. {
  119. test_unit& tu = framework::get( tu_id, TUT_ANY );
  120. BOOST_TEST_SETUP_ASSERT( tu.p_sibling_rank != (std::numeric_limits<counter_t>::max)(),
  121. "Cyclic dependency detected involving test unit \"" + tu.full_name() + "\"" );
  122. if( tu.p_sibling_rank != 0 )
  123. return tu.p_sibling_rank;
  124. order_info const& info = tuoi[tu_id];
  125. // indicate in progress
  126. tu.p_sibling_rank.value = (std::numeric_limits<counter_t>::max)();
  127. counter_t new_rank = 1;
  128. BOOST_TEST_FOREACH( test_unit_id, sibling_id, info.dependant_siblings )
  129. new_rank = (std::max)(new_rank, assign_sibling_rank( sibling_id, tuoi ) + 1);
  130. return tu.p_sibling_rank.value = new_rank;
  131. }
  132. //____________________________________________________________________________//
  133. // ************************************************************************** //
  134. // ************** test_init call wrapper ************** //
  135. // ************************************************************************** //
  136. static void
  137. invoke_init_func( init_unit_test_func init_func )
  138. {
  139. #ifdef BOOST_TEST_ALTERNATIVE_INIT_API
  140. BOOST_TEST_I_ASSRT( (*init_func)(), std::runtime_error( "test module initialization failed" ) );
  141. #else
  142. test_suite* manual_test_units = (*init_func)( framework::master_test_suite().argc, framework::master_test_suite().argv );
  143. if( manual_test_units )
  144. framework::master_test_suite().add( manual_test_units );
  145. #endif
  146. }
  147. // ************************************************************************** //
  148. // ************** name_filter ************** //
  149. // ************************************************************************** //
  150. class name_filter : public test_tree_visitor {
  151. struct component {
  152. component( const_string name ) // has to be implicit
  153. {
  154. if( name == "*" )
  155. m_kind = SFK_ALL;
  156. else if( first_char( name ) == '*' && last_char( name ) == '*' ) {
  157. m_kind = SFK_SUBSTR;
  158. m_name = name.substr( 1, name.size()-1 );
  159. }
  160. else if( first_char( name ) == '*' ) {
  161. m_kind = SFK_TRAILING;
  162. m_name = name.substr( 1 );
  163. }
  164. else if( last_char( name ) == '*' ) {
  165. m_kind = SFK_LEADING;
  166. m_name = name.substr( 0, name.size()-1 );
  167. }
  168. else {
  169. m_kind = SFK_MATCH;
  170. m_name = name;
  171. }
  172. }
  173. bool pass( test_unit const& tu ) const
  174. {
  175. const_string name( tu.p_name );
  176. switch( m_kind ) {
  177. default:
  178. case SFK_ALL:
  179. return true;
  180. case SFK_LEADING:
  181. return name.substr( 0, m_name.size() ) == m_name;
  182. case SFK_TRAILING:
  183. return name.size() >= m_name.size() && name.substr( name.size() - m_name.size() ) == m_name;
  184. case SFK_SUBSTR:
  185. return name.find( m_name ) != const_string::npos;
  186. case SFK_MATCH:
  187. return m_name == tu.p_name.get();
  188. }
  189. }
  190. enum kind { SFK_ALL, SFK_LEADING, SFK_TRAILING, SFK_SUBSTR, SFK_MATCH };
  191. kind m_kind;
  192. const_string m_name;
  193. };
  194. public:
  195. // Constructor
  196. name_filter( test_unit_id_list& targ_list, const_string filter_expr ) : m_targ_list( targ_list ), m_depth( 0 )
  197. {
  198. #ifdef BOOST_TEST_SUPPORT_TOKEN_ITERATOR
  199. utils::string_token_iterator tit( filter_expr, (utils::dropped_delimeters = "/",
  200. utils::kept_delimeters = utils::dt_none) );
  201. while( tit != utils::string_token_iterator() ) {
  202. m_components.push_back(
  203. std::vector<component>( utils::string_token_iterator( *tit, (utils::dropped_delimeters = ",",
  204. utils::kept_delimeters = utils::dt_none) ),
  205. utils::string_token_iterator() ) );
  206. ++tit;
  207. }
  208. #endif
  209. }
  210. private:
  211. bool filter_unit( test_unit const& tu )
  212. {
  213. // skip master test suite
  214. if( m_depth == 0 )
  215. return true;
  216. // corresponding name filters are at level m_depth-1
  217. std::vector<component> const& filters = m_components[m_depth-1];
  218. // look for match
  219. using namespace boost::placeholders;
  220. return std::find_if( filters.begin(), filters.end(), bind( &component::pass, _1, boost::ref(tu) ) ) != filters.end();
  221. }
  222. // test_tree_visitor interface
  223. virtual void visit( test_case const& tc )
  224. {
  225. // make sure we only accept test cases if we match last component of the filter
  226. if( m_depth == m_components.size() && filter_unit( tc ) )
  227. m_targ_list.push_back( tc.p_id ); // found a test case
  228. }
  229. virtual bool test_suite_start( test_suite const& ts )
  230. {
  231. if( !filter_unit( ts ) )
  232. return false;
  233. if( m_depth < m_components.size() ) {
  234. ++m_depth;
  235. return true;
  236. }
  237. m_targ_list.push_back( ts.p_id ); // found a test suite
  238. return false;
  239. }
  240. virtual void test_suite_finish( test_suite const& /*ts*/ )
  241. {
  242. --m_depth;
  243. }
  244. // Data members
  245. typedef std::vector<std::vector<component> > components_per_level;
  246. components_per_level m_components;
  247. test_unit_id_list& m_targ_list;
  248. unsigned m_depth;
  249. };
  250. // ************************************************************************** //
  251. // ************** label_filter ************** //
  252. // ************************************************************************** //
  253. class label_filter : public test_tree_visitor {
  254. public:
  255. label_filter( test_unit_id_list& targ_list, const_string label )
  256. : m_targ_list( targ_list )
  257. , m_label( label )
  258. {}
  259. private:
  260. // test_tree_visitor interface
  261. virtual bool visit( test_unit const& tu )
  262. {
  263. if( tu.has_label( m_label ) ) {
  264. // found a test unit; add it to list of tu to enable with children and stop recursion in case of suites
  265. m_targ_list.push_back( tu.p_id );
  266. return false;
  267. }
  268. return true;
  269. }
  270. // Data members
  271. test_unit_id_list& m_targ_list;
  272. const_string m_label;
  273. };
  274. // ************************************************************************** //
  275. // ************** set_run_status ************** //
  276. // ************************************************************************** //
  277. class set_run_status : public test_tree_visitor {
  278. public:
  279. explicit set_run_status( test_unit::run_status rs, test_unit_id_list* dep_collector = 0 )
  280. : m_new_status( rs )
  281. , m_dep_collector( dep_collector )
  282. {}
  283. // test_tree_visitor interface
  284. virtual bool visit( test_unit const& tu )
  285. {
  286. const_cast<test_unit&>(tu).p_run_status.value = m_new_status == test_unit::RS_INVALID ? tu.p_default_status : m_new_status;
  287. if( m_dep_collector ) {
  288. BOOST_TEST_FOREACH( test_unit_id, dep_id, tu.p_dependencies.get() ) {
  289. test_unit const& dep = framework::get( dep_id, TUT_ANY );
  290. if( dep.p_run_status == tu.p_run_status )
  291. continue;
  292. BOOST_TEST_FRAMEWORK_MESSAGE( "Including test " << dep.p_type_name << ' ' << dep.full_name() <<
  293. " as a dependency of test " << tu.p_type_name << ' ' << tu.full_name() );
  294. m_dep_collector->push_back( dep_id );
  295. }
  296. }
  297. return true;
  298. }
  299. private:
  300. // Data members
  301. test_unit::run_status m_new_status;
  302. test_unit_id_list* m_dep_collector;
  303. };
  304. // ************************************************************************** //
  305. // ************** parse_filters ************** //
  306. // ************************************************************************** //
  307. static void
  308. add_filtered_test_units( test_unit_id master_tu_id, const_string filter, test_unit_id_list& targ )
  309. {
  310. // Choose between two kinds of filters
  311. if( filter[0] == '@' ) {
  312. filter.trim_left( 1 );
  313. label_filter lf( targ, filter );
  314. traverse_test_tree( master_tu_id, lf, true );
  315. }
  316. else {
  317. name_filter nf( targ, filter );
  318. traverse_test_tree( master_tu_id, nf, true );
  319. }
  320. }
  321. //____________________________________________________________________________//
  322. static bool
  323. parse_filters( test_unit_id master_tu_id, test_unit_id_list& tu_to_enable, test_unit_id_list& tu_to_disable )
  324. {
  325. // 10. collect tu to enable and disable based on filters
  326. bool had_selector_filter = false;
  327. std::vector<std::string> const& filters = runtime_config::get<std::vector<std::string> >( runtime_config::btrt_run_filters );
  328. BOOST_TEST_FOREACH( const_string, filter, filters ) {
  329. BOOST_TEST_SETUP_ASSERT( !filter.is_empty(), "Invalid filter specification" );
  330. // each --run_test command may also be separated by a ':' (environment variable)
  331. utils::string_token_iterator t_filter_it( filter, (utils::dropped_delimeters = ":",
  332. utils::kept_delimeters = utils::dt_none) );
  333. while( t_filter_it != utils::string_token_iterator() ) {
  334. const_string filter_token = *t_filter_it;
  335. enum { SELECTOR, ENABLER, DISABLER } filter_type = SELECTOR;
  336. // 11. Deduce filter type
  337. if( filter_token[0] == '!' || filter_token[0] == '+' ) {
  338. filter_type = filter_token[0] == '+' ? ENABLER : DISABLER;
  339. filter_token.trim_left( 1 );
  340. BOOST_TEST_SETUP_ASSERT( !filter_token.is_empty(), "Invalid filter specification" );
  341. }
  342. had_selector_filter |= filter_type == SELECTOR;
  343. // 12. Add test units to corresponding list
  344. switch( filter_type ) {
  345. case SELECTOR:
  346. case ENABLER: add_filtered_test_units( master_tu_id, filter_token, tu_to_enable ); break;
  347. case DISABLER: add_filtered_test_units( master_tu_id, filter_token, tu_to_disable ); break;
  348. }
  349. ++t_filter_it;
  350. }
  351. }
  352. return had_selector_filter;
  353. }
  354. //____________________________________________________________________________//
  355. #ifdef BOOST_NO_CXX98_RANDOM_SHUFFLE
  356. // a poor man's implementation of random_shuffle
  357. template< class RandomIt, class RandomFunc >
  358. void random_shuffle( RandomIt first, RandomIt last, RandomFunc &r )
  359. {
  360. typedef typename std::iterator_traits<RandomIt>::difference_type difference_type;
  361. difference_type n = last - first;
  362. for (difference_type i = n-1; i > 0; --i) {
  363. difference_type j = r(i+1);
  364. if (j != i) {
  365. using std::swap;
  366. swap(first[i], first[j]);
  367. }
  368. }
  369. }
  370. #endif
  371. // A simple handle for registering the global fixtures to the master test suite
  372. // without deleting an existing static object (the global fixture itself) when the program
  373. // terminates (shared_ptr).
  374. class global_fixture_handle : public test_unit_fixture {
  375. public:
  376. global_fixture_handle(test_unit_fixture* fixture) : m_global_fixture(fixture) {}
  377. ~global_fixture_handle() {}
  378. virtual void setup() {
  379. m_global_fixture->setup();
  380. }
  381. virtual void teardown() {
  382. m_global_fixture->teardown();
  383. }
  384. private:
  385. test_unit_fixture* m_global_fixture;
  386. };
  387. } // namespace impl
  388. // ************************************************************************** //
  389. // ************** framework::state ************** //
  390. // ************************************************************************** //
  391. unsigned long int const TIMEOUT_EXCEEDED = static_cast<unsigned long int>( -1 );
  392. class state {
  393. public:
  394. state()
  395. : m_master_test_suite( 0 )
  396. , m_curr_test_unit( INV_TEST_UNIT_ID )
  397. , m_next_test_case_id( MIN_TEST_CASE_ID )
  398. , m_next_test_suite_id( MIN_TEST_SUITE_ID )
  399. , m_test_in_progress( false )
  400. , m_context_idx( 0 )
  401. , m_log_sinks( )
  402. , m_report_sink( std::cerr )
  403. {
  404. }
  405. ~state() { clear(); }
  406. void clear()
  407. {
  408. while( !m_test_units.empty() ) {
  409. test_unit_store::value_type const& tu = *m_test_units.begin();
  410. test_unit const* tu_ptr = tu.second;
  411. // the delete will erase this element from map
  412. if( ut_detail::test_id_2_unit_type( tu.second->p_id ) == TUT_SUITE )
  413. delete static_cast<test_suite const*>(tu_ptr);
  414. else
  415. delete static_cast<test_case const*>(tu_ptr);
  416. }
  417. }
  418. void set_tu_id( test_unit& tu, test_unit_id id ) { tu.p_id.value = id; }
  419. //////////////////////////////////////////////////////////////////
  420. // Validates the dependency graph and deduces the sibling dependency rank for each child
  421. void deduce_siblings_order( test_unit_id tu_id, test_unit_id master_tu_id, impl::order_info_per_tu& tuoi )
  422. {
  423. test_unit& tu = framework::get( tu_id, TUT_ANY );
  424. // collect all sibling dependancy from tu own list
  425. BOOST_TEST_FOREACH( test_unit_id, dep_id, tu.p_dependencies.get() )
  426. collect_dependant_siblings( tu_id, dep_id, master_tu_id, tuoi );
  427. if( tu.p_type != TUT_SUITE )
  428. return;
  429. test_suite& ts = static_cast<test_suite&>(tu);
  430. // recursive call to children first
  431. BOOST_TEST_FOREACH( test_unit_id, chld_id, ts.m_children )
  432. deduce_siblings_order( chld_id, master_tu_id, tuoi );
  433. ts.m_ranked_children.clear();
  434. BOOST_TEST_FOREACH( test_unit_id, chld_id, ts.m_children ) {
  435. counter_t rank = assign_sibling_rank( chld_id, tuoi );
  436. ts.m_ranked_children.insert( std::make_pair( rank, chld_id ) );
  437. }
  438. }
  439. //////////////////////////////////////////////////////////////////
  440. // Finalize default run status:
  441. // 1) inherit run status from parent where applicable
  442. // 2) if any of test units in test suite enabled enable it as well
  443. bool finalize_default_run_status( test_unit_id tu_id, test_unit::run_status parent_status )
  444. {
  445. test_unit& tu = framework::get( tu_id, TUT_ANY );
  446. if( tu.p_default_status == test_suite::RS_INHERIT )
  447. tu.p_default_status.value = parent_status;
  448. // go through list of children
  449. if( tu.p_type == TUT_SUITE ) {
  450. bool has_enabled_child = false;
  451. BOOST_TEST_FOREACH( test_unit_id, chld_id, static_cast<test_suite const&>(tu).m_children )
  452. has_enabled_child |= finalize_default_run_status( chld_id, tu.p_default_status );
  453. tu.p_default_status.value = has_enabled_child ? test_suite::RS_ENABLED : test_suite::RS_DISABLED;
  454. }
  455. return tu.p_default_status == test_suite::RS_ENABLED;
  456. }
  457. //////////////////////////////////////////////////////////////////
  458. bool finalize_run_status( test_unit_id tu_id )
  459. {
  460. test_unit& tu = framework::get( tu_id, TUT_ANY );
  461. // go through list of children
  462. if( tu.p_type == TUT_SUITE ) {
  463. bool has_enabled_child = false;
  464. BOOST_TEST_FOREACH( test_unit_id, chld_id, static_cast<test_suite const&>(tu).m_children)
  465. has_enabled_child |= finalize_run_status( chld_id );
  466. tu.p_run_status.value = has_enabled_child ? test_suite::RS_ENABLED : test_suite::RS_DISABLED;
  467. }
  468. return tu.is_enabled();
  469. }
  470. //////////////////////////////////////////////////////////////////
  471. void deduce_run_status( test_unit_id master_tu_id )
  472. {
  473. using namespace framework::impl;
  474. test_unit_id_list tu_to_enable;
  475. test_unit_id_list tu_to_disable;
  476. // 10. If there are any filters supplied, figure out lists of test units to enable/disable
  477. bool had_selector_filter = !runtime_config::get<std::vector<std::string> >( runtime_config::btrt_run_filters ).empty() &&
  478. parse_filters( master_tu_id, tu_to_enable, tu_to_disable );
  479. // 20. Set the stage: either use default run status or disable all test units
  480. set_run_status initial_setter( had_selector_filter ? test_unit::RS_DISABLED : test_unit::RS_INVALID );
  481. traverse_test_tree( master_tu_id, initial_setter, true );
  482. // 30. Apply all selectors and enablers.
  483. while( !tu_to_enable.empty() ) {
  484. test_unit& tu = framework::get( tu_to_enable.back(), TUT_ANY );
  485. tu_to_enable.pop_back();
  486. // 35. Ignore test units which are already enabled
  487. if( tu.is_enabled() )
  488. continue;
  489. // set new status and add all dependencies into tu_to_enable
  490. set_run_status enabler( test_unit::RS_ENABLED, &tu_to_enable );
  491. traverse_test_tree( tu.p_id, enabler, true );
  492. // Add the dependencies of the parent suites, see trac #13149
  493. test_unit_id parent_id = tu.p_parent_id;
  494. while( parent_id != INV_TEST_UNIT_ID
  495. && parent_id != master_tu_id )
  496. {
  497. // we do not use the traverse_test_tree as otherwise it would enable the sibblings and subtree
  498. // of the test case we want to enable (we need to enable the parent suites and their dependencies only)
  499. // the parent_id needs to be enabled in order to be properly parsed by finalize_run_status, the visit
  500. // does the job
  501. test_unit& tu_parent = framework::get( parent_id, TUT_ANY );
  502. enabler.visit( tu_parent );
  503. parent_id = tu_parent.p_parent_id;
  504. }
  505. }
  506. // 40. Apply all disablers
  507. while( !tu_to_disable.empty() ) {
  508. test_unit const& tu = framework::get( tu_to_disable.back(), TUT_ANY );
  509. tu_to_disable.pop_back();
  510. // 35. Ignore test units which already disabled
  511. if( !tu.is_enabled() )
  512. continue;
  513. set_run_status disabler( test_unit::RS_DISABLED );
  514. traverse_test_tree( tu.p_id, disabler, true );
  515. }
  516. // 50. Make sure parents of enabled test units are also enabled
  517. finalize_run_status( master_tu_id );
  518. }
  519. //////////////////////////////////////////////////////////////////
  520. typedef unit_test_monitor_t::error_level execution_result;
  521. // Random generator using the std::rand function (seeded prior to the call)
  522. struct random_generator_helper {
  523. size_t operator()(size_t i) const {
  524. return std::rand() % i;
  525. }
  526. };
  527. // Executes the test tree with the root at specified test unit
  528. execution_result execute_test_tree( test_unit_id tu_id,
  529. unsigned long int timeout_microseconds = 0,
  530. random_generator_helper const * const p_random_generator = 0)
  531. {
  532. test_unit const& tu = framework::get( tu_id, TUT_ANY );
  533. execution_result result = unit_test_monitor_t::test_ok;
  534. if( !tu.is_enabled() )
  535. return result;
  536. // 10. Check preconditions, including zero time left for execution and
  537. // successful execution of all dependencies
  538. if( timeout_microseconds == TIMEOUT_EXCEEDED ) {
  539. // notify all observers about skipped test unit
  540. BOOST_TEST_FOREACH( test_observer*, to, m_observers )
  541. to->test_unit_skipped( tu, "timeout for the test unit is exceeded" );
  542. return unit_test_monitor_t::os_timeout;
  543. }
  544. else if( timeout_microseconds == 0 || (tu.p_timeout > 0 && timeout_microseconds > (tu.p_timeout * 1000000) ) ) // deduce timeout for this test unit
  545. timeout_microseconds = tu.p_timeout * 1000000;
  546. test_tools::assertion_result const precondition_res = tu.check_preconditions();
  547. if( !precondition_res ) {
  548. // notify all observers about skipped test unit
  549. BOOST_TEST_FOREACH( test_observer*, to, m_observers )
  550. to->test_unit_skipped( tu, precondition_res.message() );
  551. // It is not an error to skip the test if any of the parent tests
  552. // have failed. This one should be reported as skipped as if it was
  553. // disabled
  554. return unit_test_monitor_t::test_ok;
  555. }
  556. // 20. Notify all observers about the start of the test unit
  557. BOOST_TEST_FOREACH( test_observer*, to, m_observers )
  558. to->test_unit_start( tu );
  559. // 30. Execute setup fixtures if any; any failure here leads to test unit abortion
  560. BOOST_TEST_FOREACH( test_unit_fixture_ptr, F, tu.p_fixtures.get() ) {
  561. ut_detail::test_unit_id_restore restore_current_test_unit(m_curr_test_unit, tu.p_id);
  562. result = unit_test_monitor.execute_and_translate( boost::bind( &test_unit_fixture::setup, F ) );
  563. if( result != unit_test_monitor_t::test_ok )
  564. break;
  565. test_results const& test_rslt = unit_test::results_collector.results( m_curr_test_unit );
  566. if( test_rslt.aborted() ) {
  567. result = unit_test_monitor_t::test_setup_failure;
  568. break;
  569. }
  570. }
  571. // This is the time we are going to spend executing the test unit (in microseconds
  572. // as expected by test_observer::test_unit_finish)
  573. unsigned long elapsed_microseconds = 0;
  574. if( result == unit_test_monitor_t::test_ok ) {
  575. // 40. We are going to time the execution
  576. boost::unit_test::timer::timer tu_timer;
  577. // we pass the random generator
  578. const random_generator_helper& rand_gen = p_random_generator ? *p_random_generator : random_generator_helper();
  579. if( tu.p_type == TUT_SUITE ) {
  580. test_suite const& ts = static_cast<test_suite const&>( tu );
  581. if( runtime_config::get<unsigned>( runtime_config::btrt_random_seed ) == 0 ) {
  582. typedef std::pair<counter_t,test_unit_id> value_type;
  583. BOOST_TEST_FOREACH( value_type, chld, ts.m_ranked_children ) {
  584. // tu_timer.elapsed() returns nanosec, timeout and child_timeout in microsec
  585. unsigned long int chld_timeout = child_timeout(
  586. timeout_microseconds,
  587. static_cast<unsigned long int>( microsecond_wall_time(tu_timer.elapsed()) ));
  588. result = (std::min)( result, execute_test_tree( chld.second, chld_timeout, &rand_gen ) );
  589. if( unit_test_monitor.is_critical_error( result ) )
  590. break;
  591. // we check for the time elapsed. If this is too high, we fail the current suite and return from here
  592. elapsed_microseconds = static_cast<unsigned long int>( microsecond_wall_time(tu_timer.elapsed()) );
  593. if( (timeout_microseconds > 0) && (elapsed_microseconds > timeout_microseconds) && (timeout_microseconds != TIMEOUT_EXCEEDED ) ) {
  594. BOOST_TEST_FOREACH( test_observer*, to, m_observers ) {
  595. to->test_unit_timed_out(tu);
  596. }
  597. result = (std::min)( result, unit_test_monitor_t::os_timeout );
  598. timeout_microseconds = TIMEOUT_EXCEEDED;
  599. //break;
  600. // we continue to explore the children, such that we can at least update their
  601. // status to skipped
  602. }
  603. }
  604. }
  605. else {
  606. // Go through ranges of children with the same dependency rank and shuffle them
  607. // independently. Execute each subtree in this order
  608. test_unit_id_list children_with_the_same_rank;
  609. typedef test_suite::children_per_rank::const_iterator it_type;
  610. it_type it = ts.m_ranked_children.begin();
  611. while( it != ts.m_ranked_children.end() ) {
  612. children_with_the_same_rank.clear();
  613. std::pair<it_type,it_type> range = ts.m_ranked_children.equal_range( it->first );
  614. it = range.first;
  615. while( it != range.second ) {
  616. children_with_the_same_rank.push_back( it->second );
  617. it++;
  618. }
  619. #ifdef BOOST_NO_CXX98_RANDOM_SHUFFLE
  620. impl::random_shuffle( children_with_the_same_rank.begin(), children_with_the_same_rank.end(), rand_gen );
  621. #else
  622. std::random_shuffle( children_with_the_same_rank.begin(), children_with_the_same_rank.end(), rand_gen );
  623. #endif
  624. BOOST_TEST_FOREACH( test_unit_id, chld, children_with_the_same_rank ) {
  625. unsigned long int chld_timeout = child_timeout(
  626. timeout_microseconds,
  627. static_cast<unsigned long int>(microsecond_wall_time(tu_timer.elapsed())) );
  628. result = (std::min)( result, execute_test_tree( chld, chld_timeout, &rand_gen ) );
  629. if( unit_test_monitor.is_critical_error( result ) )
  630. break;
  631. // we check for the time elapsed. If this is too high, we fail the current suite and return from here
  632. elapsed_microseconds = static_cast<unsigned long int>( microsecond_wall_time(tu_timer.elapsed()) );
  633. if( (timeout_microseconds > 0) && (elapsed_microseconds > timeout_microseconds) && (timeout_microseconds != TIMEOUT_EXCEEDED ) ) {
  634. BOOST_TEST_FOREACH( test_observer*, to, m_observers ) {
  635. to->test_unit_timed_out(tu);
  636. }
  637. result = (std::min)( result, unit_test_monitor_t::os_timeout );
  638. timeout_microseconds = TIMEOUT_EXCEEDED;
  639. //break;
  640. // we continue to explore the children, such that we can at least update their
  641. // status to skipped
  642. }
  643. }
  644. }
  645. }
  646. }
  647. else { // TUT_CASE
  648. test_case const& tc = static_cast<test_case const&>( tu );
  649. // setup contexts
  650. m_context_idx = 0;
  651. // setup current test case
  652. ut_detail::test_unit_id_restore restore_current_test_unit(m_curr_test_unit, tc.p_id);
  653. // execute the test case body, transforms the time out to seconds
  654. result = unit_test_monitor.execute_and_translate( tc.p_test_func, timeout_microseconds );
  655. elapsed_microseconds = static_cast<unsigned long int>( microsecond_wall_time(tu_timer.elapsed()) );
  656. // cleanup leftover context
  657. m_context.clear();
  658. // restore state (scope exit) and abort if necessary
  659. }
  660. }
  661. // if run error is critical skip teardown, who knows what the state of the program at this point
  662. if( !unit_test_monitor.is_critical_error( result ) ) {
  663. // execute teardown fixtures if any in reverse order
  664. BOOST_TEST_REVERSE_FOREACH( test_unit_fixture_ptr, F, tu.p_fixtures.get() ) {
  665. ut_detail::test_unit_id_restore restore_current_test_unit(m_curr_test_unit, tu.p_id);
  666. result = (std::min)( result, unit_test_monitor.execute_and_translate( boost::bind( &test_unit_fixture::teardown, F ), 0 ) );
  667. if( unit_test_monitor.is_critical_error( result ) )
  668. break;
  669. }
  670. }
  671. // notify all observers about abortion
  672. if( unit_test_monitor.is_critical_error( result ) ) {
  673. BOOST_TEST_FOREACH( test_observer*, to, m_observers )
  674. to->test_aborted();
  675. }
  676. // notify all observers about completion
  677. BOOST_TEST_REVERSE_FOREACH( test_observer*, to, m_observers )
  678. to->test_unit_finish( tu, elapsed_microseconds );
  679. return result;
  680. }
  681. //////////////////////////////////////////////////////////////////
  682. unsigned long int child_timeout( unsigned long tu_timeout_microseconds, unsigned long elpsed_microsec )
  683. {
  684. if( tu_timeout_microseconds == 0UL || tu_timeout_microseconds == TIMEOUT_EXCEEDED)
  685. return tu_timeout_microseconds;
  686. return tu_timeout_microseconds > elpsed_microsec ?
  687. tu_timeout_microseconds - elpsed_microsec
  688. : TIMEOUT_EXCEEDED;
  689. }
  690. struct priority_order {
  691. bool operator()( test_observer* lhs, test_observer* rhs ) const
  692. {
  693. return (lhs->priority() < rhs->priority()) || ((lhs->priority() == rhs->priority()) && (lhs < rhs));
  694. }
  695. };
  696. // Data members
  697. typedef std::map<test_unit_id,test_unit*> test_unit_store;
  698. typedef std::set<test_observer*,priority_order> observer_store;
  699. struct context_frame {
  700. context_frame( std::string const& d, int id, bool sticky )
  701. : descr( d )
  702. , frame_id( id )
  703. , is_sticky( sticky )
  704. {}
  705. std::string descr;
  706. int frame_id;
  707. bool is_sticky;
  708. };
  709. typedef std::vector<context_frame> context_data;
  710. master_test_suite_t* m_master_test_suite;
  711. std::vector<test_suite*> m_auto_test_suites;
  712. test_unit_id m_curr_test_unit;
  713. test_unit_store m_test_units;
  714. test_unit_id m_next_test_case_id;
  715. test_unit_id m_next_test_suite_id;
  716. bool m_test_in_progress;
  717. observer_store m_observers;
  718. context_data m_context;
  719. int m_context_idx;
  720. std::set<global_fixture*> m_global_fixtures;
  721. boost::execution_monitor m_aux_em;
  722. std::map<output_format, runtime_config::stream_holder> m_log_sinks;
  723. runtime_config::stream_holder m_report_sink;
  724. };
  725. //____________________________________________________________________________//
  726. namespace impl {
  727. namespace {
  728. #if defined(__CYGWIN__)
  729. framework::state& s_frk_state() { static framework::state* the_inst = 0; if(!the_inst) the_inst = new framework::state; return *the_inst; }
  730. #else
  731. framework::state& s_frk_state() { static framework::state the_inst; return the_inst; }
  732. #endif
  733. } // local namespace
  734. void
  735. setup_for_execution( test_unit const& tu )
  736. {
  737. s_frk_state().deduce_run_status( tu.p_id );
  738. }
  739. struct sum_to_first_only {
  740. sum_to_first_only() : is_first(true) {}
  741. template <class T, class U>
  742. T operator()(T const& l_, U const& r_) {
  743. if(is_first) {
  744. is_first = false;
  745. return l_ + r_.first;
  746. }
  747. return l_ + ", " + r_.first;
  748. }
  749. bool is_first;
  750. };
  751. void
  752. shutdown_loggers_and_reports()
  753. {
  754. s_frk_state().m_log_sinks.clear();
  755. s_frk_state().m_report_sink.setup( "stderr" );
  756. }
  757. void
  758. unregister_global_fixture_and_configuration()
  759. {
  760. // we make a copy as the set will change in the iteration
  761. std::set<global_fixture*> gfixture_copy(s_frk_state().m_global_fixtures);
  762. BOOST_TEST_FOREACH( global_fixture*, tuf, gfixture_copy ) {
  763. tuf->unregister_from_framework();
  764. }
  765. s_frk_state().m_global_fixtures.clear();
  766. state::observer_store gobserver_copy(s_frk_state().m_observers);
  767. BOOST_TEST_FOREACH( test_observer*, to, gobserver_copy ) {
  768. framework::deregister_observer( *to );
  769. }
  770. s_frk_state().m_observers.clear();
  771. }
  772. void
  773. setup_loggers()
  774. {
  775. BOOST_TEST_I_TRY {
  776. #ifdef BOOST_TEST_SUPPORT_TOKEN_ITERATOR
  777. bool has_combined_logger = runtime_config::has( runtime_config::btrt_combined_logger )
  778. && !runtime_config::get< std::vector<std::string> >( runtime_config::btrt_combined_logger ).empty();
  779. #else
  780. bool has_combined_logger = false;
  781. #endif
  782. if( !has_combined_logger ) {
  783. unit_test_log.set_threshold_level( runtime_config::get<log_level>( runtime_config::btrt_log_level ) );
  784. const output_format format = runtime_config::get<output_format>( runtime_config::btrt_log_format );
  785. unit_test_log.set_format( format );
  786. runtime_config::stream_holder& stream_logger = s_frk_state().m_log_sinks[format];
  787. if( runtime_config::has( runtime_config::btrt_log_sink ) ) {
  788. // we remove all streams in this case, so we do not specify the format
  789. boost::function< void () > log_cleaner = boost::bind( &unit_test_log_t::set_stream,
  790. &unit_test_log,
  791. boost::ref(std::cout)
  792. );
  793. stream_logger.setup( runtime_config::get<std::string>( runtime_config::btrt_log_sink ),
  794. log_cleaner );
  795. }
  796. unit_test_log.set_stream( stream_logger.ref() );
  797. }
  798. else
  799. {
  800. const std::vector<std::string>& v_output_format = runtime_config::get< std::vector<std::string> >( runtime_config::btrt_combined_logger ) ;
  801. static const std::pair<const char*, log_level> all_log_levels[] = {
  802. std::make_pair( "all" , log_successful_tests ),
  803. std::make_pair( "success" , log_successful_tests ),
  804. std::make_pair( "test_suite" , log_test_units ),
  805. std::make_pair( "unit_scope" , log_test_units ),
  806. std::make_pair( "message" , log_messages ),
  807. std::make_pair( "warning" , log_warnings ),
  808. std::make_pair( "error" , log_all_errors ),
  809. std::make_pair( "cpp_exception" , log_cpp_exception_errors ),
  810. std::make_pair( "system_error" , log_system_errors ),
  811. std::make_pair( "fatal_error" , log_fatal_errors ),
  812. std::make_pair( "nothing" , log_nothing )
  813. };
  814. static const std::pair<const char*, output_format> all_formats[] = {
  815. std::make_pair( "HRF" , OF_CLF ),
  816. std::make_pair( "CLF" , OF_CLF ),
  817. std::make_pair( "XML" , OF_XML ),
  818. std::make_pair( "JUNIT", OF_JUNIT )
  819. };
  820. bool is_first = true;
  821. BOOST_TEST_FOREACH( const_string, current_multi_config, v_output_format ) {
  822. #ifdef BOOST_TEST_SUPPORT_TOKEN_ITERATOR
  823. // ':' may be used for file names: C:/tmp/mylogsink.xml
  824. // we merge the tokens that start with / or \ with the previous one.
  825. std::vector<std::string> v_processed_tokens;
  826. {
  827. utils::string_token_iterator current_config( current_multi_config, (utils::dropped_delimeters = ":",
  828. utils::kept_delimeters = utils::dt_none) );
  829. for( ; current_config != utils::string_token_iterator() ; ++current_config) {
  830. std::string str_copy(current_config->begin(), current_config->end());
  831. if( ( str_copy[0] == '\\' || str_copy[0] == '/' )
  832. && v_processed_tokens.size() > 0) {
  833. v_processed_tokens.back() += ":" + str_copy; // ':' has been eaten up
  834. }
  835. else {
  836. v_processed_tokens.push_back(str_copy);
  837. }
  838. }
  839. }
  840. BOOST_TEST_FOREACH( std::string const&, current_config, v_processed_tokens ) {
  841. utils::string_token_iterator current_format_specs( current_config, (utils::keep_empty_tokens,
  842. utils::dropped_delimeters = ",",
  843. utils::kept_delimeters = utils::dt_none) );
  844. output_format format = OF_INVALID ; // default
  845. if( current_format_specs != utils::string_token_iterator() &&
  846. current_format_specs->size() ) {
  847. for(size_t elem=0; elem < sizeof(all_formats)/sizeof(all_formats[0]); elem++) {
  848. if(const_string(all_formats[elem].first) == *current_format_specs) {
  849. format = all_formats[elem].second;
  850. break;
  851. }
  852. }
  853. }
  854. BOOST_TEST_I_ASSRT( format != OF_INVALID,
  855. boost::runtime::access_to_missing_argument()
  856. << "Unable to determine the logger type from '"
  857. << current_config
  858. << "'. Possible choices are: "
  859. << std::accumulate(all_formats,
  860. all_formats + sizeof(all_formats)/sizeof(all_formats[0]),
  861. std::string(""),
  862. sum_to_first_only())
  863. );
  864. // activates this format
  865. if( is_first ) {
  866. unit_test_log.set_format( format );
  867. }
  868. else {
  869. unit_test_log.add_format( format );
  870. }
  871. is_first = false;
  872. unit_test_log_formatter * const formatter = unit_test_log.get_formatter(format);
  873. BOOST_TEST_SETUP_ASSERT( formatter, "Logger setup error" );
  874. log_level formatter_log_level = invalid_log_level;
  875. ++current_format_specs ;
  876. if( !current_format_specs->size() ) {
  877. formatter_log_level = formatter->get_log_level(); // default log level given by the formatter
  878. }
  879. else if( current_format_specs != utils::string_token_iterator() ) {
  880. for(size_t elem=0; elem < sizeof(all_log_levels)/sizeof(all_log_levels[0]); elem++) {
  881. if(const_string(all_log_levels[elem].first) == *current_format_specs) {
  882. formatter_log_level = all_log_levels[elem].second;
  883. break;
  884. }
  885. }
  886. }
  887. BOOST_TEST_I_ASSRT( formatter_log_level != invalid_log_level,
  888. boost::runtime::access_to_missing_argument()
  889. << "Unable to determine the log level from '"
  890. << current_config
  891. << "'. Possible choices are: "
  892. << std::accumulate(all_log_levels,
  893. all_log_levels + sizeof(all_log_levels)/sizeof(all_log_levels[0]),
  894. std::string(""),
  895. sum_to_first_only())
  896. );
  897. unit_test_log.set_threshold_level( format, formatter_log_level );
  898. runtime_config::stream_holder& stream_logger = s_frk_state().m_log_sinks[format];
  899. boost::function< void () > log_cleaner = boost::bind( &unit_test_log_t::set_stream,
  900. &unit_test_log,
  901. format,
  902. boost::ref(std::cout) );
  903. if( ++current_format_specs != utils::string_token_iterator() &&
  904. current_format_specs->size() ) {
  905. stream_logger.setup( *current_format_specs,
  906. log_cleaner );
  907. }
  908. else {
  909. stream_logger.setup( formatter->get_default_stream_description(),
  910. log_cleaner );
  911. }
  912. unit_test_log.set_stream( format, stream_logger.ref() );
  913. }
  914. #endif
  915. } // for each logger
  916. } // if/else new logger API
  917. } // BOOST_TEST_I_TRY
  918. BOOST_TEST_I_CATCH( boost::runtime::init_error, ex ) {
  919. BOOST_TEST_SETUP_ASSERT( false, ex.msg );
  920. }
  921. BOOST_TEST_I_CATCH( boost::runtime::input_error, ex ) {
  922. std::cerr << ex.msg << "\n\n";
  923. BOOST_TEST_I_THROW( framework::nothing_to_test( boost::exit_exception_failure ) );
  924. }
  925. }
  926. //____________________________________________________________________________//
  927. } // namespace impl
  928. //____________________________________________________________________________//
  929. // ************************************************************************** //
  930. // ************** framework::init ************** //
  931. // ************************************************************************** //
  932. void
  933. init( init_unit_test_func init_func, int argc, char* argv[] )
  934. {
  935. using namespace impl;
  936. // 10. Set up runtime parameters
  937. runtime_config::init( argc, argv );
  938. // 20. Set the desired log level, format and sink
  939. impl::setup_loggers();
  940. // 30. Set the desired report level, format and sink
  941. results_reporter::set_level( runtime_config::get<report_level>( runtime_config::btrt_report_level ) );
  942. results_reporter::set_format( runtime_config::get<output_format>( runtime_config::btrt_report_format ) );
  943. if( runtime_config::has( runtime_config::btrt_report_sink ) ) {
  944. boost::function< void () > report_cleaner = boost::bind( &results_reporter::set_stream,
  945. boost::ref(std::cerr)
  946. );
  947. s_frk_state().m_report_sink.setup( runtime_config::get<std::string>( runtime_config::btrt_report_sink ),
  948. report_cleaner );
  949. }
  950. results_reporter::set_stream( s_frk_state().m_report_sink.ref() );
  951. // 40. Register default test observers
  952. register_observer( results_collector );
  953. register_observer( unit_test_log );
  954. register_observer( framework_init_observer );
  955. if( runtime_config::get<bool>( runtime_config::btrt_show_progress ) ) {
  956. progress_monitor.set_stream( std::cout ); // defaults to stdout
  957. register_observer( progress_monitor );
  958. }
  959. // 50. Set up memory leak detection
  960. unsigned long detect_mem_leak = runtime_config::get<unsigned long>( runtime_config::btrt_detect_mem_leaks );
  961. if( detect_mem_leak > 0 ) {
  962. debug::detect_memory_leaks( true, runtime_config::get<std::string>( runtime_config::btrt_report_mem_leaks ) );
  963. debug::break_memory_alloc( (long)detect_mem_leak );
  964. }
  965. // 60. Initialize master unit test suite
  966. master_test_suite().argc = argc;
  967. master_test_suite().argv = argv;
  968. // 70. Invoke test module initialization routine
  969. BOOST_TEST_I_TRY {
  970. s_frk_state().m_aux_em.vexecute( boost::bind( &impl::invoke_init_func, init_func ) );
  971. }
  972. BOOST_TEST_I_CATCH( execution_exception, ex ) {
  973. BOOST_TEST_SETUP_ASSERT( false, ex.what() );
  974. }
  975. }
  976. //____________________________________________________________________________//
  977. void
  978. finalize_setup_phase( test_unit_id master_tu_id )
  979. {
  980. if( master_tu_id == INV_TEST_UNIT_ID )
  981. master_tu_id = master_test_suite().p_id;
  982. // 10. Apply all decorators to the auto test units
  983. // 10. checks for consistency (duplicate names, etc)
  984. class apply_decorators : public test_tree_visitor {
  985. private:
  986. // test_tree_visitor interface
  987. virtual bool test_suite_start( test_suite const& ts)
  988. {
  989. const_cast<test_suite&>(ts).generate();
  990. const_cast<test_suite&>(ts).check_for_duplicate_test_cases();
  991. return test_tree_visitor::test_suite_start(ts);
  992. }
  993. virtual bool visit( test_unit const& tu )
  994. {
  995. BOOST_TEST_FOREACH( decorator::base_ptr, d, tu.p_decorators.get() )
  996. d->apply( const_cast<test_unit&>(tu) );
  997. return true;
  998. }
  999. } ad;
  1000. traverse_test_tree( master_tu_id, ad, true );
  1001. // 20. Finalize setup phase
  1002. impl::order_info_per_tu tuoi;
  1003. impl::s_frk_state().deduce_siblings_order( master_tu_id, master_tu_id, tuoi );
  1004. impl::s_frk_state().finalize_default_run_status( master_tu_id, test_unit::RS_INVALID );
  1005. }
  1006. // ************************************************************************** //
  1007. // ************** test_in_progress ************** //
  1008. // ************************************************************************** //
  1009. bool
  1010. test_in_progress()
  1011. {
  1012. return impl::s_frk_state().m_test_in_progress;
  1013. }
  1014. //____________________________________________________________________________//
  1015. // ************************************************************************** //
  1016. // ************** framework::shutdown ************** //
  1017. // ************************************************************************** //
  1018. void
  1019. shutdown()
  1020. {
  1021. // shuts down the loggers singleton to avoid any further reference to the
  1022. // framework during the destruction of those
  1023. impl::shutdown_loggers_and_reports();
  1024. // unregisters any global fixture and configuration object
  1025. impl::unregister_global_fixture_and_configuration();
  1026. // eliminating some fake memory leak reports. See for more details:
  1027. // http://connect.microsoft.com/VisualStudio/feedback/details/106937/memory-leaks-reported-by-debug-crt-inside-typeinfo-name
  1028. # if BOOST_WORKAROUND(BOOST_MSVC, <= 1600 ) && !defined(_DLL) && defined(_DEBUG)
  1029. # if BOOST_WORKAROUND(BOOST_MSVC, < 1600 )
  1030. #define _Next next
  1031. #define _MemPtr memPtr
  1032. #endif
  1033. __type_info_node* pNode = __type_info_root_node._Next;
  1034. __type_info_node* tmpNode = &__type_info_root_node;
  1035. for( ; pNode!=NULL; pNode = tmpNode ) {
  1036. tmpNode = pNode->_Next;
  1037. delete pNode->_MemPtr;
  1038. delete pNode;
  1039. }
  1040. # if BOOST_WORKAROUND(BOOST_MSVC, < 1600 )
  1041. #undef _Next
  1042. #undef _MemPtr
  1043. #endif
  1044. # endif
  1045. }
  1046. //____________________________________________________________________________//
  1047. // ************************************************************************** //
  1048. // ************** register_test_unit ************** //
  1049. // ************************************************************************** //
  1050. void
  1051. register_test_unit( test_case* tc )
  1052. {
  1053. BOOST_TEST_SETUP_ASSERT( tc->p_id == INV_TEST_UNIT_ID, BOOST_TEST_L( "test case already registered" ) );
  1054. test_unit_id new_id = impl::s_frk_state().m_next_test_case_id;
  1055. BOOST_TEST_SETUP_ASSERT( new_id != MAX_TEST_CASE_ID, BOOST_TEST_L( "too many test cases" ) );
  1056. typedef state::test_unit_store::value_type map_value_type;
  1057. impl::s_frk_state().m_test_units.insert( map_value_type( new_id, tc ) );
  1058. impl::s_frk_state().m_next_test_case_id++;
  1059. impl::s_frk_state().set_tu_id( *tc, new_id );
  1060. }
  1061. //____________________________________________________________________________//
  1062. // ************************************************************************** //
  1063. // ************** register_test_unit ************** //
  1064. // ************************************************************************** //
  1065. void
  1066. register_test_unit( test_suite* ts )
  1067. {
  1068. BOOST_TEST_SETUP_ASSERT( ts->p_id == INV_TEST_UNIT_ID, BOOST_TEST_L( "test suite already registered" ) );
  1069. test_unit_id new_id = impl::s_frk_state().m_next_test_suite_id;
  1070. BOOST_TEST_SETUP_ASSERT( new_id != MAX_TEST_SUITE_ID, BOOST_TEST_L( "too many test suites" ) );
  1071. typedef state::test_unit_store::value_type map_value_type;
  1072. impl::s_frk_state().m_test_units.insert( map_value_type( new_id, ts ) );
  1073. impl::s_frk_state().m_next_test_suite_id++;
  1074. impl::s_frk_state().set_tu_id( *ts, new_id );
  1075. }
  1076. //____________________________________________________________________________//
  1077. // ************************************************************************** //
  1078. // ************** deregister_test_unit ************** //
  1079. // ************************************************************************** //
  1080. void
  1081. deregister_test_unit( test_unit* tu )
  1082. {
  1083. impl::s_frk_state().m_test_units.erase( tu->p_id );
  1084. }
  1085. //____________________________________________________________________________//
  1086. // ************************************************************************** //
  1087. // ************** clear ************** //
  1088. // ************************************************************************** //
  1089. void
  1090. clear()
  1091. {
  1092. impl::s_frk_state().clear();
  1093. }
  1094. //____________________________________________________________________________//
  1095. // ************************************************************************** //
  1096. // ************** register_observer ************** //
  1097. // ************************************************************************** //
  1098. void
  1099. register_observer( test_observer& to )
  1100. {
  1101. impl::s_frk_state().m_observers.insert( &to );
  1102. }
  1103. //____________________________________________________________________________//
  1104. // ************************************************************************** //
  1105. // ************** deregister_observer ************** //
  1106. // ************************************************************************** //
  1107. void
  1108. deregister_observer( test_observer& to )
  1109. {
  1110. impl::s_frk_state().m_observers.erase( &to );
  1111. }
  1112. //____________________________________________________________________________//
  1113. // ************************************************************************** //
  1114. // ************** register_global_fixture ************** //
  1115. // ************************************************************************** //
  1116. void
  1117. register_global_fixture( global_fixture& tuf )
  1118. {
  1119. impl::s_frk_state().m_global_fixtures.insert( &tuf );
  1120. }
  1121. //____________________________________________________________________________//
  1122. // ************************************************************************** //
  1123. // ************** deregister_global_fixture ************** //
  1124. // ************************************************************************** //
  1125. void
  1126. deregister_global_fixture( global_fixture &tuf )
  1127. {
  1128. impl::s_frk_state().m_global_fixtures.erase( &tuf );
  1129. }
  1130. //____________________________________________________________________________//
  1131. // ************************************************************************** //
  1132. // ************** add_context ************** //
  1133. // ************************************************************************** //
  1134. int
  1135. add_context( ::boost::unit_test::lazy_ostream const& context_descr, bool sticky )
  1136. {
  1137. std::stringstream buffer;
  1138. context_descr( buffer );
  1139. int res_idx = impl::s_frk_state().m_context_idx++;
  1140. impl::s_frk_state().m_context.push_back( state::context_frame( buffer.str(), res_idx, sticky ) );
  1141. return res_idx;
  1142. }
  1143. //____________________________________________________________________________//
  1144. // ************************************************************************** //
  1145. // ************** clear_context ************** //
  1146. // ************************************************************************** //
  1147. struct frame_with_id {
  1148. explicit frame_with_id( int id ) : m_id( id ) {}
  1149. bool operator()( state::context_frame const& f )
  1150. {
  1151. return f.frame_id == m_id;
  1152. }
  1153. int m_id;
  1154. };
  1155. //____________________________________________________________________________//
  1156. void
  1157. clear_context( int frame_id )
  1158. {
  1159. if( frame_id == -1 ) { // clear all non sticky frames
  1160. for( int i=static_cast<int>(impl::s_frk_state().m_context.size())-1; i>=0; i-- )
  1161. if( !impl::s_frk_state().m_context[i].is_sticky )
  1162. impl::s_frk_state().m_context.erase( impl::s_frk_state().m_context.begin()+i );
  1163. }
  1164. else { // clear specific frame
  1165. state::context_data::iterator it =
  1166. std::find_if( impl::s_frk_state().m_context.begin(), impl::s_frk_state().m_context.end(), frame_with_id( frame_id ) );
  1167. if( it != impl::s_frk_state().m_context.end() ) // really an internal error if this is not true
  1168. impl::s_frk_state().m_context.erase( it );
  1169. }
  1170. }
  1171. //____________________________________________________________________________//
  1172. // ************************************************************************** //
  1173. // ************** get_context ************** //
  1174. // ************************************************************************** //
  1175. context_generator
  1176. get_context()
  1177. {
  1178. return context_generator();
  1179. }
  1180. //____________________________________________________________________________//
  1181. // ************************************************************************** //
  1182. // ************** context_generator ************** //
  1183. // ************************************************************************** //
  1184. bool
  1185. context_generator::is_empty() const
  1186. {
  1187. return impl::s_frk_state().m_context.empty();
  1188. }
  1189. //____________________________________________________________________________//
  1190. const_string
  1191. context_generator::next() const
  1192. {
  1193. return m_curr_frame < impl::s_frk_state().m_context.size() ? impl::s_frk_state().m_context[m_curr_frame++].descr : const_string();
  1194. }
  1195. //____________________________________________________________________________//
  1196. // ************************************************************************** //
  1197. // ************** master_test_suite ************** //
  1198. // ************************************************************************** //
  1199. master_test_suite_t&
  1200. master_test_suite()
  1201. {
  1202. if( !impl::s_frk_state().m_master_test_suite )
  1203. impl::s_frk_state().m_master_test_suite = new master_test_suite_t;
  1204. return *impl::s_frk_state().m_master_test_suite;
  1205. }
  1206. //____________________________________________________________________________//
  1207. // ************************************************************************** //
  1208. // ************** current_auto_test_suite ************** //
  1209. // ************************************************************************** //
  1210. test_suite&
  1211. current_auto_test_suite( test_suite* ts, bool push_or_pop )
  1212. {
  1213. if( impl::s_frk_state().m_auto_test_suites.empty() )
  1214. impl::s_frk_state().m_auto_test_suites.push_back( &framework::master_test_suite() );
  1215. if( !push_or_pop )
  1216. impl::s_frk_state().m_auto_test_suites.pop_back();
  1217. else if( ts )
  1218. impl::s_frk_state().m_auto_test_suites.push_back( ts );
  1219. return *impl::s_frk_state().m_auto_test_suites.back();
  1220. }
  1221. //____________________________________________________________________________//
  1222. // ************************************************************************** //
  1223. // ************** current_test_case ************** //
  1224. // ************************************************************************** //
  1225. test_case const&
  1226. current_test_case()
  1227. {
  1228. return get<test_case>( impl::s_frk_state().m_curr_test_unit );
  1229. }
  1230. test_unit const&
  1231. current_test_unit()
  1232. {
  1233. return *impl::s_frk_state().m_test_units[impl::s_frk_state().m_curr_test_unit];
  1234. }
  1235. //____________________________________________________________________________//
  1236. test_unit_id
  1237. current_test_case_id()
  1238. {
  1239. return impl::s_frk_state().m_curr_test_unit;
  1240. }
  1241. //____________________________________________________________________________//
  1242. // ************************************************************************** //
  1243. // ************** framework::get ************** //
  1244. // ************************************************************************** //
  1245. test_unit&
  1246. get( test_unit_id id, test_unit_type t )
  1247. {
  1248. test_unit* res = impl::s_frk_state().m_test_units[id];
  1249. BOOST_TEST_I_ASSRT( (res->p_type & t) != 0, internal_error( "Invalid test unit type" ) );
  1250. return *res;
  1251. }
  1252. //____________________________________________________________________________//
  1253. // ************************************************************************** //
  1254. // ************** framework::run ************** //
  1255. // ************************************************************************** //
  1256. template <class Cont>
  1257. struct swap_on_delete {
  1258. swap_on_delete(Cont& c1, Cont& c2) : m_c1(c1), m_c2(c2){}
  1259. ~swap_on_delete() {
  1260. m_c1.swap(m_c2);
  1261. }
  1262. Cont& m_c1;
  1263. Cont& m_c2;
  1264. };
  1265. void
  1266. run( test_unit_id id, bool continue_test )
  1267. {
  1268. if( id == INV_TEST_UNIT_ID )
  1269. id = master_test_suite().p_id;
  1270. // Figure out run status for execution phase
  1271. impl::s_frk_state().deduce_run_status( id );
  1272. test_case_counter tcc;
  1273. traverse_test_tree( id, tcc );
  1274. BOOST_TEST_SETUP_ASSERT( tcc.p_count != 0 , runtime_config::get<std::vector<std::string> >( runtime_config::btrt_run_filters ).empty()
  1275. ? BOOST_TEST_L( "test tree is empty" )
  1276. : BOOST_TEST_L( "no test cases matching filter or all test cases were disabled" ) );
  1277. bool was_in_progress = framework::test_in_progress();
  1278. bool call_start_finish = !continue_test || !was_in_progress;
  1279. bool init_ok = true;
  1280. const_string setup_error;
  1281. if( call_start_finish ) {
  1282. // indicates the framework that no test is in progress now if observers need to be notified
  1283. impl::s_frk_state().m_test_in_progress = false;
  1284. // unit_test::framework_init_observer will get cleared first
  1285. BOOST_TEST_FOREACH( test_observer*, to, impl::s_frk_state().m_observers ) {
  1286. BOOST_TEST_I_TRY {
  1287. ut_detail::test_unit_id_restore restore_current_test_unit(impl::s_frk_state().m_curr_test_unit, id);
  1288. unit_test_monitor_t::error_level result = unit_test_monitor.execute_and_translate( boost::bind( &test_observer::test_start, to, tcc.p_count ) );
  1289. if( init_ok ) {
  1290. if( result != unit_test_monitor_t::test_ok ) {
  1291. init_ok = false;
  1292. }
  1293. else {
  1294. if( unit_test::framework_init_observer.has_failed() ) {
  1295. init_ok = false;
  1296. }
  1297. }
  1298. }
  1299. }
  1300. BOOST_TEST_I_CATCH( execution_exception, ex ) {
  1301. if( init_ok ) {
  1302. // log only the first error
  1303. init_ok = false;
  1304. setup_error = ex.what();
  1305. }
  1306. // break; // we should continue otherwise loggers may have improper structure (XML start missing for instance)
  1307. }
  1308. }
  1309. }
  1310. if( init_ok ) {
  1311. // attaching the global fixtures to the main entry point
  1312. test_unit& entry_test_unit = framework::get( id, TUT_ANY );
  1313. std::vector<test_unit_fixture_ptr> v_saved_fixture(entry_test_unit.p_fixtures.value.begin(),
  1314. entry_test_unit.p_fixtures.value.end());
  1315. BOOST_TEST_FOREACH( test_unit_fixture*, tuf, impl::s_frk_state().m_global_fixtures ) {
  1316. entry_test_unit.p_fixtures.value.insert( entry_test_unit.p_fixtures.value.begin(),
  1317. test_unit_fixture_ptr(new impl::global_fixture_handle(tuf)) );
  1318. }
  1319. swap_on_delete< std::vector<test_unit_fixture_ptr> > raii_fixture(v_saved_fixture, entry_test_unit.p_fixtures.value);
  1320. // now work in progress
  1321. impl::s_frk_state().m_test_in_progress = true;
  1322. unsigned seed = runtime_config::get<unsigned>( runtime_config::btrt_random_seed );
  1323. switch( seed ) {
  1324. case 0:
  1325. break;
  1326. case 1:
  1327. seed = static_cast<unsigned>( std::rand() ^ std::time( 0 ) ); // better init using std::rand() ^ ...
  1328. BOOST_FALLTHROUGH;
  1329. default:
  1330. BOOST_TEST_FRAMEWORK_MESSAGE( "Test cases order is shuffled using seed: " << seed );
  1331. std::srand( seed );
  1332. }
  1333. // executing the test tree
  1334. impl::s_frk_state().execute_test_tree( id );
  1335. // removing previously added global fixtures: dtor raii_fixture
  1336. }
  1337. impl::s_frk_state().m_test_in_progress = false;
  1338. results_reporter::make_report( INV_REPORT_LEVEL, id );
  1339. unit_test::framework_init_observer.clear();
  1340. if( call_start_finish ) {
  1341. // indicates the framework that no test is in progress anymore if observers need to be notified
  1342. // and this is a teardown, so assertions should not raise any exception otherwise an exception
  1343. // might be raised in a dtor of a global fixture
  1344. impl::s_frk_state().m_test_in_progress = false;
  1345. BOOST_TEST_REVERSE_FOREACH( test_observer*, to, impl::s_frk_state().m_observers ) {
  1346. ut_detail::test_unit_id_restore restore_current_test_unit(impl::s_frk_state().m_curr_test_unit, id);
  1347. to->test_finish();
  1348. }
  1349. }
  1350. impl::s_frk_state().m_test_in_progress = was_in_progress;
  1351. // propagates the init/teardown error if any
  1352. BOOST_TEST_SETUP_ASSERT( init_ok && !unit_test::framework_init_observer.has_failed(), setup_error );
  1353. }
  1354. //____________________________________________________________________________//
  1355. void
  1356. run( test_unit const* tu, bool continue_test )
  1357. {
  1358. run( tu->p_id, continue_test );
  1359. }
  1360. //____________________________________________________________________________//
  1361. // ************************************************************************** //
  1362. // ************** assertion_result ************** //
  1363. // ************************************************************************** //
  1364. void
  1365. assertion_result( unit_test::assertion_result ar )
  1366. {
  1367. BOOST_TEST_FOREACH( test_observer*, to, impl::s_frk_state().m_observers )
  1368. to->assertion_result( ar );
  1369. }
  1370. //____________________________________________________________________________//
  1371. // ************************************************************************** //
  1372. // ************** exception_caught ************** //
  1373. // ************************************************************************** //
  1374. void
  1375. exception_caught( execution_exception const& ex )
  1376. {
  1377. BOOST_TEST_FOREACH( test_observer*, to, impl::s_frk_state().m_observers )
  1378. to->exception_caught( ex );
  1379. }
  1380. //____________________________________________________________________________//
  1381. // ************************************************************************** //
  1382. // ************** test_unit_aborted ************** //
  1383. // ************************************************************************** //
  1384. void
  1385. test_unit_aborted( test_unit const& tu )
  1386. {
  1387. BOOST_TEST_FOREACH( test_observer*, to, impl::s_frk_state().m_observers )
  1388. to->test_unit_aborted( tu );
  1389. }
  1390. // ************************************************************************** //
  1391. // ************** test_aborted ************** //
  1392. // ************************************************************************** //
  1393. void
  1394. test_aborted( )
  1395. {
  1396. BOOST_TEST_FOREACH( test_observer*, to, impl::s_frk_state().m_observers )
  1397. to->test_aborted( );
  1398. }
  1399. //____________________________________________________________________________//
  1400. } // namespace framework
  1401. } // namespace unit_test
  1402. } // namespace boost
  1403. #include <boost/test/detail/enable_warnings.hpp>
  1404. #endif // BOOST_TEST_FRAMEWORK_IPP_021005GER