Not repeatable in maria-5.3-mwl89, though the query plan in that tree is different. At the very least, please push the test case into the tree so that it is run automatically.

In maria-5.3 , the following query

SELECT * FROM t2
WHERE ( f3 , f2 ) IN (SELECT f3 , f3 )
OR ( f1 , f3 , f1 ) NOT IN ( SELECT COUNT( DISTINCT f2 ), f1, f3 FROM t1 GROUP BY f1 , f3 );

causes the following assertion:

mysqld: ../include/my_bitmap.h:124: void bitmap_set_bit(MY_BITMAP*, uint): Assertion `bit < (map)->n_bits' failed.
1

backtrace:

Ordered_key::set_null(unsigned long long)+0x27) [0x824295b]
subselect_rowid_merge_engine::init(st_bitmap*, st_bitmap*)+0x6e1) [0x8241183]
subselect_hash_sj_engine::exec()+0x3cb) [0x823f7b7]
Item_subselect::exec()+0x83) [0x82374e1]
Item_in_subselect::exec()+0xec) [0x82376da]
Item_in_subselect::val_bool()+0x7d) [0x8239059]
Item::val_bool_result()+0x18) [0x81d70e6]
Item_in_optimizer::val_int()+0x253) [0x8202b43]
Item::val_int_result()+0x18) [0x81d7088]
Item_cache_int::cache_value()+0x36) [0x81d2db2]
Item_cache_wrapper::cache()+0x44) [0x81daccc]
Item_cache_wrapper::val_bool()+0xfb) [0x81d0b31]
Item_func_not::val_int()+0x51) [0x81fed3f]
Item::val_bool()+0x48) [0x81c0b9e]
Item_cond_or::val_int()+0x65) [0x820a841]
) [0x83242db]
JOIN::exec()+0x22f9) [0x830af57]
mysql_select(THD*, Item**, TABLE_LIST, unsigned int, List<Item>&, Item*, unsigned int, st_order*, st_order*, Item*, st_order*, unsigned long long, select_result*, st_select_lex_unit*, st_select_lex*)+0x3c2) [0x830b725]
handle_select(THD*, st_lex*, select_result*, unsigned long)+0x1cd) [0x8303fb1]
) [0x82a1a4f]
mysql_execute_command(THD*)+0x719) [0x82989cb]
mysql_parse(THD*, char*, unsigned int, char const**)+0x26f) [0x82a3fe7]
dispatch_command(enum_server_command, THD*, char*, unsigned int)+0x8bc) [0x8296663]
do_command(THD*)+0x271) [0x8295b10]
handle_one_connection+0x152) [0x8292bee]

explain in maria-5.3:

id select_type table type possible_keys key key_len ref rows Extra
1 PRIMARY t2 system NULL NULL NULL NULL 1
3 SUBQUERY t1 ALL NULL NULL NULL NULL 100 Using filesort
2 DEPENDENT SUBQUERY NULL NULL NULL NULL NULL NULL NULL No tables used

explain in maria-5.3-mwl89:

id select_type table type possible_keys key key_len ref rows Extra
1 PRIMARY t2 system NULL NULL NULL NULL 1
3 DEPENDENT SUBQUERY t1 ALL NULL NULL NULL NULL 100 Using filesort
2 DEPENDENT SUBQUERY NULL NULL NULL NULL NULL NULL NULL No tables used

test case:

CREATE TABLE t1 ( f1 int NOT NULL , f2 int, f3 int) ;
INSERT IGNORE INTO t1 VALUES ('1','6',NULL),('2','8','0'),('3','6','0'),('4','6','7'),('5','3','8'),('6','9','4'),('7','2','6'),('8','1','5'),('9','8',NULL),('10','0',NULL),('11','9','8'),('12','8','8'),('13',NULL,'8'),('14','0','1'),('15','5','1'),('16','1','5'),('17','1','2'),('18','6','7'),('19','6','6'),('20','1','4'),('21','8','7'),('22','1','0'),('23','0','9'),('24','4','3'),('25','9','5'),('26','8','1'),('27','5','1'),('28','9','5'),('29','5','9'),('30',NULL,'2'),('31',NULL,'5'),('32','105','248'),('33','0','0'),('34','3','8'),('35','1','1'),('36','75','255'),('37','9','9'),('38','7','9'),('39',NULL,'3'),('40',NULL,'9'),('41','4','6'),('42','0','4'),('43','204','60'),('44','0','7'),('45','9','1'),('46','8','6'),('47','7','4'),('48','8',NULL),('49',NULL,'8'),('50','6','0'),('51','5','8'),('52','2','151'),('53','3','7'),('54','7','6'),('55',NULL,NULL),('56','145','23'),('57','0','2'),('58','1','4'),('59','7',NULL),('60','3',NULL),('61',NULL,'77'),('62','2',NULL),('63','2',NULL),('64','2','7'),('65','8','1'),('66','6','9'),('67','1','6'),('68',NULL,'2'),('69','1','3'),('70','0','0'),('71','4',NULL),('72','1','6'),('73','1','3'),('74','27','195'),('75','4','5'),('76','6','2'),('77','2','7'),('78','248','25'),('79',NULL,NULL),('80','9','0'),('81','75','98'),('82','2','6'),('83','9','5'),('84','4','0'),('85','0','3'),('86','0','1'),('87','1','1'),('88','119','147'),('89','1','3'),('90','7','3'),('91','2',NULL),('92','7','2'),('93','2','1'),('94','6','8'),('95','4','8'),('96','5','8'),('97','7','0'),('98','1','1'),('99','7','8'),('100','2','5');

CREATE TABLE t2 ( f1 int NOT NULL , f2 int, f3 int) ;
INSERT IGNORE INTO t2 VALUES ('1000','6','2');

SELECT *
FROM t2
WHERE
( f3 , f2 ) IN (SELECT f3 , f3 )
OR ( f1 , f3 , f1 ) NOT IN (
SELECT
        COUNT( DISTINCT f2 ),
        f1, f3
        FROM t1
        GROUP BY f1 , f3
);