From edaddc065a8123fc7fcbc6fdc68bbc5acdde20e7 Mon Sep 17 00:00:00 2001 From: vkatsuba Date: Tue, 22 Dec 2020 18:04:41 +0200 Subject: [PATCH] Mnesia migration: Added CT for migrate_pubsub_nodes, migrate_pubsub_subscriptions, migrate_pubsub_affiliations, migrate_pubsub_items, migrate_users, migrate_vcard_search, migrate_vcard, migrate_event_pusher_push_subscription --- big_tests/tests/mnesia_migration_SUITE.erl | 201 +++++++++++++++--- .../service_admin_extra_migration.erl | 5 +- src/pubsub/mod_pubsub_db_mnesia.erl | 12 +- 3 files changed, 185 insertions(+), 33 deletions(-) diff --git a/big_tests/tests/mnesia_migration_SUITE.erl b/big_tests/tests/mnesia_migration_SUITE.erl index b5c8a0b41c4..bcf8a5fbea1 100644 --- a/big_tests/tests/mnesia_migration_SUITE.erl +++ b/big_tests/tests/mnesia_migration_SUITE.erl @@ -15,13 +15,31 @@ %%% ================================================================== -define(RPC_MIGRATE(Act), mongoose_helper:successful_rpc(service_admin_extra_migration, migrate, [<<"mnesia">>, <<"rdbms">>, Act])). +-define(JID, jid:binary_to_bare(<<"migration_user@localhost/test">>)). +-define(NODE_NAME, <<"migration_node">>). +-define(USERNAME, <<"migration_user">>). +-define(NODE_HOST, ct:get_config({hosts, mim, domain})). +-define(BASE_NODE, {pubsub_node, {?NODE_HOST, ?NODE_NAME}, undefined, [], <<"flat">>, [?JID], []}). +-define(BASE_ITEM(Nidx), {pubsub_item, {?NODE_NAME, Nidx}, {erlang:timestamp(), ?JID}, {erlang:timestamp(),?JID}, ?JID, [{xmlcdata, []}]}). +-define(BASE_VCARD, {xmlel,<<"vCard">>, + [{<<"xmlns">>,<<"vcard-temp">>}], + [{xmlel,<<"NICKNAME">>,[],[{xmlcdata, ?USERNAME}]}, + {xmlel,<<"TEL">>,[], + [{xmlel,<<"HOME">>,[],[]}, + {xmlel,<<"VOICE">>,[],[]}, + {xmlel,<<"NUMBER">>,[], + [{xmlcdata,<<"+00000000000">>}]}]}]}). +-define(BASE_SEARCH_VCARD, {vcard_search, {<<"migration_user">>, <<"localhost">>}, <>, ?USERNAME, <<"fn">>, <<"lfn">>, <<"family">>, + <<"lfamily">>, <<"given">>, <<"lgiven">>, <<"middle">>, <<"lmiddle">>, <<"nickname">>, <<"lnickname">>, <<"bday">>, <<"lbday">>, + <<"ctry">>, <<"lctry">>, <<"locality">>, <<"llocality">>, <<"email">>, <<"lemail">>, <<"orgname">>, <<"lorgname">>, <<"orgunit">>, + <<"lorgunit">>}). %%-------------------------------------------------------------------- %% Suite configuration %%-------------------------------------------------------------------- all() -> - case (not ct_helper:is_ct_running()) orelse mongoose_helper:is_rdbms_enabled(ct:get_config({hosts, mim, domain})) of + case (not ct_helper:is_ct_running()) orelse mongoose_helper:is_rdbms_enabled(?NODE_HOST) of true -> tests(); false -> @@ -63,9 +81,18 @@ suite() -> escalus:suite(). init_per_suite(Config) -> + ok = mnesia:create_schema([node()]), + ok = mnesia:start(), + ok = mongoose_helper:successful_rpc(mod_pubsub_db_mnesia, start, []), + ok = mongoose_helper:successful_rpc(ejabberd_auth_internal, start, [?NODE_HOST]), + ok = mongoose_helper:successful_rpc(mod_vcard_mnesia, init, [<<>>, <<>>]), + ok = mongoose_helper:successful_rpc(mod_event_pusher_push_mnesia, init, [<<>>, <<>>]), + {ok, _} = application:ensure_all_started(jid), escalus:init_per_suite(Config). end_per_suite(Config) -> + ok = mongoose_helper:successful_rpc(ejabberd_auth_internal, stop, [?NODE_HOST]), + _ = mnesia:stop(), escalus_fresh:clean(), escalus:end_per_suite(Config). @@ -82,68 +109,143 @@ end_per_testcase(CaseName, Config) -> escalus:end_per_testcase(CaseName, Config). %%% ================================================================== -%%% Test case for migrate pubsub_nodes @TODO +%%% Test case for migrate pubsub_nodes %%% ================================================================== -migrate_pubsub_nodes(Config) -> - R = ?RPC_MIGRATE(<<"pubsub_nodes">>), - ct:comment("TEST CASE ~p", [{?FUNCTION_NAME, R}]). +migrate_pubsub_nodes(_Config) -> + Nidx = create_migration_node(), + {ok, _} = ?RPC_MIGRATE(<<"pubsub_nodes">>), + SqlData = #{table => <<"pubsub_nodes">>, where => <<"name='", ?NODE_NAME/binary, "'">>}, + case sql_to_rdbms(SqlData#{act => <<"SELECT">>, column => <<"nidx">>}) of + {selected, [{Nidx}]} -> + {updated, 1} = sql_to_rdbms(SqlData#{act => <<"DELETE">>, column => <<"">>}), + _ = clear_tables(), + ct:comment("Migration of 'pubsub_nodes' is successful for Nidx: ~p", [Nidx]); + Any -> + ct:fail("Unexpected result of 'pubsub_nodes' migration ~p~n", [Any]) + end. %%% ================================================================== -%%% Test case for migrate pubsub_subscriptions @TODO +%%% Test case for migrate pubsub_subscriptions %%% ================================================================== -migrate_pubsub_subscriptions(Config) -> - ?RPC_MIGRATE(<<"pubsub_subscriptions">>), - ct:comment("TEST CASE ~p", [?FUNCTION_NAME]). +migrate_pubsub_subscriptions(_Config) -> + Nidx = create_migration_node(), + ok = mongoose_helper:successful_rpc(mod_pubsub_db_mnesia, transaction, [#{name => add_subscription, args => [Nidx, ?JID, 'subscribed', <<"0000-0000-0000000">>, []]}]), + {ok, _} = ?RPC_MIGRATE(<<"pubsub_subscriptions">>), + SqlData = #{table => <<"pubsub_subscriptions">>, where => <<"nidx=", (list_to_binary(integer_to_list(Nidx)))/binary>>}, + case sql_to_rdbms(SqlData#{act => <<"SELECT">>, column => <<"nidx">>}) of + {selected, [{Nidx}]} -> + {updated, 1} = sql_to_rdbms(SqlData#{act => <<"DELETE">>, column => <<"">>}), + _ = clear_tables(), + ct:comment("Migration of 'pubsub_subscriptions' is successful for Nidx: ~p", [Nidx]); + Any -> + ct:fail("Unexpected result of 'pubsub_subscriptions' migration ~p~n", [Any]) + end. %%% ================================================================== -%%% Test case for migrate pubsub_affiliations @TODO +%%% Test case for migrate pubsub_affiliations %%% ================================================================== -migrate_pubsub_affiliations(Config) -> - ?RPC_MIGRATE(<<"pubsub_affiliations">>), - ct:comment("TEST CASE ~p", [?FUNCTION_NAME]). +migrate_pubsub_affiliations(_Config) -> + Nidx = create_migration_node(), + {ok, _} = ?RPC_MIGRATE(<<"pubsub_affiliations">>), + SqlData = #{table => <<"pubsub_affiliations">>, where => <<"nidx=", (list_to_binary(integer_to_list(Nidx)))/binary>>}, + case sql_to_rdbms(SqlData#{act => <<"SELECT">>, column => <<"nidx">>}) of + {selected, [{Nidx}]} -> + {updated, 1} = sql_to_rdbms(SqlData#{act => <<"DELETE">>, column => <<"">>}), + _ = clear_tables(), + ct:comment("Migration of 'pubsub_affiliations' is successful for Nidx: ~p", [Nidx]); + Any -> + ct:fail("Unexpected result of 'pubsub_affiliations' migration ~p~n", [Any]) + end. %%% ================================================================== -%%% Test case for migrate pubsub_items @TODO +%%% Test case for migrate pubsub_items %%% ================================================================== -migrate_pubsub_items(Config) -> - ?RPC_MIGRATE(<<"pubsub_items">>), - ct:comment("TEST CASE ~p", [?FUNCTION_NAME]). +migrate_pubsub_items(_Config) -> + Nidx = create_migration_node(), + ok = mongoose_helper:successful_rpc(mod_pubsub_db_mnesia, transaction, [#{name => add_item, args => [Nidx, ?JID, ?BASE_ITEM(Nidx)]}]), + {ok, _} = ?RPC_MIGRATE(<<"pubsub_items">>), + SqlData = #{table => <<"pubsub_items">>, where => <<"nidx=", (list_to_binary(integer_to_list(Nidx)))/binary>>}, + case sql_to_rdbms(SqlData#{act => <<"SELECT">>, column => <<"nidx">>}) of + {selected, [{Nidx}]} -> + {updated, 1} = sql_to_rdbms(SqlData#{act => <<"DELETE">>, column => <<"">>}), + _ = clear_tables(), + ct:comment("Migration of 'pubsub_items' is successful for Nidx: ~p", [Nidx]); + Any -> + ct:fail("Unexpected result of 'pubsub_items' migration ~p~n", [Any]) + end. %%% ================================================================== -%%% Test case for migrate users @TODO +%%% Test case for migrate users %%% ================================================================== -migrate_users(Config) -> - ?RPC_MIGRATE(<<"users">>), - ct:comment("TEST CASE ~p", [?FUNCTION_NAME]). +migrate_users(_Config) -> + ok = mongoose_helper:successful_rpc(ejabberd_auth_internal, set_password, [?USERNAME, <<"localhost">>, <<"migration_pass">>]), + {ok, _} = ?RPC_MIGRATE(<<"users">>), + SqlData = #{table => <<"users">>, where => <<"username='", ?USERNAME/binary, "'">>}, + case sql_to_rdbms(SqlData#{act => <<"SELECT">>, column => <<"username">>}) of + {selected, [{?USERNAME}]} -> + {updated, 1} = sql_to_rdbms(SqlData#{act => <<"DELETE">>, column => <<"">>}), + _ = clear_tables(), + ct:comment("Migration of 'users' is successful for Username: ~p", [?USERNAME]); + Any -> + ct:fail("Unexpected result of 'users' migration ~p~n", [Any]) + end. %%% ================================================================== -%%% Test case for migrate vcard_search @TODO +%%% Test case for migrate vcard_search %%% ================================================================== -migrate_vcard_search(Config) -> - ?RPC_MIGRATE(<<"vcard_search">>), - ct:comment("TEST CASE ~p", [?FUNCTION_NAME]). +migrate_vcard_search(_Config) -> + ok = set_vcard(), + {ok, _} = ?RPC_MIGRATE(<<"vcard_search">>), + SqlData = #{table => <<"vcard_search">>, where => <<"username='", ?USERNAME/binary, "'">>}, + case sql_to_rdbms(SqlData#{act => <<"SELECT">>, column => <<"username">>}) of + {selected, [{?USERNAME}]} -> + {updated, 1} = sql_to_rdbms(SqlData#{act => <<"DELETE">>, column => <<"">>}), + _ = clear_tables(), + ct:comment("Migration of 'vcard_search' is successful for Username: ~p", [?USERNAME]); + Any -> + ct:fail("Unexpected result of 'vcard_search' migration ~p~n", [Any]) + end. %%% ================================================================== -%%% Test case for migrate vcard @TODO +%%% Test case for migrate vcard %%% ================================================================== migrate_vcard(Config) -> - ?RPC_MIGRATE(<<"vcard">>), - ct:comment("TEST CASE ~p", [?FUNCTION_NAME]). + ok = set_vcard(), + {ok, _} = ?RPC_MIGRATE(<<"vcard">>), + SqlData = #{table => <<"vcard">>, where => <<"username='", ?USERNAME/binary, "'">>}, + case sql_to_rdbms(SqlData#{act => <<"SELECT">>, column => <<"username">>}) of + {selected, [{?USERNAME}]} -> + {updated, 1} = sql_to_rdbms(SqlData#{act => <<"DELETE">>, column => <<"">>}), + _ = clear_tables(), + ct:comment("Migration of 'vcard' is successful for Username: ~p", [?USERNAME]); + Any -> + ct:fail("Unexpected result of 'vcard' migration ~p~n", [Any]) + end. %%% ================================================================== -%%% Test case for migrate event_pusher_push_subscription @TODO +%%% Test case for migrate event_pusher_push_subscription %%% ================================================================== migrate_event_pusher_push_subscription(Config) -> - ?RPC_MIGRATE(<<"event_pusher_push_subscription">>), - ct:comment("TEST CASE ~p", [?FUNCTION_NAME]). + PubsubNode = <<"migration_pub_sub_node">>, + ok = mongoose_helper:successful_rpc(mod_event_pusher_push_mnesia, enable, [?JID, ?JID, PubsubNode, [{<<"name">>, <<"value">>}]]), + {ok, _} = ?RPC_MIGRATE(<<"event_pusher_push_subscription">>), + SqlData = #{table => <<"event_pusher_push_subscription">>, where => <<"node='", PubsubNode/binary, "'">>}, + case sql_to_rdbms(SqlData#{act => <<"SELECT">>, column => <<"node">>}) of + {selected, [{PubsubNode}]} -> + {updated, 1} = sql_to_rdbms(SqlData#{act => <<"DELETE">>, column => <<"">>}), + _ = clear_tables(), + ct:comment("Migration of 'event_pusher_push_subscription' is successful for Node: ~p", [PubsubNode]); + Any -> + ct:fail("Unexpected result of 'event_pusher_push_subscription' migration ~p~n", [Any]) + end. %%% ================================================================== %%% Test case for migrate rosterusers @TODO @@ -208,3 +310,40 @@ migrate_muc_light_rooms(Config) -> migrate_all(Config) -> ?RPC_MIGRATE(<<"all">>), ct:comment("TEST CASE ~p", [?FUNCTION_NAME]). + +%%% ================================================================== +%%% Internal functions +%%% ================================================================== + +sql_query(Query) -> + slow_rpc(mongoose_rdbms, sql_query, [?NODE_HOST, Query]). + +slow_rpc(M, F, A) -> + Node = ct:get_config({hosts, mim, node}), + Cookie = escalus_ct:get_config(ejabberd_cookie), + Res = escalus_rpc:call(Node, M, F, A, timer:seconds(30), Cookie), + case Res of + {badrpc, timeout} -> + {badrpc, {timeout, M, F}}; + _ -> + Res + end. + +sql_to_rdbms(#{act := Act, column := Column, table := Table, where := Where}) -> + SelectQuery = <>, + SelectResult = sql_query(SelectQuery). + +clear_tables() -> + Tables = [pubsub_state, pubsub_item, pubsub_node, + pubsub_subscription, pubsub_subnode, + passwd, reg_users_counter, + vcard, vcard_search, push_subscription], + _ = [mongoose_helper:successful_rpc(mod_pubsub_db_mnesia, clear_table, [T]) || T <- Tables]. + +create_migration_node() -> + {ok, Nidx} = mongoose_helper:successful_rpc(mod_pubsub_db_mnesia, transaction,[#{name => set_node, args => [?BASE_NODE]}]), + ok = mongoose_helper:successful_rpc(mod_pubsub_db_mnesia, transaction,[#{name => create_node, args => [Nidx, ?JID]}]), + Nidx. + +set_vcard() -> + mongoose_helper:successful_rpc(mod_vcard_mnesia, set_vcard, [?USERNAME, <<"localhost">>, ?BASE_VCARD, ?BASE_SEARCH_VCARD]). diff --git a/src/admin_extra/service_admin_extra_migration.erl b/src/admin_extra/service_admin_extra_migration.erl index 726afc101ca..f0fb1e0c48c 100644 --- a/src/admin_extra/service_admin_extra_migration.erl +++ b/src/admin_extra/service_admin_extra_migration.erl @@ -157,6 +157,9 @@ try_migrate({Act, Table}) when is_atom(Table) -> try_migrate({Act, Table, TableData}) -> case TableData of + {ok, [_] = Data, '$end_of_table'} -> + ?MODULE:(list_to_existing_atom("migrate_" ++ binary_to_list(Act)))(Data), + {ok, io_lib:format("Completed the migration of the table: '~s'", [Act])}; {ok, _, '$end_of_table'} -> {ok, io_lib:format("Completed the migration of the table: '~s'", [Act])}; {ok, Data, NextKey} -> @@ -276,7 +279,7 @@ migrate_pubsub_items([H|_]) -> Cols = ["nidx", "itemid", "created_luser", "created_lserver", "created_at", "modified_luser", "modified_lserver", "modified_lresource", "modified_at", "payload"], Vals = [NodeIdx, IID, JID#jid.luser, JID#jid.lserver, CT, JID#jid.lserver, - JID#jid.lserver, JID#jid.lresource, CT, XMLB], + JID#jid.lserver, JID#jid.lresource, CT, XMLB], Q = ["INSERT INTO pubsub_items ", expand_sql_vals(Cols, Vals), ";"], case mongoose_rdbms:sql_query(?MYNAME, Q) of {error, Reason} -> diff --git a/src/pubsub/mod_pubsub_db_mnesia.erl b/src/pubsub/mod_pubsub_db_mnesia.erl index a4eeba23d71..16c66d2420a 100644 --- a/src/pubsub/mod_pubsub_db_mnesia.erl +++ b/src/pubsub/mod_pubsub_db_mnesia.erl @@ -17,7 +17,7 @@ -export([start/0, stop/0]). % Funs execution --export([transaction/2, dirty/2]). +-export([clear_table/1, transaction/1, transaction/2, dirty/2]). % Direct #pubsub_state access -export([del_node/1, get_state/2, get_states/1, get_states_by_lus/1, get_states_by_bare/1, @@ -134,6 +134,16 @@ maybe_fill_subnode_table(_Other) -> %% ------------------------ Fun execution ------------------------ +-spec clear_table(atom()) -> + {atomic, ok} | {aborted, term()}. +clear_table(Tab) -> + mnesia:clear_table(Tab). + +-spec transaction(map()) -> + {result | error, any()}. +transaction(#{name := Name, args := Args}) -> + transaction(fun() -> apply(?MODULE, Name, Args) end, #{}). + -spec transaction(Fun :: fun(() -> {result | error, any()}), ErrorDebug :: map()) -> {result | error, any()}.